Hugging Face's logo

andyshu
/
opensysone

Text Classification
opensysone
decision-scoring
lora
experimental
Model card Files
xet
 Community
andyshu commited on
Commit
8cb06c7
·
verified ·
1 Parent(s): f2d6f8d

Back up completed OpenSysOne fleet deployment

Browse files
Files changed (17) hide show
  1. final/20260916T194403396250Z-fleet/backup_manifest.json +76 -0
  2. final/20260916T194403396250Z-fleet/evaluation/correctness.json +9 -0
  3. final/20260916T194403396250Z-fleet/evaluation/data_filter.json +100 -0
  4. final/20260916T194403396250Z-fleet/evaluation/manifest.json +96 -0
  5. final/20260916T194403396250Z-fleet/evaluation/metrics.json +2355 -0
  6. final/20260916T194403396250Z-fleet/fleet/api_probe.json +70 -0
  7. final/20260916T194403396250Z-fleet/fleet/deployment.json +10 -0
  8. final/20260916T194403396250Z-fleet/fleet/exit_code +1 -0
  9. final/20260916T194403396250Z-fleet/fleet/plan.json +76 -0
  10. final/20260916T194403396250Z-fleet/fleet/selection.json +0 -0
  11. final/20260916T194403396250Z-fleet/fleet/state.json +501 -0
  12. final/20260916T194403396250Z-fleet/model.pt +3 -0
  13. final/20260916T194403396250Z-fleet/selected/best.pt +3 -0
  14. final/20260916T194403396250Z-fleet/selected/training_correctness.json +9 -0
  15. final/20260916T194403396250Z-fleet/selected/training_manifest.json +416 -0
  16. final/20260916T194403396250Z-fleet/selected/validation_predictions.json +0 -0
  17. final/20260916T194403396250Z-fleet/source.tar.gz +3 -0
final/20260916T194403396250Z-fleet/backup_manifest.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet",
3
+ "checkpoint_sha256": "c4f781e80ade257544100b03c525709d0559eb2bf6a992c97708554c7d360aca",
4
+ "created_utc": "2026-09-17T09:20:42.481050+00:00",
5
+ "evaluation_source_commit": "07f10e791061a679b829ed1dc5b33897e001d67d",
6
+ "files": {
7
+ "evaluation/correctness.json": {
8
+ "sha256": "cd6b7a551708c15a09099a58d7863fc0eee17e8b5ff36fd3ed2fdb7147ffbf3a",
9
+ "size": 356
10
+ },
11
+ "evaluation/data_filter.json": {
12
+ "sha256": "f07eef84b3081ad86bb5b48f810bbed569a76ee8e82cba9228beec908232d79e",
13
+ "size": 2462
14
+ },
15
+ "evaluation/manifest.json": {
16
+ "sha256": "afa2f10d486209a016a22905b9c360aa3e17b283c4675ac169c4ae74bf96b2b2",
17
+ "size": 5269
18
+ },
19
+ "evaluation/metrics.json": {
20
+ "sha256": "b1c5cfa6e4e05672be6debe6af18e52f02fe1a2f095f94e5899ba8b32799ce35",
21
+ "size": 65136
22
+ },
23
+ "fleet/api_probe.json": {
24
+ "sha256": "453ab426a46e6a6ddb3d13c78500c4b75120a346544a1901f8e8b5da240b0b3c",
25
+ "size": 1948
26
+ },
27
+ "fleet/deployment.json": {
28
+ "sha256": "e9107d22a57d71b4210f56e87dbbdc4830acc99adb4ecbe216517ef5482ed0f7",
29
+ "size": 584
30
+ },
31
+ "fleet/exit_code": {
32
+ "sha256": "9a271f2a916b0b6ee6cecb2426f0b3206ef074578be55d9bc94f6f3fe3ab86aa",
33
+ "size": 2
34
+ },
35
+ "fleet/plan.json": {
36
+ "sha256": "169c8ca8dc371a13094668f8943a5d67f38e6950d6873c78b869b0436cba7255",
37
+ "size": 3255
38
+ },
39
+ "fleet/selection.json": {
40
+ "sha256": "53c8554ae5ff5627fd9664f1b7bc3c1f83b446a989cad88ffe3f72705a51b268",
41
+ "size": 178746
42
+ },
43
+ "fleet/state.json": {
44
+ "sha256": "a204da42dc55280344cd77b8596904c57bd0ec92a9078d6ea57dcaeed3101de3",
45
+ "size": 19042
46
+ },
47
+ "model.pt": {
48
+ "sha256": "e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348",
49
+ "size": 66221579
50
+ },
51
+ "selected/best.pt": {
52
+ "sha256": "c4f781e80ade257544100b03c525709d0559eb2bf6a992c97708554c7d360aca",
53
+ "size": 66220939
54
+ },
55
+ "selected/training_correctness.json": {
56
+ "sha256": "d5a9fc3e02e6e5d204d4c5154c21d30dcf92aa7621620f1ea7f64e824a4ecfef",
57
+ "size": 355
58
+ },
59
+ "selected/training_manifest.json": {
60
+ "sha256": "5a0437bf2e3bc4422a2ff45e18e4431ba916ac682d28934e82e6c6bfa0208772",
61
+ "size": 16521
62
+ },
63
+ "selected/validation_predictions.json": {
64
+ "sha256": "1e0a2f838251c8eb411b59f4fa061db67ec308ce249d166901760b097c64bd6c",
65
+ "size": 331142
66
+ },
67
+ "source.tar.gz": {
68
+ "sha256": "aecd66cb0bea0e1e7f1f275d46ffb173f40bd0095898f2a197b409d6243ca52e",
69
+ "size": 2115783
70
+ }
71
+ },
72
+ "format": 1,
73
+ "model_sha256": "e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348",
74
+ "scope": "Calibrated decision-scoring adapter/head and frozen selection evidence; base weights excluded.",
75
+ "source_commit": "6729461ccaad32e239c2148fa0c8f9ca23513a7a"
76
+ }
final/20260916T194403396250Z-fleet/evaluation/correctness.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "branch_chunks_1_probability_max_abs": 0.0,
3
+ "branch_chunks_2_probability_max_abs": 2.6496127247810364e-07,
4
+ "branch_chunks_4_probability_max_abs": 1.7369166016578674e-07,
5
+ "question_isolation_probability_max_abs": 0.0,
6
+ "candidate_permutation_probability_max_abs": 0.0,
7
+ "repeat_probability_max_abs": 0.0,
8
+ "tolerance_probability_abs": 0.0001
9
+ }
final/20260916T194403396250Z-fleet/evaluation/data_filter.json ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "calibration": {
3
+ "retained": 510,
4
+ "dropped_ids": [
5
+ "boolq:validation:200",
6
+ "boolq:validation:1836"
7
+ ],
8
+ "family_counts": {
9
+ "boolq": 126,
10
+ "snli": 128,
11
+ "banking": 128,
12
+ "arc": 128
13
+ },
14
+ "retained_id_sha256": "8a0d4add2dd95717d34915195dc7100878b8f443bf714656d54b336b629f6476",
15
+ "max_branch_tokens": 481
16
+ },
17
+ "holdout": {
18
+ "retained": 768,
19
+ "dropped_ids": [],
20
+ "family_counts": {
21
+ "social": 768
22
+ },
23
+ "retained_id_sha256": "805387bd9156d12d4d40b33e5926f209451f323814ad876a3f04a8abece3ae0b",
24
+ "max_branch_tokens": 123
25
+ },
26
+ "test": {
27
+ "retained": 2042,
28
+ "dropped_ids": [
29
+ "boolq:validation:1681",
30
+ "boolq:validation:1661",
31
+ "boolq:validation:2150",
32
+ "boolq:validation:2153",
33
+ "boolq:validation:3154",
34
+ "boolq:validation:561"
35
+ ],
36
+ "family_counts": {
37
+ "banking": 512,
38
+ "boolq": 506,
39
+ "arc": 512,
40
+ "snli": 512
41
+ },
42
+ "retained_id_sha256": "343960f63f954a0c05884459a13a3ef8560fd98c2caead3aa3c3b70022917ec5",
43
+ "max_branch_tokens": 440
44
+ },
45
+ "train": {
46
+ "retained": 80765,
47
+ "dropped_ids": [
48
+ "boolq:train:5085",
49
+ "boolq:train:1430",
50
+ "boolq:train:353",
51
+ "boolq:train:3547",
52
+ "boolq:train:5618",
53
+ "boolq:train:3872",
54
+ "boolq:train:6128",
55
+ "boolq:train:899",
56
+ "boolq:train:711",
57
+ "boolq:train:6969",
58
+ "boolq:train:7410",
59
+ "boolq:train:9405",
60
+ "boolq:train:3362",
61
+ "boolq:train:8317",
62
+ "boolq:train:4726",
63
+ "boolq:train:3163",
64
+ "boolq:train:7445",
65
+ "boolq:train:2181",
66
+ "boolq:train:8140",
67
+ "boolq:train:2141",
68
+ "boolq:train:204",
69
+ "boolq:train:1505",
70
+ "boolq:train:2352",
71
+ "boolq:train:9421",
72
+ "boolq:train:5517",
73
+ "boolq:train:6869",
74
+ "piqa:train:13223"
75
+ ],
76
+ "family_counts": {
77
+ "banking": 9608,
78
+ "snli": 20000,
79
+ "boolq": 7962,
80
+ "arc": 3345,
81
+ "hellaswag": 16000,
82
+ "piqa": 14360,
83
+ "commonsenseqa": 9490
84
+ },
85
+ "retained_id_sha256": "79e791c0e8b0f4312fdadcd62042a689d32d2bcf04c80419c8eebd94db99249c",
86
+ "max_branch_tokens": 509
87
+ },
88
+ "validation": {
89
+ "retained": 512,
90
+ "dropped_ids": [],
91
+ "family_counts": {
92
+ "boolq": 128,
93
+ "snli": 128,
94
+ "banking": 128,
95
+ "arc": 128
96
+ },
97
+ "retained_id_sha256": "b0ea1f550bee363d58849a00295771707fca92d9d8e630840d0f870a91a9f8a7",
98
+ "max_branch_tokens": 477
99
+ }
100
+ }
final/20260916T194403396250Z-fleet/evaluation/manifest.json ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "pid": 1673217,
3
+ "source_commit": "07f10e791061a679b829ed1dc5b33897e001d67d",
4
+ "checkpoint_sha256": "c4f781e80ade257544100b03c525709d0559eb2bf6a992c97708554c7d360aca",
5
+ "selected_step": 0,
6
+ "data_signature": "76183c642668602f42b7f3e71a3fe03bd5bd76f064fce4ba92351d8703396207",
7
+ "model_provenance": {
8
+ "model_id": "Qwen/Qwen3-4B-Instruct-2507",
9
+ "revision": "cdbee75f17c01a7cc42f958dc650907174af0554",
10
+ "license": "apache-2.0"
11
+ },
12
+ "training_config": {
13
+ "command": "train",
14
+ "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
15
+ "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917",
16
+ "output": "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts",
17
+ "resume": null,
18
+ "warm_start": "/home/andy/ai/opensysone/runs/20260917T070415Z-expanded-parent/best.pt",
19
+ "allow_train_data_change": true,
20
+ "steps": 8,
21
+ "epochs": 3,
22
+ "rank": 8,
23
+ "alpha": 16.0,
24
+ "head_only": false,
25
+ "two_pass": true,
26
+ "lr": 2e-05,
27
+ "head_lr": 2e-05,
28
+ "seed": 433,
29
+ "effective_batch": 4,
30
+ "branch_batch_size": 1,
31
+ "max_tokens": 512,
32
+ "save_steps": 250,
33
+ "save_seconds": 900,
34
+ "eval_steps": 500,
35
+ "validation_per_family": 128,
36
+ "patience": 8,
37
+ "deadline": "2026-09-17T16:00:00Z",
38
+ "schedule_steps": 3500,
39
+ "selection_metric": "crossfit_temperature_nll_v1",
40
+ "adapters": true
41
+ },
42
+ "training_source_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86",
43
+ "source_sha256": {
44
+ "playground.py": "b10c400421dd8558a7fef8ddde632676cdfe7f63edf94184f299f9ed569c010a",
45
+ "selection.py": "be0a7a8496b5b830aa572ceba93606320f442063fd38180503fd6980dc1c578f",
46
+ "training_model.py": "d5b0aefeeb5290816bc0b669aa0a8cbbe27f6a12b9cb23c141ac9b9ae9ee4e65",
47
+ "decision_model.py": "a3d8aeb02a1ac765c6cc30ff175acad0664560f01ab5403e22cade924d17371e",
48
+ "data_transition.py": "93aaa89b4de3aa78c34f03a5643e31f91f738c832395334966368df9b902621c",
49
+ "smoke_data.py": "06b3cbac1c8c4a86b8aecbee4459073cc3e46d4ddcd576392f3cb4805924f815",
50
+ "jev_harness.py": "4d4e979cb7ae352bcdacaaa6d64045e6b5e550b1a9721d4bad545045bee6c67f",
51
+ "experiment.py": "c779c3936aa1c2c51052f035df7bc0895a2de79c9ffc6c50fb0ee848832e17c7",
52
+ "smoke_train.py": "8cdeb2b397177fc9c26638aaa871501ddab8e3871aa1573ecd98f66590f5c228",
53
+ "scripts/run_experiment.sh": "661a6309fc54a2a8aff918f14a553c72dcb21730bd6a3cfc55d6ccd4700d11d3",
54
+ "scripts/download_candidate.py": "d06a2c01be0cf6577f927fb37e3bc1eab014949fd934e4f4d9825674adba608e",
55
+ "scripts/prepare_public_data.py": "32ea84aa719818e1141b958b6ef27a85f7ddb86bfcd7c1585fc487b25253d977",
56
+ "scripts/prepare_expansion_backup.py": "0b546fd6b96e34316fddcb06314fa072d1ddb9c06d7e6979ccd790421b93f419",
57
+ "scripts/verify_playground_layout.cjs": "dea3fa07c568c141cd58fead8c547db7196f4a48fe4ea4cea0312d2ba7bd8bb4",
58
+ "scripts/run_smoke.sh": "39d59f2120f362729d1c2e384391b82be1e580dcc1aca0eaa7ab231115225574",
59
+ "scripts/start_spark_candidate.sh": "c2ca18b008a144de7cb264c9fcca634d68c4a8a317db3638e8a70b3dbcff064b",
60
+ "scripts/launch_24h.py": "39c26dc10535d3adac09209b0743edf2ed384563e732512ad62f8c6f37161a83",
61
+ "scripts/verify_artifact.py": "9833350e9d72c0065b15206bb71c5a8b5a6b3185219db90074e369ede563a985",
62
+ "scripts/publish_hf_snapshot.py": "b5fe16a00fcbc5ab97121428c6ce750275ee193438e3ffe24aac5e3bb325c018",
63
+ "scripts/prepare_expanded_data.py": "5c05478b29c84218784690f3c7c3ec994615fec1c57826591607189f191c56b4",
64
+ "scripts/fleet_campaign.py": "e69fdff96f92c6943b7895be11df444f016d0b744a1b9441995a5f8bb7af9d54",
65
+ "scripts/run_precision.sh": "766d82b30cf3e83951f662685b4472ee053c7fbc171c4ff140823bf4d1c2782f",
66
+ "scripts/fleet_status.py": "2519ced157ef4ac4fa449eebabae5740d7527778d578b4ac6720583010fa5217",
67
+ "scripts/profile_inference.py": "84e3032b5965049606e2486391eece49436ff66c1badad5dd5169d2f9eb0e97f",
68
+ "scripts/verify_playground.cjs": "a55aa0e945baeb9e536c2b9cce7a9aa2364244b587e90477190add96f78601dc",
69
+ "scripts/diagnose_parity.py": "08b5d66d316ebda98a2226251a4f952701f86a1d5726ce7a4d7e8fb22755da5a",
70
+ "scripts/publish_hf_final.py": "278efc5878d7ebc5d1171f3a735d6c7d78275d93650e5a8a16c3f55b8353a577",
71
+ "scripts/final_validation.py": "f5cea8bad330dd066f43b3dea5a977dbe3e8d5d69336d2850279bb743b628643",
72
+ "scripts/verify_expanded_startup.py": "deabba820a3580e578d2d955d3ac3fd2e3982af999523c1d9a8b6dc6e2e1b755",
73
+ "scripts/download_model.py": "72ad9a5de44d09e2ee4ed8afb7c3c0ff6fb48987410a3f7f0368353572bf1f4d",
74
+ "scripts/investigate_precision.py": "609b744a926d8a45b87ba8d225e5312ee0c71a7096b21bd3589c5846e1dc847c",
75
+ "scripts/campaign_status.py": "1500b5e24f06231c7aafd7277aefd840582e35997e265f79db93614828d34411"
76
+ },
77
+ "packages": {
78
+ "torch": "2.11.0+cu130",
79
+ "transformers": "5.15.0",
80
+ "pyarrow": "25.0.1",
81
+ "numpy": "2.5.2"
82
+ },
83
+ "hostname": "gx10-9dd0",
84
+ "cuda": "13.0",
85
+ "gpu": "NVIDIA GB10",
86
+ "cuda_cap_bytes": 17179869184,
87
+ "initial_mem_available_bytes": 87274446848,
88
+ "oom_score_adj": "0",
89
+ "selection": {
90
+ "metric": "crossfit_temperature_nll_v1",
91
+ "score": 0.1701497127614862,
92
+ "raw_macro_nll": 0.190872636672039,
93
+ "scope": "validation only; reserved calibration/test/holdout not used"
94
+ },
95
+ "started_utc": "2026-09-17T08:09:57.452960+00:00"
96
+ }
final/20260916T194403396250Z-fleet/evaluation/metrics.json ADDED
@@ -0,0 +1,2355 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "temperature": 1.7458220720291138,
3
+ "selected_step": 0,
4
+ "status": "complete",
5
+ "claim_scope": "Public decision benchmark; no claim of Jev-level intelligence or general calibration",
6
+ "test": {
7
+ "trained": {
8
+ "overall": {
9
+ "n": 2042,
10
+ "accuracy": 0.9289911851126347,
11
+ "nll": 0.2548728303419246,
12
+ "brier_multiclass_sum": 0.11724661735348839,
13
+ "ece_top_label_10_equal_width_bins": 0.042709464978984944,
14
+ "reliability_bins": [
15
+ {
16
+ "count": 0,
17
+ "confidence_sum": 0.0,
18
+ "correct_sum": 0.0
19
+ },
20
+ {
21
+ "count": 0,
22
+ "confidence_sum": 0.0,
23
+ "correct_sum": 0.0
24
+ },
25
+ {
26
+ "count": 0,
27
+ "confidence_sum": 0.0,
28
+ "correct_sum": 0.0
29
+ },
30
+ {
31
+ "count": 1,
32
+ "confidence_sum": 0.3873113691806793,
33
+ "correct_sum": 0.0
34
+ },
35
+ {
36
+ "count": 6,
37
+ "confidence_sum": 2.761519640684128,
38
+ "correct_sum": 1.0
39
+ },
40
+ {
41
+ "count": 25,
42
+ "confidence_sum": 13.707988917827606,
43
+ "correct_sum": 13.0
44
+ },
45
+ {
46
+ "count": 37,
47
+ "confidence_sum": 23.83959299325943,
48
+ "correct_sum": 26.0
49
+ },
50
+ {
51
+ "count": 44,
52
+ "confidence_sum": 33.35772943496704,
53
+ "correct_sum": 29.0
54
+ },
55
+ {
56
+ "count": 71,
57
+ "confidence_sum": 60.919248819351196,
58
+ "correct_sum": 48.0
59
+ },
60
+ {
61
+ "count": 1858,
62
+ "confidence_sum": 1844.918522298336,
63
+ "correct_sum": 1780.0
64
+ }
65
+ ],
66
+ "accuracy_vs_coverage": {
67
+ "0.25": {
68
+ "n": 511,
69
+ "accuracy": 1.0,
70
+ "min_confidence": 0.9999915361404419
71
+ },
72
+ "0.5": {
73
+ "n": 1021,
74
+ "accuracy": 0.9921645445641528,
75
+ "min_confidence": 0.9986754059791565
76
+ },
77
+ "0.75": {
78
+ "n": 1532,
79
+ "accuracy": 0.9843342036553525,
80
+ "min_confidence": 0.9908576607704163
81
+ },
82
+ "1.0": {
83
+ "n": 2042,
84
+ "accuracy": 0.9289911851126347,
85
+ "min_confidence": 0.3873113691806793
86
+ }
87
+ }
88
+ },
89
+ "per_family": {
90
+ "banking": {
91
+ "n": 512,
92
+ "accuracy": 0.978515625,
93
+ "nll": 0.12226520271792657,
94
+ "brier_multiclass_sum": 0.037994640677064956,
95
+ "ece_top_label_10_equal_width_bins": 0.016155527671799064,
96
+ "reliability_bins": [
97
+ {
98
+ "count": 0,
99
+ "confidence_sum": 0.0,
100
+ "correct_sum": 0.0
101
+ },
102
+ {
103
+ "count": 0,
104
+ "confidence_sum": 0.0,
105
+ "correct_sum": 0.0
106
+ },
107
+ {
108
+ "count": 0,
109
+ "confidence_sum": 0.0,
110
+ "correct_sum": 0.0
111
+ },
112
+ {
113
+ "count": 0,
114
+ "confidence_sum": 0.0,
115
+ "correct_sum": 0.0
116
+ },
117
+ {
118
+ "count": 0,
119
+ "confidence_sum": 0.0,
120
+ "correct_sum": 0.0
121
+ },
122
+ {
123
+ "count": 3,
124
+ "confidence_sum": 1.705095112323761,
125
+ "correct_sum": 2.0
126
+ },
127
+ {
128
+ "count": 1,
129
+ "confidence_sum": 0.6073724627494812,
130
+ "correct_sum": 0.0
131
+ },
132
+ {
133
+ "count": 7,
134
+ "confidence_sum": 5.329456865787506,
135
+ "correct_sum": 5.0
136
+ },
137
+ {
138
+ "count": 5,
139
+ "confidence_sum": 4.3502872586250305,
140
+ "correct_sum": 5.0
141
+ },
142
+ {
143
+ "count": 496,
144
+ "confidence_sum": 495.3901832103729,
145
+ "correct_sum": 489.0
146
+ }
147
+ ],
148
+ "accuracy_vs_coverage": {
149
+ "0.25": {
150
+ "n": 128,
151
+ "accuracy": 1.0,
152
+ "min_confidence": 1.0
153
+ },
154
+ "0.5": {
155
+ "n": 256,
156
+ "accuracy": 1.0,
157
+ "min_confidence": 0.9999991655349731
158
+ },
159
+ "0.75": {
160
+ "n": 384,
161
+ "accuracy": 0.9973958333333334,
162
+ "min_confidence": 0.9999103546142578
163
+ },
164
+ "1.0": {
165
+ "n": 512,
166
+ "accuracy": 0.978515625,
167
+ "min_confidence": 0.5397089123725891
168
+ }
169
+ }
170
+ },
171
+ "boolq": {
172
+ "n": 506,
173
+ "accuracy": 0.8952569169960475,
174
+ "nll": 0.30082020614212623,
175
+ "brier_multiclass_sum": 0.1645830420203524,
176
+ "ece_top_label_10_equal_width_bins": 0.061629810352099273,
177
+ "reliability_bins": [
178
+ {
179
+ "count": 0,
180
+ "confidence_sum": 0.0,
181
+ "correct_sum": 0.0
182
+ },
183
+ {
184
+ "count": 0,
185
+ "confidence_sum": 0.0,
186
+ "correct_sum": 0.0
187
+ },
188
+ {
189
+ "count": 0,
190
+ "confidence_sum": 0.0,
191
+ "correct_sum": 0.0
192
+ },
193
+ {
194
+ "count": 0,
195
+ "confidence_sum": 0.0,
196
+ "correct_sum": 0.0
197
+ },
198
+ {
199
+ "count": 0,
200
+ "confidence_sum": 0.0,
201
+ "correct_sum": 0.0
202
+ },
203
+ {
204
+ "count": 8,
205
+ "confidence_sum": 4.3537238240242,
206
+ "correct_sum": 4.0
207
+ },
208
+ {
209
+ "count": 18,
210
+ "confidence_sum": 11.685294926166534,
211
+ "correct_sum": 10.0
212
+ },
213
+ {
214
+ "count": 12,
215
+ "confidence_sum": 9.267768621444702,
216
+ "correct_sum": 7.0
217
+ },
218
+ {
219
+ "count": 29,
220
+ "confidence_sum": 24.727146327495575,
221
+ "correct_sum": 19.0
222
+ },
223
+ {
224
+ "count": 439,
225
+ "confidence_sum": 434.1507503390312,
226
+ "correct_sum": 413.0
227
+ }
228
+ ],
229
+ "accuracy_vs_coverage": {
230
+ "0.25": {
231
+ "n": 127,
232
+ "accuracy": 0.9921259842519685,
233
+ "min_confidence": 0.9984140396118164
234
+ },
235
+ "0.5": {
236
+ "n": 253,
237
+ "accuracy": 0.9960474308300395,
238
+ "min_confidence": 0.995837926864624
239
+ },
240
+ "0.75": {
241
+ "n": 380,
242
+ "accuracy": 0.9631578947368421,
243
+ "min_confidence": 0.9789161086082458
244
+ },
245
+ "1.0": {
246
+ "n": 506,
247
+ "accuracy": 0.8952569169960475,
248
+ "min_confidence": 0.5028058886528015
249
+ }
250
+ }
251
+ },
252
+ "arc": {
253
+ "n": 512,
254
+ "accuracy": 0.94140625,
255
+ "nll": 0.23545075006863606,
256
+ "brier_multiclass_sum": 0.09582074176202018,
257
+ "ece_top_label_10_equal_width_bins": 0.03452872653724626,
258
+ "reliability_bins": [
259
+ {
260
+ "count": 0,
261
+ "confidence_sum": 0.0,
262
+ "correct_sum": 0.0
263
+ },
264
+ {
265
+ "count": 0,
266
+ "confidence_sum": 0.0,
267
+ "correct_sum": 0.0
268
+ },
269
+ {
270
+ "count": 0,
271
+ "confidence_sum": 0.0,
272
+ "correct_sum": 0.0
273
+ },
274
+ {
275
+ "count": 1,
276
+ "confidence_sum": 0.3873113691806793,
277
+ "correct_sum": 0.0
278
+ },
279
+ {
280
+ "count": 5,
281
+ "confidence_sum": 2.285579204559326,
282
+ "correct_sum": 1.0
283
+ },
284
+ {
285
+ "count": 7,
286
+ "confidence_sum": 3.7234585285186768,
287
+ "correct_sum": 3.0
288
+ },
289
+ {
290
+ "count": 10,
291
+ "confidence_sum": 6.485258996486664,
292
+ "correct_sum": 9.0
293
+ },
294
+ {
295
+ "count": 15,
296
+ "confidence_sum": 11.392396628856659,
297
+ "correct_sum": 12.0
298
+ },
299
+ {
300
+ "count": 17,
301
+ "confidence_sum": 14.561253845691681,
302
+ "correct_sum": 12.0
303
+ },
304
+ {
305
+ "count": 457,
306
+ "confidence_sum": 454.59876066446304,
307
+ "correct_sum": 445.0
308
+ }
309
+ ],
310
+ "accuracy_vs_coverage": {
311
+ "0.25": {
312
+ "n": 128,
313
+ "accuracy": 1.0,
314
+ "min_confidence": 0.9999977350234985
315
+ },
316
+ "0.5": {
317
+ "n": 256,
318
+ "accuracy": 1.0,
319
+ "min_confidence": 0.9999246597290039
320
+ },
321
+ "0.75": {
322
+ "n": 384,
323
+ "accuracy": 0.9895833333333334,
324
+ "min_confidence": 0.9971269965171814
325
+ },
326
+ "1.0": {
327
+ "n": 512,
328
+ "accuracy": 0.94140625,
329
+ "min_confidence": 0.3873113691806793
330
+ }
331
+ }
332
+ },
333
+ "snli": {
334
+ "n": 512,
335
+ "accuracy": 0.900390625,
336
+ "nll": 0.3614936082491682,
337
+ "brier_multiclass_sum": 0.1711427686810808,
338
+ "ece_top_label_10_equal_width_bins": 0.07405480305897072,
339
+ "reliability_bins": [
340
+ {
341
+ "count": 0,
342
+ "confidence_sum": 0.0,
343
+ "correct_sum": 0.0
344
+ },
345
+ {
346
+ "count": 0,
347
+ "confidence_sum": 0.0,
348
+ "correct_sum": 0.0
349
+ },
350
+ {
351
+ "count": 0,
352
+ "confidence_sum": 0.0,
353
+ "correct_sum": 0.0
354
+ },
355
+ {
356
+ "count": 0,
357
+ "confidence_sum": 0.0,
358
+ "correct_sum": 0.0
359
+ },
360
+ {
361
+ "count": 1,
362
+ "confidence_sum": 0.47594043612480164,
363
+ "correct_sum": 0.0
364
+ },
365
+ {
366
+ "count": 7,
367
+ "confidence_sum": 3.925711452960968,
368
+ "correct_sum": 4.0
369
+ },
370
+ {
371
+ "count": 8,
372
+ "confidence_sum": 5.0616666078567505,
373
+ "correct_sum": 7.0
374
+ },
375
+ {
376
+ "count": 10,
377
+ "confidence_sum": 7.368107318878174,
378
+ "correct_sum": 5.0
379
+ },
380
+ {
381
+ "count": 20,
382
+ "confidence_sum": 17.28056138753891,
383
+ "correct_sum": 12.0
384
+ },
385
+ {
386
+ "count": 466,
387
+ "confidence_sum": 460.77882808446884,
388
+ "correct_sum": 433.0
389
+ }
390
+ ],
391
+ "accuracy_vs_coverage": {
392
+ "0.25": {
393
+ "n": 128,
394
+ "accuracy": 0.96875,
395
+ "min_confidence": 0.9978107810020447
396
+ },
397
+ "0.5": {
398
+ "n": 256,
399
+ "accuracy": 0.9765625,
400
+ "min_confidence": 0.9943430423736572
401
+ },
402
+ "0.75": {
403
+ "n": 384,
404
+ "accuracy": 0.9609375,
405
+ "min_confidence": 0.9827606678009033
406
+ },
407
+ "1.0": {
408
+ "n": 512,
409
+ "accuracy": 0.900390625,
410
+ "min_confidence": 0.47594043612480164
411
+ }
412
+ }
413
+ }
414
+ }
415
+ },
416
+ "calibrated": {
417
+ "overall": {
418
+ "n": 2042,
419
+ "accuracy": 0.9289911851126347,
420
+ "nll": 0.2050675208059285,
421
+ "brier_multiclass_sum": 0.10981914968288821,
422
+ "ece_top_label_10_equal_width_bins": 0.010821329873058868,
423
+ "reliability_bins": [
424
+ {
425
+ "count": 0,
426
+ "confidence_sum": 0.0,
427
+ "correct_sum": 0.0
428
+ },
429
+ {
430
+ "count": 0,
431
+ "confidence_sum": 0.0,
432
+ "correct_sum": 0.0
433
+ },
434
+ {
435
+ "count": 0,
436
+ "confidence_sum": 0.0,
437
+ "correct_sum": 0.0
438
+ },
439
+ {
440
+ "count": 4,
441
+ "confidence_sum": 1.5274722874164581,
442
+ "correct_sum": 1.0
443
+ },
444
+ {
445
+ "count": 11,
446
+ "confidence_sum": 4.950159549713135,
447
+ "correct_sum": 4.0
448
+ },
449
+ {
450
+ "count": 58,
451
+ "confidence_sum": 32.086635649204254,
452
+ "correct_sum": 40.0
453
+ },
454
+ {
455
+ "count": 58,
456
+ "confidence_sum": 37.935491383075714,
457
+ "correct_sum": 34.0
458
+ },
459
+ {
460
+ "count": 101,
461
+ "confidence_sum": 76.33427423238754,
462
+ "correct_sum": 69.0
463
+ },
464
+ {
465
+ "count": 158,
466
+ "confidence_sum": 135.905029296875,
467
+ "correct_sum": 136.0
468
+ },
469
+ {
470
+ "count": 1652,
471
+ "confidence_sum": 1614.3414230942726,
472
+ "correct_sum": 1613.0
473
+ }
474
+ ],
475
+ "accuracy_vs_coverage": {
476
+ "0.25": {
477
+ "n": 511,
478
+ "accuracy": 1.0,
479
+ "min_confidence": 0.9984586238861084
480
+ },
481
+ "0.5": {
482
+ "n": 1021,
483
+ "accuracy": 0.9921645445641528,
484
+ "min_confidence": 0.976774275302887
485
+ },
486
+ "0.75": {
487
+ "n": 1532,
488
+ "accuracy": 0.9830287206266318,
489
+ "min_confidence": 0.9275727272033691
490
+ },
491
+ "1.0": {
492
+ "n": 2042,
493
+ "accuracy": 0.9289911851126347,
494
+ "min_confidence": 0.36327579617500305
495
+ }
496
+ }
497
+ },
498
+ "per_family": {
499
+ "banking": {
500
+ "n": 512,
501
+ "accuracy": 0.978515625,
502
+ "nll": 0.08680507836434942,
503
+ "brier_multiclass_sum": 0.03854156218229658,
504
+ "ece_top_label_10_equal_width_bins": 0.010919157532043755,
505
+ "reliability_bins": [
506
+ {
507
+ "count": 0,
508
+ "confidence_sum": 0.0,
509
+ "correct_sum": 0.0
510
+ },
511
+ {
512
+ "count": 0,
513
+ "confidence_sum": 0.0,
514
+ "correct_sum": 0.0
515
+ },
516
+ {
517
+ "count": 0,
518
+ "confidence_sum": 0.0,
519
+ "correct_sum": 0.0
520
+ },
521
+ {
522
+ "count": 0,
523
+ "confidence_sum": 0.0,
524
+ "correct_sum": 0.0
525
+ },
526
+ {
527
+ "count": 0,
528
+ "confidence_sum": 0.0,
529
+ "correct_sum": 0.0
530
+ },
531
+ {
532
+ "count": 6,
533
+ "confidence_sum": 3.359699547290802,
534
+ "correct_sum": 4.0
535
+ },
536
+ {
537
+ "count": 5,
538
+ "confidence_sum": 3.226902723312378,
539
+ "correct_sum": 3.0
540
+ },
541
+ {
542
+ "count": 6,
543
+ "confidence_sum": 4.473788321018219,
544
+ "correct_sum": 6.0
545
+ },
546
+ {
547
+ "count": 9,
548
+ "confidence_sum": 7.847302138805389,
549
+ "correct_sum": 8.0
550
+ },
551
+ {
552
+ "count": 486,
553
+ "confidence_sum": 483.04449594020844,
554
+ "correct_sum": 480.0
555
+ }
556
+ ],
557
+ "accuracy_vs_coverage": {
558
+ "0.25": {
559
+ "n": 128,
560
+ "accuracy": 1.0,
561
+ "min_confidence": 0.9999681711196899
562
+ },
563
+ "0.5": {
564
+ "n": 256,
565
+ "accuracy": 1.0,
566
+ "min_confidence": 0.9996205568313599
567
+ },
568
+ "0.75": {
569
+ "n": 384,
570
+ "accuracy": 0.9973958333333334,
571
+ "min_confidence": 0.9949487447738647
572
+ },
573
+ "1.0": {
574
+ "n": 512,
575
+ "accuracy": 0.978515625,
576
+ "min_confidence": 0.5222792029380798
577
+ }
578
+ }
579
+ },
580
+ "boolq": {
581
+ "n": 506,
582
+ "accuracy": 0.8952569169960475,
583
+ "nll": 0.24933799422114145,
584
+ "brier_multiclass_sum": 0.15022579183164184,
585
+ "ece_top_label_10_equal_width_bins": 0.016275467844348652,
586
+ "reliability_bins": [
587
+ {
588
+ "count": 0,
589
+ "confidence_sum": 0.0,
590
+ "correct_sum": 0.0
591
+ },
592
+ {
593
+ "count": 0,
594
+ "confidence_sum": 0.0,
595
+ "correct_sum": 0.0
596
+ },
597
+ {
598
+ "count": 0,
599
+ "confidence_sum": 0.0,
600
+ "correct_sum": 0.0
601
+ },
602
+ {
603
+ "count": 0,
604
+ "confidence_sum": 0.0,
605
+ "correct_sum": 0.0
606
+ },
607
+ {
608
+ "count": 0,
609
+ "confidence_sum": 0.0,
610
+ "correct_sum": 0.0
611
+ },
612
+ {
613
+ "count": 21,
614
+ "confidence_sum": 11.7305428981781,
615
+ "correct_sum": 10.0
616
+ },
617
+ {
618
+ "count": 22,
619
+ "confidence_sum": 14.534841537475586,
620
+ "correct_sum": 15.0
621
+ },
622
+ {
623
+ "count": 34,
624
+ "confidence_sum": 25.722497761249542,
625
+ "correct_sum": 22.0
626
+ },
627
+ {
628
+ "count": 49,
629
+ "confidence_sum": 41.86055135726929,
630
+ "correct_sum": 40.0
631
+ },
632
+ {
633
+ "count": 380,
634
+ "confidence_sum": 365.5433637499809,
635
+ "correct_sum": 366.0
636
+ }
637
+ ],
638
+ "accuracy_vs_coverage": {
639
+ "0.25": {
640
+ "n": 127,
641
+ "accuracy": 0.9921259842519685,
642
+ "min_confidence": 0.9756761789321899
643
+ },
644
+ "0.5": {
645
+ "n": 253,
646
+ "accuracy": 0.9960474308300395,
647
+ "min_confidence": 0.9584143757820129
648
+ },
649
+ "0.75": {
650
+ "n": 380,
651
+ "accuracy": 0.9631578947368421,
652
+ "min_confidence": 0.9001014232635498
653
+ },
654
+ "1.0": {
655
+ "n": 506,
656
+ "accuracy": 0.8952569169960475,
657
+ "min_confidence": 0.5016071796417236
658
+ }
659
+ }
660
+ },
661
+ "arc": {
662
+ "n": 512,
663
+ "accuracy": 0.94140625,
664
+ "nll": 0.19397132420263175,
665
+ "brier_multiclass_sum": 0.09487676147069625,
666
+ "ece_top_label_10_equal_width_bins": 0.02615507983136922,
667
+ "reliability_bins": [
668
+ {
669
+ "count": 0,
670
+ "confidence_sum": 0.0,
671
+ "correct_sum": 0.0
672
+ },
673
+ {
674
+ "count": 0,
675
+ "confidence_sum": 0.0,
676
+ "correct_sum": 0.0
677
+ },
678
+ {
679
+ "count": 0,
680
+ "confidence_sum": 0.0,
681
+ "correct_sum": 0.0
682
+ },
683
+ {
684
+ "count": 4,
685
+ "confidence_sum": 1.5274722874164581,
686
+ "correct_sum": 1.0
687
+ },
688
+ {
689
+ "count": 9,
690
+ "confidence_sum": 4.047605782747269,
691
+ "correct_sum": 4.0
692
+ },
693
+ {
694
+ "count": 15,
695
+ "confidence_sum": 8.381983816623688,
696
+ "correct_sum": 14.0
697
+ },
698
+ {
699
+ "count": 17,
700
+ "confidence_sum": 11.08071506023407,
701
+ "correct_sum": 9.0
702
+ },
703
+ {
704
+ "count": 26,
705
+ "confidence_sum": 19.58449637889862,
706
+ "correct_sum": 22.0
707
+ },
708
+ {
709
+ "count": 27,
710
+ "confidence_sum": 22.931695699691772,
711
+ "correct_sum": 24.0
712
+ },
713
+ {
714
+ "count": 414,
715
+ "confidence_sum": 409.6337836384773,
716
+ "correct_sum": 408.0
717
+ }
718
+ ],
719
+ "accuracy_vs_coverage": {
720
+ "0.25": {
721
+ "n": 128,
722
+ "accuracy": 1.0,
723
+ "min_confidence": 0.9992086291313171
724
+ },
725
+ "0.5": {
726
+ "n": 256,
727
+ "accuracy": 1.0,
728
+ "min_confidence": 0.9945464134216309
729
+ },
730
+ "0.75": {
731
+ "n": 384,
732
+ "accuracy": 0.9895833333333334,
733
+ "min_confidence": 0.9568338990211487
734
+ },
735
+ "1.0": {
736
+ "n": 512,
737
+ "accuracy": 0.94140625,
738
+ "min_confidence": 0.36327579617500305
739
+ }
740
+ }
741
+ },
742
+ "snli": {
743
+ "n": 512,
744
+ "accuracy": 0.900390625,
745
+ "nll": 0.29067448104592586,
746
+ "brier_multiclass_sum": 0.15610599858459887,
747
+ "ece_top_label_10_equal_width_bins": 0.03427618817659095,
748
+ "reliability_bins": [
749
+ {
750
+ "count": 0,
751
+ "confidence_sum": 0.0,
752
+ "correct_sum": 0.0
753
+ },
754
+ {
755
+ "count": 0,
756
+ "confidence_sum": 0.0,
757
+ "correct_sum": 0.0
758
+ },
759
+ {
760
+ "count": 0,
761
+ "confidence_sum": 0.0,
762
+ "correct_sum": 0.0
763
+ },
764
+ {
765
+ "count": 0,
766
+ "confidence_sum": 0.0,
767
+ "correct_sum": 0.0
768
+ },
769
+ {
770
+ "count": 2,
771
+ "confidence_sum": 0.9025537669658661,
772
+ "correct_sum": 0.0
773
+ },
774
+ {
775
+ "count": 16,
776
+ "confidence_sum": 8.614409387111664,
777
+ "correct_sum": 12.0
778
+ },
779
+ {
780
+ "count": 14,
781
+ "confidence_sum": 9.09303206205368,
782
+ "correct_sum": 7.0
783
+ },
784
+ {
785
+ "count": 35,
786
+ "confidence_sum": 26.55349177122116,
787
+ "correct_sum": 19.0
788
+ },
789
+ {
790
+ "count": 73,
791
+ "confidence_sum": 63.26548010110855,
792
+ "correct_sum": 64.0
793
+ },
794
+ {
795
+ "count": 372,
796
+ "confidence_sum": 356.1197797656059,
797
+ "correct_sum": 359.0
798
+ }
799
+ ],
800
+ "accuracy_vs_coverage": {
801
+ "0.25": {
802
+ "n": 128,
803
+ "accuracy": 0.9765625,
804
+ "min_confidence": 0.9674575924873352
805
+ },
806
+ "0.5": {
807
+ "n": 256,
808
+ "accuracy": 0.9765625,
809
+ "min_confidence": 0.9432480931282043
810
+ },
811
+ "0.75": {
812
+ "n": 384,
813
+ "accuracy": 0.9583333333333334,
814
+ "min_confidence": 0.8896382451057434
815
+ },
816
+ "1.0": {
817
+ "n": 512,
818
+ "accuracy": 0.900390625,
819
+ "min_confidence": 0.4136597514152527
820
+ }
821
+ }
822
+ }
823
+ }
824
+ },
825
+ "base": {
826
+ "overall": {
827
+ "n": 2042,
828
+ "accuracy": 0.8447600391772772,
829
+ "nll": 1.6827827308918881,
830
+ "brier_multiclass_sum": 0.2927494974423544,
831
+ "ece_top_label_10_equal_width_bins": 0.13822684455279854,
832
+ "reliability_bins": [
833
+ {
834
+ "count": 0,
835
+ "confidence_sum": 0.0,
836
+ "correct_sum": 0.0
837
+ },
838
+ {
839
+ "count": 0,
840
+ "confidence_sum": 0.0,
841
+ "correct_sum": 0.0
842
+ },
843
+ {
844
+ "count": 0,
845
+ "confidence_sum": 0.0,
846
+ "correct_sum": 0.0
847
+ },
848
+ {
849
+ "count": 2,
850
+ "confidence_sum": 0.732365071773529,
851
+ "correct_sum": 0.0
852
+ },
853
+ {
854
+ "count": 4,
855
+ "confidence_sum": 1.8684652149677277,
856
+ "correct_sum": 2.0
857
+ },
858
+ {
859
+ "count": 25,
860
+ "confidence_sum": 13.689240455627441,
861
+ "correct_sum": 17.0
862
+ },
863
+ {
864
+ "count": 18,
865
+ "confidence_sum": 11.746700882911682,
866
+ "correct_sum": 11.0
867
+ },
868
+ {
869
+ "count": 36,
870
+ "confidence_sum": 27.12858122587204,
871
+ "correct_sum": 20.0
872
+ },
873
+ {
874
+ "count": 46,
875
+ "confidence_sum": 39.98839032649994,
876
+ "correct_sum": 26.0
877
+ },
878
+ {
879
+ "count": 1911,
880
+ "confidence_sum": 1905.2208847403526,
881
+ "correct_sum": 1649.0
882
+ }
883
+ ],
884
+ "accuracy_vs_coverage": {
885
+ "0.25": {
886
+ "n": 511,
887
+ "accuracy": 0.9354207436399217,
888
+ "min_confidence": 1.0
889
+ },
890
+ "0.5": {
891
+ "n": 1021,
892
+ "accuracy": 0.9422135161606269,
893
+ "min_confidence": 1.0
894
+ },
895
+ "0.75": {
896
+ "n": 1532,
897
+ "accuracy": 0.9007832898172323,
898
+ "min_confidence": 0.9998598098754883
899
+ },
900
+ "1.0": {
901
+ "n": 2042,
902
+ "accuracy": 0.8447600391772772,
903
+ "min_confidence": 0.3561801314353943
904
+ }
905
+ }
906
+ },
907
+ "per_family": {
908
+ "banking": {
909
+ "n": 512,
910
+ "accuracy": 0.8828125,
911
+ "nll": 0.8035370189185151,
912
+ "brier_multiclass_sum": 0.2032958888533993,
913
+ "ece_top_label_10_equal_width_bins": 0.08182763156946748,
914
+ "reliability_bins": [
915
+ {
916
+ "count": 0,
917
+ "confidence_sum": 0.0,
918
+ "correct_sum": 0.0
919
+ },
920
+ {
921
+ "count": 0,
922
+ "confidence_sum": 0.0,
923
+ "correct_sum": 0.0
924
+ },
925
+ {
926
+ "count": 0,
927
+ "confidence_sum": 0.0,
928
+ "correct_sum": 0.0
929
+ },
930
+ {
931
+ "count": 2,
932
+ "confidence_sum": 0.732365071773529,
933
+ "correct_sum": 0.0
934
+ },
935
+ {
936
+ "count": 2,
937
+ "confidence_sum": 0.9544804692268372,
938
+ "correct_sum": 1.0
939
+ },
940
+ {
941
+ "count": 9,
942
+ "confidence_sum": 4.947599828243256,
943
+ "correct_sum": 5.0
944
+ },
945
+ {
946
+ "count": 8,
947
+ "confidence_sum": 5.206531882286072,
948
+ "correct_sum": 4.0
949
+ },
950
+ {
951
+ "count": 16,
952
+ "confidence_sum": 11.929070949554443,
953
+ "correct_sum": 9.0
954
+ },
955
+ {
956
+ "count": 19,
957
+ "confidence_sum": 16.535522401332855,
958
+ "correct_sum": 15.0
959
+ },
960
+ {
961
+ "count": 456,
962
+ "confidence_sum": 453.39433735609055,
963
+ "correct_sum": 418.0
964
+ }
965
+ ],
966
+ "accuracy_vs_coverage": {
967
+ "0.25": {
968
+ "n": 128,
969
+ "accuracy": 0.9921875,
970
+ "min_confidence": 1.0
971
+ },
972
+ "0.5": {
973
+ "n": 256,
974
+ "accuracy": 0.9765625,
975
+ "min_confidence": 0.9999994039535522
976
+ },
977
+ "0.75": {
978
+ "n": 384,
979
+ "accuracy": 0.9401041666666666,
980
+ "min_confidence": 0.9933443069458008
981
+ },
982
+ "1.0": {
983
+ "n": 512,
984
+ "accuracy": 0.8828125,
985
+ "min_confidence": 0.3561801314353943
986
+ }
987
+ }
988
+ },
989
+ "boolq": {
990
+ "n": 506,
991
+ "accuracy": 0.8557312252964426,
992
+ "nll": 2.0259620797809275,
993
+ "brier_multiclass_sum": 0.2843932332075561,
994
+ "ece_top_label_10_equal_width_bins": 0.14410379540778903,
995
+ "reliability_bins": [
996
+ {
997
+ "count": 0,
998
+ "confidence_sum": 0.0,
999
+ "correct_sum": 0.0
1000
+ },
1001
+ {
1002
+ "count": 0,
1003
+ "confidence_sum": 0.0,
1004
+ "correct_sum": 0.0
1005
+ },
1006
+ {
1007
+ "count": 0,
1008
+ "confidence_sum": 0.0,
1009
+ "correct_sum": 0.0
1010
+ },
1011
+ {
1012
+ "count": 0,
1013
+ "confidence_sum": 0.0,
1014
+ "correct_sum": 0.0
1015
+ },
1016
+ {
1017
+ "count": 0,
1018
+ "confidence_sum": 0.0,
1019
+ "correct_sum": 0.0
1020
+ },
1021
+ {
1022
+ "count": 4,
1023
+ "confidence_sum": 2.20137357711792,
1024
+ "correct_sum": 4.0
1025
+ },
1026
+ {
1027
+ "count": 1,
1028
+ "confidence_sum": 0.645257830619812,
1029
+ "correct_sum": 1.0
1030
+ },
1031
+ {
1032
+ "count": 3,
1033
+ "confidence_sum": 2.2964596152305603,
1034
+ "correct_sum": 1.0
1035
+ },
1036
+ {
1037
+ "count": 6,
1038
+ "confidence_sum": 5.2520251870155334,
1039
+ "correct_sum": 2.0
1040
+ },
1041
+ {
1042
+ "count": 492,
1043
+ "confidence_sum": 491.2146670818329,
1044
+ "correct_sum": 425.0
1045
+ }
1046
+ ],
1047
+ "accuracy_vs_coverage": {
1048
+ "0.25": {
1049
+ "n": 127,
1050
+ "accuracy": 0.8976377952755905,
1051
+ "min_confidence": 1.0
1052
+ },
1053
+ "0.5": {
1054
+ "n": 253,
1055
+ "accuracy": 0.9169960474308301,
1056
+ "min_confidence": 1.0
1057
+ },
1058
+ "0.75": {
1059
+ "n": 380,
1060
+ "accuracy": 0.9289473684210526,
1061
+ "min_confidence": 0.9999997615814209
1062
+ },
1063
+ "1.0": {
1064
+ "n": 506,
1065
+ "accuracy": 0.8557312252964426,
1066
+ "min_confidence": 0.5300353169441223
1067
+ }
1068
+ }
1069
+ },
1070
+ "arc": {
1071
+ "n": 512,
1072
+ "accuracy": 0.9296875,
1073
+ "nll": 0.8372381083637264,
1074
+ "brier_multiclass_sum": 0.13284523221990113,
1075
+ "ece_top_label_10_equal_width_bins": 0.06219080294249579,
1076
+ "reliability_bins": [
1077
+ {
1078
+ "count": 0,
1079
+ "confidence_sum": 0.0,
1080
+ "correct_sum": 0.0
1081
+ },
1082
+ {
1083
+ "count": 0,
1084
+ "confidence_sum": 0.0,
1085
+ "correct_sum": 0.0
1086
+ },
1087
+ {
1088
+ "count": 0,
1089
+ "confidence_sum": 0.0,
1090
+ "correct_sum": 0.0
1091
+ },
1092
+ {
1093
+ "count": 0,
1094
+ "confidence_sum": 0.0,
1095
+ "correct_sum": 0.0
1096
+ },
1097
+ {
1098
+ "count": 2,
1099
+ "confidence_sum": 0.9139847457408905,
1100
+ "correct_sum": 1.0
1101
+ },
1102
+ {
1103
+ "count": 2,
1104
+ "confidence_sum": 1.0453534722328186,
1105
+ "correct_sum": 1.0
1106
+ },
1107
+ {
1108
+ "count": 5,
1109
+ "confidence_sum": 3.2189850211143494,
1110
+ "correct_sum": 3.0
1111
+ },
1112
+ {
1113
+ "count": 6,
1114
+ "confidence_sum": 4.524070084095001,
1115
+ "correct_sum": 6.0
1116
+ },
1117
+ {
1118
+ "count": 8,
1119
+ "confidence_sum": 6.894674122333527,
1120
+ "correct_sum": 5.0
1121
+ },
1122
+ {
1123
+ "count": 489,
1124
+ "confidence_sum": 488.12073332071304,
1125
+ "correct_sum": 460.0
1126
+ }
1127
+ ],
1128
+ "accuracy_vs_coverage": {
1129
+ "0.25": {
1130
+ "n": 128,
1131
+ "accuracy": 0.984375,
1132
+ "min_confidence": 1.0
1133
+ },
1134
+ "0.5": {
1135
+ "n": 256,
1136
+ "accuracy": 0.98828125,
1137
+ "min_confidence": 1.0
1138
+ },
1139
+ "0.75": {
1140
+ "n": 384,
1141
+ "accuracy": 0.9817708333333334,
1142
+ "min_confidence": 0.9999997615814209
1143
+ },
1144
+ "1.0": {
1145
+ "n": 512,
1146
+ "accuracy": 0.9296875,
1147
+ "min_confidence": 0.4202010929584503
1148
+ }
1149
+ }
1150
+ },
1151
+ "snli": {
1152
+ "n": 512,
1153
+ "accuracy": 0.7109375,
1154
+ "nll": 3.0684153494991766,
1155
+ "brier_multiclass_sum": 0.5503657105170595,
1156
+ "ece_top_label_10_equal_width_bins": 0.2734481571242213,
1157
+ "reliability_bins": [
1158
+ {
1159
+ "count": 0,
1160
+ "confidence_sum": 0.0,
1161
+ "correct_sum": 0.0
1162
+ },
1163
+ {
1164
+ "count": 0,
1165
+ "confidence_sum": 0.0,
1166
+ "correct_sum": 0.0
1167
+ },
1168
+ {
1169
+ "count": 0,
1170
+ "confidence_sum": 0.0,
1171
+ "correct_sum": 0.0
1172
+ },
1173
+ {
1174
+ "count": 0,
1175
+ "confidence_sum": 0.0,
1176
+ "correct_sum": 0.0
1177
+ },
1178
+ {
1179
+ "count": 0,
1180
+ "confidence_sum": 0.0,
1181
+ "correct_sum": 0.0
1182
+ },
1183
+ {
1184
+ "count": 10,
1185
+ "confidence_sum": 5.494913578033447,
1186
+ "correct_sum": 7.0
1187
+ },
1188
+ {
1189
+ "count": 4,
1190
+ "confidence_sum": 2.675926148891449,
1191
+ "correct_sum": 3.0
1192
+ },
1193
+ {
1194
+ "count": 11,
1195
+ "confidence_sum": 8.378980576992035,
1196
+ "correct_sum": 4.0
1197
+ },
1198
+ {
1199
+ "count": 13,
1200
+ "confidence_sum": 11.306168615818024,
1201
+ "correct_sum": 4.0
1202
+ },
1203
+ {
1204
+ "count": 474,
1205
+ "confidence_sum": 472.49114698171616,
1206
+ "correct_sum": 346.0
1207
+ }
1208
+ ],
1209
+ "accuracy_vs_coverage": {
1210
+ "0.25": {
1211
+ "n": 128,
1212
+ "accuracy": 0.8125,
1213
+ "min_confidence": 1.0
1214
+ },
1215
+ "0.5": {
1216
+ "n": 256,
1217
+ "accuracy": 0.80078125,
1218
+ "min_confidence": 0.9999966621398926
1219
+ },
1220
+ "0.75": {
1221
+ "n": 384,
1222
+ "accuracy": 0.7630208333333334,
1223
+ "min_confidence": 0.9995673298835754
1224
+ },
1225
+ "1.0": {
1226
+ "n": 512,
1227
+ "accuracy": 0.7109375,
1228
+ "min_confidence": 0.5036829113960266
1229
+ }
1230
+ }
1231
+ }
1232
+ }
1233
+ },
1234
+ "base_calibrated": {
1235
+ "overall": {
1236
+ "n": 2042,
1237
+ "accuracy": 0.8447600391772772,
1238
+ "nll": 0.452739927518432,
1239
+ "brier_multiclass_sum": 0.25478263453519945,
1240
+ "ece_top_label_10_equal_width_bins": 0.06263403555088248,
1241
+ "reliability_bins": [
1242
+ {
1243
+ "count": 0,
1244
+ "confidence_sum": 0.0,
1245
+ "correct_sum": 0.0
1246
+ },
1247
+ {
1248
+ "count": 0,
1249
+ "confidence_sum": 0.0,
1250
+ "correct_sum": 0.0
1251
+ },
1252
+ {
1253
+ "count": 8,
1254
+ "confidence_sum": 2.2937141954898834,
1255
+ "correct_sum": 3.0
1256
+ },
1257
+ {
1258
+ "count": 62,
1259
+ "confidence_sum": 21.780550003051758,
1260
+ "correct_sum": 42.0
1261
+ },
1262
+ {
1263
+ "count": 87,
1264
+ "confidence_sum": 39.44270572066307,
1265
+ "correct_sum": 70.0
1266
+ },
1267
+ {
1268
+ "count": 147,
1269
+ "confidence_sum": 80.74226522445679,
1270
+ "correct_sum": 103.0
1271
+ },
1272
+ {
1273
+ "count": 150,
1274
+ "confidence_sum": 97.46173959970474,
1275
+ "correct_sum": 93.0
1276
+ },
1277
+ {
1278
+ "count": 199,
1279
+ "confidence_sum": 150.20955330133438,
1280
+ "correct_sum": 151.0
1281
+ },
1282
+ {
1283
+ "count": 311,
1284
+ "confidence_sum": 265.94291496276855,
1285
+ "correct_sum": 250.0
1286
+ },
1287
+ {
1288
+ "count": 1078,
1289
+ "confidence_sum": 1045.9628344774246,
1290
+ "correct_sum": 1013.0
1291
+ }
1292
+ ],
1293
+ "accuracy_vs_coverage": {
1294
+ "0.25": {
1295
+ "n": 511,
1296
+ "accuracy": 0.9843444227005871,
1297
+ "min_confidence": 0.9829810857772827
1298
+ },
1299
+ "0.5": {
1300
+ "n": 1021,
1301
+ "accuracy": 0.9480901077375122,
1302
+ "min_confidence": 0.9118173122406006
1303
+ },
1304
+ "0.75": {
1305
+ "n": 1532,
1306
+ "accuracy": 0.8955613577023499,
1307
+ "min_confidence": 0.7337803840637207
1308
+ },
1309
+ "1.0": {
1310
+ "n": 2042,
1311
+ "accuracy": 0.8447600391772772,
1312
+ "min_confidence": 0.26615577936172485
1313
+ }
1314
+ }
1315
+ },
1316
+ "per_family": {
1317
+ "banking": {
1318
+ "n": 512,
1319
+ "accuracy": 0.8828125,
1320
+ "nll": 0.4803847811426749,
1321
+ "brier_multiclass_sum": 0.24231384330718306,
1322
+ "ece_top_label_10_equal_width_bins": 0.15125263947993517,
1323
+ "reliability_bins": [
1324
+ {
1325
+ "count": 0,
1326
+ "confidence_sum": 0.0,
1327
+ "correct_sum": 0.0
1328
+ },
1329
+ {
1330
+ "count": 0,
1331
+ "confidence_sum": 0.0,
1332
+ "correct_sum": 0.0
1333
+ },
1334
+ {
1335
+ "count": 6,
1336
+ "confidence_sum": 1.7256377339363098,
1337
+ "correct_sum": 2.0
1338
+ },
1339
+ {
1340
+ "count": 47,
1341
+ "confidence_sum": 16.507239133119583,
1342
+ "correct_sum": 34.0
1343
+ },
1344
+ {
1345
+ "count": 61,
1346
+ "confidence_sum": 27.371423810720444,
1347
+ "correct_sum": 50.0
1348
+ },
1349
+ {
1350
+ "count": 56,
1351
+ "confidence_sum": 30.601767003536224,
1352
+ "correct_sum": 47.0
1353
+ },
1354
+ {
1355
+ "count": 47,
1356
+ "confidence_sum": 30.510030925273895,
1357
+ "correct_sum": 34.0
1358
+ },
1359
+ {
1360
+ "count": 38,
1361
+ "confidence_sum": 28.63151115179062,
1362
+ "correct_sum": 35.0
1363
+ },
1364
+ {
1365
+ "count": 68,
1366
+ "confidence_sum": 58.034259259700775,
1367
+ "correct_sum": 66.0
1368
+ },
1369
+ {
1370
+ "count": 189,
1371
+ "confidence_sum": 181.17677956819534,
1372
+ "correct_sum": 184.0
1373
+ }
1374
+ ],
1375
+ "accuracy_vs_coverage": {
1376
+ "0.25": {
1377
+ "n": 128,
1378
+ "accuracy": 0.984375,
1379
+ "min_confidence": 0.9501156210899353
1380
+ },
1381
+ "0.5": {
1382
+ "n": 256,
1383
+ "accuracy": 0.9765625,
1384
+ "min_confidence": 0.8002985715866089
1385
+ },
1386
+ "0.75": {
1387
+ "n": 384,
1388
+ "accuracy": 0.9322916666666666,
1389
+ "min_confidence": 0.5225431323051453
1390
+ },
1391
+ "1.0": {
1392
+ "n": 512,
1393
+ "accuracy": 0.8828125,
1394
+ "min_confidence": 0.26615577936172485
1395
+ }
1396
+ }
1397
+ },
1398
+ "boolq": {
1399
+ "n": 506,
1400
+ "accuracy": 0.8557312252964426,
1401
+ "nll": 0.3844228962146085,
1402
+ "brier_multiclass_sum": 0.22706346727506466,
1403
+ "ece_top_label_10_equal_width_bins": 0.07297720126954935,
1404
+ "reliability_bins": [
1405
+ {
1406
+ "count": 0,
1407
+ "confidence_sum": 0.0,
1408
+ "correct_sum": 0.0
1409
+ },
1410
+ {
1411
+ "count": 0,
1412
+ "confidence_sum": 0.0,
1413
+ "correct_sum": 0.0
1414
+ },
1415
+ {
1416
+ "count": 0,
1417
+ "confidence_sum": 0.0,
1418
+ "correct_sum": 0.0
1419
+ },
1420
+ {
1421
+ "count": 0,
1422
+ "confidence_sum": 0.0,
1423
+ "correct_sum": 0.0
1424
+ },
1425
+ {
1426
+ "count": 0,
1427
+ "confidence_sum": 0.0,
1428
+ "correct_sum": 0.0
1429
+ },
1430
+ {
1431
+ "count": 18,
1432
+ "confidence_sum": 9.970384955406189,
1433
+ "correct_sum": 12.0
1434
+ },
1435
+ {
1436
+ "count": 29,
1437
+ "confidence_sum": 18.950466096401215,
1438
+ "correct_sum": 18.0
1439
+ },
1440
+ {
1441
+ "count": 29,
1442
+ "confidence_sum": 21.715792536735535,
1443
+ "correct_sum": 16.0
1444
+ },
1445
+ {
1446
+ "count": 50,
1447
+ "confidence_sum": 42.18575972318649,
1448
+ "correct_sum": 34.0
1449
+ },
1450
+ {
1451
+ "count": 380,
1452
+ "confidence_sum": 373.0448304414749,
1453
+ "correct_sum": 353.0
1454
+ }
1455
+ ],
1456
+ "accuracy_vs_coverage": {
1457
+ "0.25": {
1458
+ "n": 127,
1459
+ "accuracy": 0.9921259842519685,
1460
+ "min_confidence": 0.9963729381561279
1461
+ },
1462
+ "0.5": {
1463
+ "n": 253,
1464
+ "accuracy": 0.9802371541501976,
1465
+ "min_confidence": 0.9850993752479553
1466
+ },
1467
+ "0.75": {
1468
+ "n": 380,
1469
+ "accuracy": 0.9289473684210526,
1470
+ "min_confidence": 0.9002280831336975
1471
+ },
1472
+ "1.0": {
1473
+ "n": 506,
1474
+ "accuracy": 0.8557312252964426,
1475
+ "min_confidence": 0.5043465495109558
1476
+ }
1477
+ }
1478
+ },
1479
+ "arc": {
1480
+ "n": 512,
1481
+ "accuracy": 0.9296875,
1482
+ "nll": 0.2752359951973631,
1483
+ "brier_multiclass_sum": 0.13593068181824372,
1484
+ "ece_top_label_10_equal_width_bins": 0.05345189612125978,
1485
+ "reliability_bins": [
1486
+ {
1487
+ "count": 0,
1488
+ "confidence_sum": 0.0,
1489
+ "correct_sum": 0.0
1490
+ },
1491
+ {
1492
+ "count": 0,
1493
+ "confidence_sum": 0.0,
1494
+ "correct_sum": 0.0
1495
+ },
1496
+ {
1497
+ "count": 2,
1498
+ "confidence_sum": 0.5680764615535736,
1499
+ "correct_sum": 1.0
1500
+ },
1501
+ {
1502
+ "count": 15,
1503
+ "confidence_sum": 5.273310869932175,
1504
+ "correct_sum": 8.0
1505
+ },
1506
+ {
1507
+ "count": 19,
1508
+ "confidence_sum": 8.764264017343521,
1509
+ "correct_sum": 16.0
1510
+ },
1511
+ {
1512
+ "count": 25,
1513
+ "confidence_sum": 13.797979295253754,
1514
+ "correct_sum": 21.0
1515
+ },
1516
+ {
1517
+ "count": 20,
1518
+ "confidence_sum": 13.02436488866806,
1519
+ "correct_sum": 13.0
1520
+ },
1521
+ {
1522
+ "count": 28,
1523
+ "confidence_sum": 20.95827430486679,
1524
+ "correct_sum": 27.0
1525
+ },
1526
+ {
1527
+ "count": 52,
1528
+ "confidence_sum": 44.90535968542099,
1529
+ "correct_sum": 44.0
1530
+ },
1531
+ {
1532
+ "count": 351,
1533
+ "confidence_sum": 343.20044881105423,
1534
+ "correct_sum": 346.0
1535
+ }
1536
+ ],
1537
+ "accuracy_vs_coverage": {
1538
+ "0.25": {
1539
+ "n": 128,
1540
+ "accuracy": 1.0,
1541
+ "min_confidence": 0.9898765683174133
1542
+ },
1543
+ "0.5": {
1544
+ "n": 256,
1545
+ "accuracy": 0.99609375,
1546
+ "min_confidence": 0.9743005633354187
1547
+ },
1548
+ "0.75": {
1549
+ "n": 384,
1550
+ "accuracy": 0.9791666666666666,
1551
+ "min_confidence": 0.8585602045059204
1552
+ },
1553
+ "1.0": {
1554
+ "n": 512,
1555
+ "accuracy": 0.9296875,
1556
+ "min_confidence": 0.27811235189437866
1557
+ }
1558
+ }
1559
+ },
1560
+ "snli": {
1561
+ "n": 512,
1562
+ "accuracy": 0.7109375,
1563
+ "nll": 0.6701154473084898,
1564
+ "brier_multiclass_sum": 0.4134977117489767,
1565
+ "ece_top_label_10_equal_width_bins": 0.0982505488791503,
1566
+ "reliability_bins": [
1567
+ {
1568
+ "count": 0,
1569
+ "confidence_sum": 0.0,
1570
+ "correct_sum": 0.0
1571
+ },
1572
+ {
1573
+ "count": 0,
1574
+ "confidence_sum": 0.0,
1575
+ "correct_sum": 0.0
1576
+ },
1577
+ {
1578
+ "count": 0,
1579
+ "confidence_sum": 0.0,
1580
+ "correct_sum": 0.0
1581
+ },
1582
+ {
1583
+ "count": 0,
1584
+ "confidence_sum": 0.0,
1585
+ "correct_sum": 0.0
1586
+ },
1587
+ {
1588
+ "count": 7,
1589
+ "confidence_sum": 3.307017892599106,
1590
+ "correct_sum": 4.0
1591
+ },
1592
+ {
1593
+ "count": 48,
1594
+ "confidence_sum": 26.37213397026062,
1595
+ "correct_sum": 23.0
1596
+ },
1597
+ {
1598
+ "count": 54,
1599
+ "confidence_sum": 34.97687768936157,
1600
+ "correct_sum": 28.0
1601
+ },
1602
+ {
1603
+ "count": 104,
1604
+ "confidence_sum": 78.90397530794144,
1605
+ "correct_sum": 73.0
1606
+ },
1607
+ {
1608
+ "count": 141,
1609
+ "confidence_sum": 120.8175362944603,
1610
+ "correct_sum": 106.0
1611
+ },
1612
+ {
1613
+ "count": 158,
1614
+ "confidence_sum": 148.54077565670013,
1615
+ "correct_sum": 130.0
1616
+ }
1617
+ ],
1618
+ "accuracy_vs_coverage": {
1619
+ "0.25": {
1620
+ "n": 128,
1621
+ "accuracy": 0.859375,
1622
+ "min_confidence": 0.9148098826408386
1623
+ },
1624
+ "0.5": {
1625
+ "n": 256,
1626
+ "accuracy": 0.80859375,
1627
+ "min_confidence": 0.8419564962387085
1628
+ },
1629
+ "0.75": {
1630
+ "n": 384,
1631
+ "accuracy": 0.7708333333333334,
1632
+ "min_confidence": 0.7315099835395813
1633
+ },
1634
+ "1.0": {
1635
+ "n": 512,
1636
+ "accuracy": 0.7109375,
1637
+ "min_confidence": 0.43296143412590027
1638
+ }
1639
+ }
1640
+ }
1641
+ }
1642
+ },
1643
+ "calibrated_difference_95pct": {
1644
+ "method": "400 stratified source-group bootstrap resamples; decision-weighted tuned minus base",
1645
+ "point_delta": {
1646
+ "accuracy": 0.0842311459353575,
1647
+ "nll": -0.2476724067125035,
1648
+ "brier": -0.1449634848523113
1649
+ },
1650
+ "accuracy": [
1651
+ 0.06854799216454456,
1652
+ 0.098922624877571
1653
+ ],
1654
+ "nll": [
1655
+ -0.27686958949075574,
1656
+ -0.21881351599842205
1657
+ ],
1658
+ "brier": [
1659
+ -0.16374186238337493,
1660
+ -0.1257739683435538
1661
+ ]
1662
+ }
1663
+ },
1664
+ "holdout": {
1665
+ "trained": {
1666
+ "overall": {
1667
+ "n": 768,
1668
+ "accuracy": 0.7291666666666666,
1669
+ "nll": 0.9046048978141895,
1670
+ "brier_multiclass_sum": 0.41865507801212026,
1671
+ "ece_top_label_10_equal_width_bins": 0.1554523635810862,
1672
+ "reliability_bins": [
1673
+ {
1674
+ "count": 0,
1675
+ "confidence_sum": 0.0,
1676
+ "correct_sum": 0.0
1677
+ },
1678
+ {
1679
+ "count": 0,
1680
+ "confidence_sum": 0.0,
1681
+ "correct_sum": 0.0
1682
+ },
1683
+ {
1684
+ "count": 0,
1685
+ "confidence_sum": 0.0,
1686
+ "correct_sum": 0.0
1687
+ },
1688
+ {
1689
+ "count": 3,
1690
+ "confidence_sum": 1.1163514852523804,
1691
+ "correct_sum": 0.0
1692
+ },
1693
+ {
1694
+ "count": 13,
1695
+ "confidence_sum": 6.152260005474091,
1696
+ "correct_sum": 3.0
1697
+ },
1698
+ {
1699
+ "count": 54,
1700
+ "confidence_sum": 29.515426993370056,
1701
+ "correct_sum": 27.0
1702
+ },
1703
+ {
1704
+ "count": 50,
1705
+ "confidence_sum": 32.41912567615509,
1706
+ "correct_sum": 27.0
1707
+ },
1708
+ {
1709
+ "count": 65,
1710
+ "confidence_sum": 48.71525001525879,
1711
+ "correct_sum": 33.0
1712
+ },
1713
+ {
1714
+ "count": 84,
1715
+ "confidence_sum": 71.8504301905632,
1716
+ "correct_sum": 44.0
1717
+ },
1718
+ {
1719
+ "count": 499,
1720
+ "confidence_sum": 489.6185708642006,
1721
+ "correct_sum": 426.0
1722
+ }
1723
+ ],
1724
+ "accuracy_vs_coverage": {
1725
+ "0.25": {
1726
+ "n": 192,
1727
+ "accuracy": 0.9375,
1728
+ "min_confidence": 0.9978193044662476
1729
+ },
1730
+ "0.5": {
1731
+ "n": 384,
1732
+ "accuracy": 0.8828125,
1733
+ "min_confidence": 0.9679536819458008
1734
+ },
1735
+ "0.75": {
1736
+ "n": 576,
1737
+ "accuracy": 0.8107638888888888,
1738
+ "min_confidence": 0.8080979585647583
1739
+ },
1740
+ "1.0": {
1741
+ "n": 768,
1742
+ "accuracy": 0.7291666666666666,
1743
+ "min_confidence": 0.3503410816192627
1744
+ }
1745
+ }
1746
+ },
1747
+ "per_family": {
1748
+ "social": {
1749
+ "n": 768,
1750
+ "accuracy": 0.7291666666666666,
1751
+ "nll": 0.9046048978141895,
1752
+ "brier_multiclass_sum": 0.41865507801212026,
1753
+ "ece_top_label_10_equal_width_bins": 0.1554523635810862,
1754
+ "reliability_bins": [
1755
+ {
1756
+ "count": 0,
1757
+ "confidence_sum": 0.0,
1758
+ "correct_sum": 0.0
1759
+ },
1760
+ {
1761
+ "count": 0,
1762
+ "confidence_sum": 0.0,
1763
+ "correct_sum": 0.0
1764
+ },
1765
+ {
1766
+ "count": 0,
1767
+ "confidence_sum": 0.0,
1768
+ "correct_sum": 0.0
1769
+ },
1770
+ {
1771
+ "count": 3,
1772
+ "confidence_sum": 1.1163514852523804,
1773
+ "correct_sum": 0.0
1774
+ },
1775
+ {
1776
+ "count": 13,
1777
+ "confidence_sum": 6.152260005474091,
1778
+ "correct_sum": 3.0
1779
+ },
1780
+ {
1781
+ "count": 54,
1782
+ "confidence_sum": 29.515426993370056,
1783
+ "correct_sum": 27.0
1784
+ },
1785
+ {
1786
+ "count": 50,
1787
+ "confidence_sum": 32.41912567615509,
1788
+ "correct_sum": 27.0
1789
+ },
1790
+ {
1791
+ "count": 65,
1792
+ "confidence_sum": 48.71525001525879,
1793
+ "correct_sum": 33.0
1794
+ },
1795
+ {
1796
+ "count": 84,
1797
+ "confidence_sum": 71.8504301905632,
1798
+ "correct_sum": 44.0
1799
+ },
1800
+ {
1801
+ "count": 499,
1802
+ "confidence_sum": 489.6185708642006,
1803
+ "correct_sum": 426.0
1804
+ }
1805
+ ],
1806
+ "accuracy_vs_coverage": {
1807
+ "0.25": {
1808
+ "n": 192,
1809
+ "accuracy": 0.9375,
1810
+ "min_confidence": 0.9978193044662476
1811
+ },
1812
+ "0.5": {
1813
+ "n": 384,
1814
+ "accuracy": 0.8828125,
1815
+ "min_confidence": 0.9679536819458008
1816
+ },
1817
+ "0.75": {
1818
+ "n": 576,
1819
+ "accuracy": 0.8107638888888888,
1820
+ "min_confidence": 0.8080979585647583
1821
+ },
1822
+ "1.0": {
1823
+ "n": 768,
1824
+ "accuracy": 0.7291666666666666,
1825
+ "min_confidence": 0.3503410816192627
1826
+ }
1827
+ }
1828
+ }
1829
+ }
1830
+ },
1831
+ "calibrated": {
1832
+ "overall": {
1833
+ "n": 768,
1834
+ "accuracy": 0.7291666666666666,
1835
+ "nll": 0.6782619158996491,
1836
+ "brier_multiclass_sum": 0.37973740706466513,
1837
+ "ece_top_label_10_equal_width_bins": 0.08299602890231957,
1838
+ "reliability_bins": [
1839
+ {
1840
+ "count": 0,
1841
+ "confidence_sum": 0.0,
1842
+ "correct_sum": 0.0
1843
+ },
1844
+ {
1845
+ "count": 0,
1846
+ "confidence_sum": 0.0,
1847
+ "correct_sum": 0.0
1848
+ },
1849
+ {
1850
+ "count": 0,
1851
+ "confidence_sum": 0.0,
1852
+ "correct_sum": 0.0
1853
+ },
1854
+ {
1855
+ "count": 4,
1856
+ "confidence_sum": 1.4600901305675507,
1857
+ "correct_sum": 0.0
1858
+ },
1859
+ {
1860
+ "count": 43,
1861
+ "confidence_sum": 19.645778954029083,
1862
+ "correct_sum": 15.0
1863
+ },
1864
+ {
1865
+ "count": 84,
1866
+ "confidence_sum": 46.113935708999634,
1867
+ "correct_sum": 48.0
1868
+ },
1869
+ {
1870
+ "count": 84,
1871
+ "confidence_sum": 54.28614550828934,
1872
+ "correct_sum": 39.0
1873
+ },
1874
+ {
1875
+ "count": 97,
1876
+ "confidence_sum": 72.91012477874756,
1877
+ "correct_sum": 59.0
1878
+ },
1879
+ {
1880
+ "count": 130,
1881
+ "confidence_sum": 110.77482378482819,
1882
+ "correct_sum": 107.0
1883
+ },
1884
+ {
1885
+ "count": 326,
1886
+ "confidence_sum": 314.77792274951935,
1887
+ "correct_sum": 292.0
1888
+ }
1889
+ ],
1890
+ "accuracy_vs_coverage": {
1891
+ "0.25": {
1892
+ "n": 192,
1893
+ "accuracy": 0.9375,
1894
+ "min_confidence": 0.9662192463874817
1895
+ },
1896
+ "0.5": {
1897
+ "n": 384,
1898
+ "accuracy": 0.8854166666666666,
1899
+ "min_confidence": 0.8613662123680115
1900
+ },
1901
+ "0.75": {
1902
+ "n": 576,
1903
+ "accuracy": 0.8072916666666666,
1904
+ "min_confidence": 0.670463502407074
1905
+ },
1906
+ "1.0": {
1907
+ "n": 768,
1908
+ "accuracy": 0.7291666666666666,
1909
+ "min_confidence": 0.3430308401584625
1910
+ }
1911
+ }
1912
+ },
1913
+ "per_family": {
1914
+ "social": {
1915
+ "n": 768,
1916
+ "accuracy": 0.7291666666666666,
1917
+ "nll": 0.6782619158996491,
1918
+ "brier_multiclass_sum": 0.37973740706466513,
1919
+ "ece_top_label_10_equal_width_bins": 0.08299602890231957,
1920
+ "reliability_bins": [
1921
+ {
1922
+ "count": 0,
1923
+ "confidence_sum": 0.0,
1924
+ "correct_sum": 0.0
1925
+ },
1926
+ {
1927
+ "count": 0,
1928
+ "confidence_sum": 0.0,
1929
+ "correct_sum": 0.0
1930
+ },
1931
+ {
1932
+ "count": 0,
1933
+ "confidence_sum": 0.0,
1934
+ "correct_sum": 0.0
1935
+ },
1936
+ {
1937
+ "count": 4,
1938
+ "confidence_sum": 1.4600901305675507,
1939
+ "correct_sum": 0.0
1940
+ },
1941
+ {
1942
+ "count": 43,
1943
+ "confidence_sum": 19.645778954029083,
1944
+ "correct_sum": 15.0
1945
+ },
1946
+ {
1947
+ "count": 84,
1948
+ "confidence_sum": 46.113935708999634,
1949
+ "correct_sum": 48.0
1950
+ },
1951
+ {
1952
+ "count": 84,
1953
+ "confidence_sum": 54.28614550828934,
1954
+ "correct_sum": 39.0
1955
+ },
1956
+ {
1957
+ "count": 97,
1958
+ "confidence_sum": 72.91012477874756,
1959
+ "correct_sum": 59.0
1960
+ },
1961
+ {
1962
+ "count": 130,
1963
+ "confidence_sum": 110.77482378482819,
1964
+ "correct_sum": 107.0
1965
+ },
1966
+ {
1967
+ "count": 326,
1968
+ "confidence_sum": 314.77792274951935,
1969
+ "correct_sum": 292.0
1970
+ }
1971
+ ],
1972
+ "accuracy_vs_coverage": {
1973
+ "0.25": {
1974
+ "n": 192,
1975
+ "accuracy": 0.9375,
1976
+ "min_confidence": 0.9662192463874817
1977
+ },
1978
+ "0.5": {
1979
+ "n": 384,
1980
+ "accuracy": 0.8854166666666666,
1981
+ "min_confidence": 0.8613662123680115
1982
+ },
1983
+ "0.75": {
1984
+ "n": 576,
1985
+ "accuracy": 0.8072916666666666,
1986
+ "min_confidence": 0.670463502407074
1987
+ },
1988
+ "1.0": {
1989
+ "n": 768,
1990
+ "accuracy": 0.7291666666666666,
1991
+ "min_confidence": 0.3430308401584625
1992
+ }
1993
+ }
1994
+ }
1995
+ }
1996
+ },
1997
+ "base": {
1998
+ "overall": {
1999
+ "n": 768,
2000
+ "accuracy": 0.703125,
2001
+ "nll": 2.087191693346451,
2002
+ "brier_multiclass_sum": 0.5129237150352639,
2003
+ "ece_top_label_10_equal_width_bins": 0.22570987732615322,
2004
+ "reliability_bins": [
2005
+ {
2006
+ "count": 0,
2007
+ "confidence_sum": 0.0,
2008
+ "correct_sum": 0.0
2009
+ },
2010
+ {
2011
+ "count": 0,
2012
+ "confidence_sum": 0.0,
2013
+ "correct_sum": 0.0
2014
+ },
2015
+ {
2016
+ "count": 0,
2017
+ "confidence_sum": 0.0,
2018
+ "correct_sum": 0.0
2019
+ },
2020
+ {
2021
+ "count": 1,
2022
+ "confidence_sum": 0.3542192578315735,
2023
+ "correct_sum": 1.0
2024
+ },
2025
+ {
2026
+ "count": 18,
2027
+ "confidence_sum": 8.336367458105087,
2028
+ "correct_sum": 10.0
2029
+ },
2030
+ {
2031
+ "count": 39,
2032
+ "confidence_sum": 21.718455612659454,
2033
+ "correct_sum": 17.0
2034
+ },
2035
+ {
2036
+ "count": 22,
2037
+ "confidence_sum": 14.276574194431305,
2038
+ "correct_sum": 13.0
2039
+ },
2040
+ {
2041
+ "count": 33,
2042
+ "confidence_sum": 24.695184469223022,
2043
+ "correct_sum": 15.0
2044
+ },
2045
+ {
2046
+ "count": 62,
2047
+ "confidence_sum": 52.76100742816925,
2048
+ "correct_sum": 33.0
2049
+ },
2050
+ {
2051
+ "count": 593,
2052
+ "confidence_sum": 586.5845507979393,
2053
+ "correct_sum": 451.0
2054
+ }
2055
+ ],
2056
+ "accuracy_vs_coverage": {
2057
+ "0.25": {
2058
+ "n": 192,
2059
+ "accuracy": 0.90625,
2060
+ "min_confidence": 0.9999998807907104
2061
+ },
2062
+ "0.5": {
2063
+ "n": 384,
2064
+ "accuracy": 0.8255208333333334,
2065
+ "min_confidence": 0.9991338849067688
2066
+ },
2067
+ "0.75": {
2068
+ "n": 576,
2069
+ "accuracy": 0.7690972222222222,
2070
+ "min_confidence": 0.9154785871505737
2071
+ },
2072
+ "1.0": {
2073
+ "n": 768,
2074
+ "accuracy": 0.703125,
2075
+ "min_confidence": 0.3542192578315735
2076
+ }
2077
+ }
2078
+ },
2079
+ "per_family": {
2080
+ "social": {
2081
+ "n": 768,
2082
+ "accuracy": 0.703125,
2083
+ "nll": 2.087191693346451,
2084
+ "brier_multiclass_sum": 0.5129237150352639,
2085
+ "ece_top_label_10_equal_width_bins": 0.22570987732615322,
2086
+ "reliability_bins": [
2087
+ {
2088
+ "count": 0,
2089
+ "confidence_sum": 0.0,
2090
+ "correct_sum": 0.0
2091
+ },
2092
+ {
2093
+ "count": 0,
2094
+ "confidence_sum": 0.0,
2095
+ "correct_sum": 0.0
2096
+ },
2097
+ {
2098
+ "count": 0,
2099
+ "confidence_sum": 0.0,
2100
+ "correct_sum": 0.0
2101
+ },
2102
+ {
2103
+ "count": 1,
2104
+ "confidence_sum": 0.3542192578315735,
2105
+ "correct_sum": 1.0
2106
+ },
2107
+ {
2108
+ "count": 18,
2109
+ "confidence_sum": 8.336367458105087,
2110
+ "correct_sum": 10.0
2111
+ },
2112
+ {
2113
+ "count": 39,
2114
+ "confidence_sum": 21.718455612659454,
2115
+ "correct_sum": 17.0
2116
+ },
2117
+ {
2118
+ "count": 22,
2119
+ "confidence_sum": 14.276574194431305,
2120
+ "correct_sum": 13.0
2121
+ },
2122
+ {
2123
+ "count": 33,
2124
+ "confidence_sum": 24.695184469223022,
2125
+ "correct_sum": 15.0
2126
+ },
2127
+ {
2128
+ "count": 62,
2129
+ "confidence_sum": 52.76100742816925,
2130
+ "correct_sum": 33.0
2131
+ },
2132
+ {
2133
+ "count": 593,
2134
+ "confidence_sum": 586.5845507979393,
2135
+ "correct_sum": 451.0
2136
+ }
2137
+ ],
2138
+ "accuracy_vs_coverage": {
2139
+ "0.25": {
2140
+ "n": 192,
2141
+ "accuracy": 0.90625,
2142
+ "min_confidence": 0.9999998807907104
2143
+ },
2144
+ "0.5": {
2145
+ "n": 384,
2146
+ "accuracy": 0.8255208333333334,
2147
+ "min_confidence": 0.9991338849067688
2148
+ },
2149
+ "0.75": {
2150
+ "n": 576,
2151
+ "accuracy": 0.7690972222222222,
2152
+ "min_confidence": 0.9154785871505737
2153
+ },
2154
+ "1.0": {
2155
+ "n": 768,
2156
+ "accuracy": 0.703125,
2157
+ "min_confidence": 0.3542192578315735
2158
+ }
2159
+ }
2160
+ }
2161
+ }
2162
+ },
2163
+ "base_calibrated": {
2164
+ "overall": {
2165
+ "n": 768,
2166
+ "accuracy": 0.703125,
2167
+ "nll": 0.7425018713104995,
2168
+ "brier_multiclass_sum": 0.4306530777581018,
2169
+ "ece_top_label_10_equal_width_bins": 0.08665639813989401,
2170
+ "reliability_bins": [
2171
+ {
2172
+ "count": 0,
2173
+ "confidence_sum": 0.0,
2174
+ "correct_sum": 0.0
2175
+ },
2176
+ {
2177
+ "count": 0,
2178
+ "confidence_sum": 0.0,
2179
+ "correct_sum": 0.0
2180
+ },
2181
+ {
2182
+ "count": 0,
2183
+ "confidence_sum": 0.0,
2184
+ "correct_sum": 0.0
2185
+ },
2186
+ {
2187
+ "count": 69,
2188
+ "confidence_sum": 25.828479051589966,
2189
+ "correct_sum": 33.0
2190
+ },
2191
+ {
2192
+ "count": 165,
2193
+ "confidence_sum": 74.53762590885162,
2194
+ "correct_sum": 90.0
2195
+ },
2196
+ {
2197
+ "count": 107,
2198
+ "confidence_sum": 58.80419135093689,
2199
+ "correct_sum": 72.0
2200
+ },
2201
+ {
2202
+ "count": 95,
2203
+ "confidence_sum": 61.89751034975052,
2204
+ "correct_sum": 73.0
2205
+ },
2206
+ {
2207
+ "count": 86,
2208
+ "confidence_sum": 64.67725455760956,
2209
+ "correct_sum": 60.0
2210
+ },
2211
+ {
2212
+ "count": 86,
2213
+ "confidence_sum": 73.19006419181824,
2214
+ "correct_sum": 66.0
2215
+ },
2216
+ {
2217
+ "count": 160,
2218
+ "confidence_sum": 153.7526016831398,
2219
+ "correct_sum": 146.0
2220
+ }
2221
+ ],
2222
+ "accuracy_vs_coverage": {
2223
+ "0.25": {
2224
+ "n": 192,
2225
+ "accuracy": 0.9010416666666666,
2226
+ "min_confidence": 0.8640128970146179
2227
+ },
2228
+ "0.5": {
2229
+ "n": 384,
2230
+ "accuracy": 0.8098958333333334,
2231
+ "min_confidence": 0.6482440233230591
2232
+ },
2233
+ "0.75": {
2234
+ "n": 576,
2235
+ "accuracy": 0.7638888888888888,
2236
+ "min_confidence": 0.4738004803657532
2237
+ },
2238
+ "1.0": {
2239
+ "n": 768,
2240
+ "accuracy": 0.703125,
2241
+ "min_confidence": 0.33634451031684875
2242
+ }
2243
+ }
2244
+ },
2245
+ "per_family": {
2246
+ "social": {
2247
+ "n": 768,
2248
+ "accuracy": 0.703125,
2249
+ "nll": 0.7425018713104995,
2250
+ "brier_multiclass_sum": 0.4306530777581018,
2251
+ "ece_top_label_10_equal_width_bins": 0.08665639813989401,
2252
+ "reliability_bins": [
2253
+ {
2254
+ "count": 0,
2255
+ "confidence_sum": 0.0,
2256
+ "correct_sum": 0.0
2257
+ },
2258
+ {
2259
+ "count": 0,
2260
+ "confidence_sum": 0.0,
2261
+ "correct_sum": 0.0
2262
+ },
2263
+ {
2264
+ "count": 0,
2265
+ "confidence_sum": 0.0,
2266
+ "correct_sum": 0.0
2267
+ },
2268
+ {
2269
+ "count": 69,
2270
+ "confidence_sum": 25.828479051589966,
2271
+ "correct_sum": 33.0
2272
+ },
2273
+ {
2274
+ "count": 165,
2275
+ "confidence_sum": 74.53762590885162,
2276
+ "correct_sum": 90.0
2277
+ },
2278
+ {
2279
+ "count": 107,
2280
+ "confidence_sum": 58.80419135093689,
2281
+ "correct_sum": 72.0
2282
+ },
2283
+ {
2284
+ "count": 95,
2285
+ "confidence_sum": 61.89751034975052,
2286
+ "correct_sum": 73.0
2287
+ },
2288
+ {
2289
+ "count": 86,
2290
+ "confidence_sum": 64.67725455760956,
2291
+ "correct_sum": 60.0
2292
+ },
2293
+ {
2294
+ "count": 86,
2295
+ "confidence_sum": 73.19006419181824,
2296
+ "correct_sum": 66.0
2297
+ },
2298
+ {
2299
+ "count": 160,
2300
+ "confidence_sum": 153.7526016831398,
2301
+ "correct_sum": 146.0
2302
+ }
2303
+ ],
2304
+ "accuracy_vs_coverage": {
2305
+ "0.25": {
2306
+ "n": 192,
2307
+ "accuracy": 0.9010416666666666,
2308
+ "min_confidence": 0.8640128970146179
2309
+ },
2310
+ "0.5": {
2311
+ "n": 384,
2312
+ "accuracy": 0.8098958333333334,
2313
+ "min_confidence": 0.6482440233230591
2314
+ },
2315
+ "0.75": {
2316
+ "n": 576,
2317
+ "accuracy": 0.7638888888888888,
2318
+ "min_confidence": 0.4738004803657532
2319
+ },
2320
+ "1.0": {
2321
+ "n": 768,
2322
+ "accuracy": 0.703125,
2323
+ "min_confidence": 0.33634451031684875
2324
+ }
2325
+ }
2326
+ }
2327
+ }
2328
+ },
2329
+ "calibrated_difference_95pct": {
2330
+ "method": "400 stratified source-group bootstrap resamples; decision-weighted tuned minus base",
2331
+ "point_delta": {
2332
+ "accuracy": 0.026041666666666668,
2333
+ "nll": -0.0642399554108503,
2334
+ "brier": -0.050915670693436714
2335
+ },
2336
+ "accuracy": [
2337
+ 0.0013020833333333333,
2338
+ 0.05341796874999997
2339
+ ],
2340
+ "nll": [
2341
+ -0.11439402483851543,
2342
+ -0.016014919936165502
2343
+ ],
2344
+ "brier": [
2345
+ -0.0754793339156752,
2346
+ -0.02637446983897743
2347
+ ]
2348
+ }
2349
+ },
2350
+ "base_temperature": 6.918309211730957,
2351
+ "model_sha256": "e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348",
2352
+ "peak_cuda_allocated_bytes": 16356398592,
2353
+ "peak_cuda_reserved_bytes": 16393437184,
2354
+ "final_mem_available_bytes": 94531235840
2355
+ }
final/20260916T194403396250Z-fleet/fleet/api_probe.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "request": {
3
+ "model": "jev-latest",
4
+ "state": "I've tried to connect my account for three days. Payments keep failing and I'm losing sales. Please help today.",
5
+ "questions": {
6
+ "urgency": {
7
+ "type": "noul",
8
+ "instructions": "Does the customer express urgency?"
9
+ },
10
+ "department": {
11
+ "type": "choice",
12
+ "instructions": "Which department should investigate this issue?",
13
+ "criteria": {
14
+ "technical": "Integration failures and bugs",
15
+ "sales": "New purchases and pricing",
16
+ "billing": "Invoices and subscriptions"
17
+ }
18
+ },
19
+ "frustration": {
20
+ "type": "score",
21
+ "instructions": "How frustrated is the customer?",
22
+ "criteria": [
23
+ "Calm",
24
+ "Frustrated but civil",
25
+ "Very angry"
26
+ ]
27
+ }
28
+ }
29
+ },
30
+ "response": {
31
+ "model": "opensysone-qwen3-4b-instruct-2507",
32
+ "answers": {
33
+ "urgency": {
34
+ "type": "noul",
35
+ "noul": 0.9683617353439331
36
+ },
37
+ "department": {
38
+ "type": "choice",
39
+ "probabilities": {
40
+ "technical": 0.9963529109954834,
41
+ "sales": 0.000122277153423056,
42
+ "billing": 0.0035248342901468277
43
+ },
44
+ "confidence": 0.9775625660066237,
45
+ "choice": "technical"
46
+ },
47
+ "frustration": {
48
+ "type": "score",
49
+ "probabilities": {
50
+ "0": 0.012018980458378792,
51
+ "1": 0.8133298754692078,
52
+ "2": 0.1746511459350586
53
+ },
54
+ "confidence": 0.5212612917680768,
55
+ "score": 1.162632167339325,
56
+ "legend": {
57
+ "0": "Calm",
58
+ "1": "Frustrated but civil",
59
+ "2": "Very angry"
60
+ }
61
+ }
62
+ },
63
+ "usage": {
64
+ "input_tokens": 753,
65
+ "output_tokens": 0
66
+ }
67
+ },
68
+ "verified_utc": "2026-09-17T09:20:23.665064+00:00",
69
+ "checkpoint_sha256": "e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348"
70
+ }
final/20260916T194403396250Z-fleet/fleet/deployment.json ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "model": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/evaluation/model.pt",
3
+ "sha256": "e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348",
4
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet",
5
+ "evaluation": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/evaluation/metrics.json",
6
+ "selection": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/selection.json",
7
+ "inference_max_tokens": 1024,
8
+ "model_id": "opensysone-qwen3-4b-instruct-2507",
9
+ "finalized_utc": "2026-09-17T09:20:23.727161+00:00"
10
+ }
final/20260916T194403396250Z-fleet/fleet/exit_code ADDED
@@ -0,0 +1 @@
 
 
1
+ 0
final/20260916T194403396250Z-fleet/fleet/plan.json ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "selection_cutoff": "2026-09-17T08:07:30.700583+00:00",
3
+ "final_deadline": "2026-09-17T18:16:10Z",
4
+ "dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
5
+ "reference_predictions": "/home/andy/ai/opensysone/runs/20260916T183823Z-train/artifacts/validation_step_000040_predictions.json",
6
+ "reference_sha256": "e671e1508185765552b0f933ba03f356be62143c531d8ef534457d34b1645c9b",
7
+ "port": 18081,
8
+ "inference_max_tokens": 1024,
9
+ "candidates": [
10
+ {
11
+ "project": "/home/andy/projects/opensysone",
12
+ "python": "/home/andy/ai/envs/opensysone/bin/python",
13
+ "name": "gx10-4b",
14
+ "host": "local",
15
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h",
16
+ "training": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training",
17
+ "evidence_dirs": [
18
+ "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent"
19
+ ]
20
+ },
21
+ {
22
+ "project": "/home/andy/projects/opensysone",
23
+ "python": "/home/andy/ai/envs/opensysone/bin/python",
24
+ "name": "spark-a-4b-low-lr",
25
+ "host": "andy@192.168.8.111",
26
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h",
27
+ "training": "/home/andy/ai/opensysone/runs/20260917T075209Z-spark-a-final-validation",
28
+ "evidence_dirs": [
29
+ "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts"
30
+ ]
31
+ },
32
+ {
33
+ "project": "/home/andy/projects/opensysone",
34
+ "python": "/home/andy/ai/envs/opensysone/bin/python",
35
+ "name": "spark-b-2b",
36
+ "host": "andy@192.168.8.204",
37
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h",
38
+ "training": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training",
39
+ "evidence_dirs": [
40
+ "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts"
41
+ ]
42
+ },
43
+ {
44
+ "project": "/home/andy/projects/opensysone",
45
+ "python": "/home/andy/ai/envs/opensysone/bin/python",
46
+ "name": "spark-b-4b-refinement",
47
+ "host": "andy@192.168.8.204",
48
+ "campaign": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h",
49
+ "training": "/home/andy/ai/opensysone/runs/20260917T075209Z-spark-b-final-validation",
50
+ "evidence_dirs": [
51
+ "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts"
52
+ ]
53
+ },
54
+ {
55
+ "project": "/home/andy/ai/opensysone/source/expanded-24b8ccf",
56
+ "python": "/home/andy/ai/envs/opensysone/bin/python",
57
+ "name": "gx10-4b-expanded",
58
+ "host": "local",
59
+ "campaign": "/home/andy/ai/opensysone/runs/20260917T072142Z-24h",
60
+ "training": "/home/andy/ai/opensysone/runs/20260917T075209Z-gx10-final-validation",
61
+ "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917",
62
+ "evidence_dirs": [
63
+ "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts"
64
+ ]
65
+ }
66
+ ],
67
+ "original_selection_cutoff": "2026-09-17T16:00:00Z",
68
+ "wrapup": {
69
+ "requested_by": "user",
70
+ "reason": "Wrap up training and profile accuracy and speed",
71
+ "effective_utc": "2026-09-17T08:07:30.700583+00:00",
72
+ "optimizer_updates_after_stop": 0,
73
+ "final_validation_source_commit": "18f2b39506d4f4e6bdbed0eb6cdcb08cd50dcb57",
74
+ "evaluation_source_commit": "07f10e791061a679b829ed1dc5b33897e001d67d"
75
+ }
76
+ }
final/20260916T194403396250Z-fleet/fleet/selection.json ADDED
The diff for this file is too large to render. See raw diff
 
final/20260916T194403396250Z-fleet/fleet/state.json ADDED
@@ -0,0 +1,501 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet",
3
+ "supervisor_pid": 1672566,
4
+ "supervisor_command": [
5
+ "/home/andy/ai/envs/opensysone/bin/python",
6
+ "/home/andy/ai/opensysone/source/profile-07f10e7/scripts/fleet_campaign.py",
7
+ "--campaign",
8
+ "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet",
9
+ "--run"
10
+ ],
11
+ "status": "complete",
12
+ "stage": "serving",
13
+ "started_utc": "2026-09-17T08:07:30.786625+00:00",
14
+ "source_commit": "07f10e791061a679b829ed1dc5b33897e001d67d",
15
+ "source_sha256": "e69fdff96f92c6943b7895be11df444f016d0b744a1b9441995a5f8bb7af9d54",
16
+ "candidates": {
17
+ "gx10-4b": {
18
+ "stop_status": {
19
+ "plan": {
20
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt",
21
+ "resume_sha256": "1516441922c7a5a77e3fe3fe2fb5e511c6bd6a531e180d182f43d3e8cc460b3c",
22
+ "final_deadline": "2026-09-17T18:16:10+00:00",
23
+ "eval_reserve_seconds": 7200,
24
+ "evaluation_estimate": {
25
+ "pilot_validation_decisions": 512,
26
+ "pilot_validation_seconds": 329.73491883277893,
27
+ "evaluation_decisions": 6640,
28
+ "estimated_prediction_seconds": 4276.249728612602
29
+ },
30
+ "port": 18081,
31
+ "training_deadline": "2026-09-17T16:00:00+00:00",
32
+ "train_only": true,
33
+ "selection_metric": "crossfit_temperature_nll_v1",
34
+ "inference_max_tokens": 1024,
35
+ "created_utc": "2026-09-16T19:37:41.824450+00:00",
36
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
37
+ },
38
+ "state": {
39
+ "supervisor_pid": 1096483,
40
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h",
41
+ "stage": "stopped",
42
+ "status": "interrupted",
43
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
44
+ "child_pid": null,
45
+ "child_command": [
46
+ "/home/andy/ai/envs/opensysone/bin/python",
47
+ "/home/andy/projects/opensysone/experiment.py",
48
+ "train",
49
+ "--output",
50
+ "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training",
51
+ "--resume",
52
+ "/home/andy/ai/opensysone/runs/20260916T193721Z-selection-parent/checkpoint.pt",
53
+ "--deadline",
54
+ "2026-09-17T16:00:00+00:00",
55
+ "--model",
56
+ "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
57
+ "--dataset",
58
+ "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
59
+ "--epochs",
60
+ "3",
61
+ "--rank",
62
+ "8",
63
+ "--alpha",
64
+ "16.0",
65
+ "--lr",
66
+ "0.0001",
67
+ "--head-lr",
68
+ "0.0001",
69
+ "--seed",
70
+ "431",
71
+ "--effective-batch",
72
+ "4",
73
+ "--branch-batch-size",
74
+ "1",
75
+ "--max-tokens",
76
+ "512",
77
+ "--save-steps",
78
+ "250",
79
+ "--save-seconds",
80
+ "900",
81
+ "--eval-steps",
82
+ "500",
83
+ "--validation-per-family",
84
+ "128",
85
+ "--patience",
86
+ "8",
87
+ "--selection-metric",
88
+ "crossfit_temperature_nll_v1",
89
+ "--two-pass"
90
+ ],
91
+ "stage_started_utc": "2026-09-16T19:37:42.516405+00:00",
92
+ "heartbeat_utc": "2026-09-17T07:07:20.870658+00:00",
93
+ "termination_reason": "requested_stop",
94
+ "training_exit_code": 0,
95
+ "training_output": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training",
96
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193741Z-24h/training/checkpoint.pt"
97
+ },
98
+ "supervisor_running": false,
99
+ "child_running": false,
100
+ "api_running": false
101
+ }
102
+ },
103
+ "spark-a-4b-low-lr": {
104
+ "stop_status": {
105
+ "plan": {
106
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts/checkpoint.pt",
107
+ "resume_sha256": "cada37aff491907a0d5c8d3dde0aa5bc5a826219bbf062e6931d892d5ab6950f",
108
+ "final_deadline": "2026-09-17T18:16:10+00:00",
109
+ "eval_reserve_seconds": 7200,
110
+ "evaluation_estimate": {
111
+ "pilot_validation_decisions": 512,
112
+ "pilot_validation_seconds": 305.2858760356903,
113
+ "evaluation_decisions": 6640,
114
+ "estimated_prediction_seconds": 3959.1762048378587
115
+ },
116
+ "port": 18081,
117
+ "training_deadline": "2026-09-17T16:00:00+00:00",
118
+ "train_only": true,
119
+ "selection_metric": "crossfit_temperature_nll_v1",
120
+ "inference_max_tokens": 1024,
121
+ "created_utc": "2026-09-16T19:42:58.450848+00:00",
122
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
123
+ },
124
+ "state": {
125
+ "supervisor_pid": 327084,
126
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h",
127
+ "stage": "stopped",
128
+ "status": "interrupted",
129
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
130
+ "child_pid": null,
131
+ "child_command": [
132
+ "/home/andy/ai/envs/opensysone/bin/python",
133
+ "/home/andy/projects/opensysone/experiment.py",
134
+ "train",
135
+ "--output",
136
+ "/home/andy/ai/opensysone/runs/20260916T194258Z-24h/training",
137
+ "--resume",
138
+ "/home/andy/ai/opensysone/runs/20260916T192910Z-train/artifacts/checkpoint.pt",
139
+ "--deadline",
140
+ "2026-09-17T16:00:00+00:00",
141
+ "--model",
142
+ "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
143
+ "--dataset",
144
+ "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
145
+ "--epochs",
146
+ "3",
147
+ "--rank",
148
+ "8",
149
+ "--alpha",
150
+ "16.0",
151
+ "--lr",
152
+ "3e-05",
153
+ "--head-lr",
154
+ "3e-05",
155
+ "--seed",
156
+ "431",
157
+ "--effective-batch",
158
+ "4",
159
+ "--branch-batch-size",
160
+ "1",
161
+ "--max-tokens",
162
+ "512",
163
+ "--save-steps",
164
+ "250",
165
+ "--save-seconds",
166
+ "900",
167
+ "--eval-steps",
168
+ "500",
169
+ "--validation-per-family",
170
+ "128",
171
+ "--patience",
172
+ "8",
173
+ "--schedule-steps",
174
+ "7500",
175
+ "--selection-metric",
176
+ "crossfit_temperature_nll_v1",
177
+ "--two-pass"
178
+ ],
179
+ "stage_started_utc": "2026-09-16T19:42:59.139378+00:00",
180
+ "heartbeat_utc": "2026-09-17T07:52:19.261075+00:00",
181
+ "termination_reason": "requested_stop",
182
+ "training_exit_code": 0,
183
+ "training_output": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h/training",
184
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T194258Z-24h/training/checkpoint.pt"
185
+ },
186
+ "supervisor_running": false,
187
+ "child_running": false,
188
+ "api_running": false
189
+ }
190
+ },
191
+ "spark-b-2b": {
192
+ "stop_status": {
193
+ "plan": {
194
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt",
195
+ "resume_sha256": "af5790ae2f2b56477ebbdf6ab9c418d895e48d2bf5a6416e11b6e9863ad1db55",
196
+ "final_deadline": "2026-09-17T18:16:10+00:00",
197
+ "eval_reserve_seconds": 7200,
198
+ "evaluation_estimate": {
199
+ "pilot_validation_decisions": 512,
200
+ "pilot_validation_seconds": 130.97493624687195,
201
+ "evaluation_decisions": 6654,
202
+ "estimated_prediction_seconds": 1702.162550364621
203
+ },
204
+ "port": 18081,
205
+ "training_deadline": "2026-09-17T16:00:00+00:00",
206
+ "train_only": true,
207
+ "selection_metric": "crossfit_temperature_nll_v1",
208
+ "inference_max_tokens": 1024,
209
+ "created_utc": "2026-09-16T19:38:03.883204+00:00",
210
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
211
+ },
212
+ "state": {
213
+ "supervisor_pid": 286092,
214
+ "campaign": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h",
215
+ "stage": "training_complete",
216
+ "status": "training_complete",
217
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
218
+ "child_pid": null,
219
+ "child_command": [
220
+ "/home/andy/ai/envs/opensysone/bin/python",
221
+ "/home/andy/projects/opensysone/experiment.py",
222
+ "train",
223
+ "--output",
224
+ "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training",
225
+ "--resume",
226
+ "/home/andy/ai/opensysone/runs/20260916T182352Z-train/artifacts/checkpoint.pt",
227
+ "--deadline",
228
+ "2026-09-17T16:00:00+00:00",
229
+ "--model",
230
+ "/home/andy/ai/models/opensysone/Qwen3.5-2B-15852e8c",
231
+ "--dataset",
232
+ "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
233
+ "--epochs",
234
+ "3",
235
+ "--rank",
236
+ "16",
237
+ "--alpha",
238
+ "32",
239
+ "--lr",
240
+ "0.0001",
241
+ "--head-lr",
242
+ "0.0001",
243
+ "--seed",
244
+ "431",
245
+ "--effective-batch",
246
+ "4",
247
+ "--branch-batch-size",
248
+ "2",
249
+ "--max-tokens",
250
+ "768",
251
+ "--save-steps",
252
+ "250",
253
+ "--save-seconds",
254
+ "900",
255
+ "--eval-steps",
256
+ "500",
257
+ "--validation-per-family",
258
+ "128",
259
+ "--patience",
260
+ "8",
261
+ "--selection-metric",
262
+ "crossfit_temperature_nll_v1"
263
+ ],
264
+ "stage_started_utc": "2026-09-16T19:38:04.584070+00:00",
265
+ "heartbeat_utc": "2026-09-17T01:59:24.923675+00:00",
266
+ "training_exit_code": 0,
267
+ "training_output": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training",
268
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/checkpoint.pt",
269
+ "training_summary": {
270
+ "completed_steps": 6000,
271
+ "best_validation_macro_nll": 0.25676974726429014,
272
+ "status": "validation_early_stop",
273
+ "peak_cuda_allocated_bytes": 8786432512,
274
+ "selection_metric": "crossfit_temperature_nll_v1",
275
+ "best_validation_selection_score": 0.2552943737691716
276
+ },
277
+ "train_only": true,
278
+ "best_checkpoint": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best.pt",
279
+ "best_sha256": "be08efdc9121a64547b5c9331d5908eeb0978c4fbc962885fd34f9758ec89d11",
280
+ "finished_utc": "2026-09-17T01:59:29.971800+00:00",
281
+ "selection": "Await cross-candidate validation selection before calibration/test/holdout",
282
+ "best_validation_predictions": "/home/andy/ai/opensysone/runs/20260916T193803Z-24h/training/best_validation_predictions.json",
283
+ "best_validation_predictions_sha256": "980c9df18461f5de79e54701bfee1b5fbff19dd3b85125e64f619a88c60f80e4"
284
+ },
285
+ "supervisor_running": false,
286
+ "child_running": false,
287
+ "api_running": false
288
+ }
289
+ },
290
+ "spark-b-4b-refinement": {
291
+ "stop_status": {
292
+ "plan": {
293
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt",
294
+ "resume_sha256": "ece94b096b7fa37eec9e180a5f652da86a9f4f42d49f7f124850f046915237a8",
295
+ "final_deadline": "2026-09-17T18:16:10+00:00",
296
+ "eval_reserve_seconds": 7200,
297
+ "evaluation_estimate": {
298
+ "pilot_validation_decisions": 512,
299
+ "pilot_validation_seconds": 317.40339064598083,
300
+ "evaluation_decisions": 6640,
301
+ "estimated_prediction_seconds": 4116.325222440064
302
+ },
303
+ "port": 18081,
304
+ "training_deadline": "2026-09-17T16:00:00+00:00",
305
+ "train_only": true,
306
+ "selection_metric": "crossfit_temperature_nll_v1",
307
+ "inference_max_tokens": 1024,
308
+ "created_utc": "2026-09-17T02:31:37.778693+00:00",
309
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1"
310
+ },
311
+ "state": {
312
+ "supervisor_pid": 483974,
313
+ "campaign": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h",
314
+ "stage": "stopped",
315
+ "status": "interrupted",
316
+ "source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
317
+ "child_pid": null,
318
+ "child_command": [
319
+ "/home/andy/ai/envs/opensysone/bin/python",
320
+ "/home/andy/projects/opensysone/experiment.py",
321
+ "train",
322
+ "--output",
323
+ "/home/andy/ai/opensysone/runs/20260917T023137Z-24h/training",
324
+ "--resume",
325
+ "/home/andy/ai/opensysone/runs/20260917T021605Z-train/artifacts/checkpoint.pt",
326
+ "--deadline",
327
+ "2026-09-17T16:00:00+00:00",
328
+ "--model",
329
+ "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
330
+ "--dataset",
331
+ "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
332
+ "--epochs",
333
+ "3",
334
+ "--rank",
335
+ "8",
336
+ "--alpha",
337
+ "16.0",
338
+ "--lr",
339
+ "1e-05",
340
+ "--head-lr",
341
+ "1e-05",
342
+ "--seed",
343
+ "432",
344
+ "--effective-batch",
345
+ "4",
346
+ "--branch-batch-size",
347
+ "1",
348
+ "--max-tokens",
349
+ "512",
350
+ "--save-steps",
351
+ "250",
352
+ "--save-seconds",
353
+ "900",
354
+ "--eval-steps",
355
+ "500",
356
+ "--validation-per-family",
357
+ "128",
358
+ "--patience",
359
+ "8",
360
+ "--schedule-steps",
361
+ "5000",
362
+ "--selection-metric",
363
+ "crossfit_temperature_nll_v1",
364
+ "--two-pass"
365
+ ],
366
+ "stage_started_utc": "2026-09-17T02:31:38.483925+00:00",
367
+ "heartbeat_utc": "2026-09-17T07:52:13.484532+00:00",
368
+ "training_exit_code": 0,
369
+ "training_output": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h/training",
370
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260917T023137Z-24h/training/checkpoint.pt"
371
+ },
372
+ "supervisor_running": false,
373
+ "child_running": false,
374
+ "api_running": false
375
+ }
376
+ },
377
+ "gx10-4b-expanded": {
378
+ "stop_status": {
379
+ "plan": {
380
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts/checkpoint.pt",
381
+ "resume_sha256": "cf4cd775165cc2bb02113d17b94facd0db7b94fcff87b27389b3280fbb712469",
382
+ "final_deadline": "2026-09-17T18:16:10+00:00",
383
+ "eval_reserve_seconds": 7200,
384
+ "evaluation_estimate": {
385
+ "pilot_validation_decisions": 512,
386
+ "pilot_validation_seconds": 328.67159247398376,
387
+ "evaluation_decisions": 6640,
388
+ "estimated_prediction_seconds": 4262.459714896977
389
+ },
390
+ "port": 18081,
391
+ "training_deadline": "2026-09-17T16:00:00+00:00",
392
+ "train_only": true,
393
+ "selection_metric": "crossfit_temperature_nll_v1",
394
+ "inference_max_tokens": 1024,
395
+ "created_utc": "2026-09-17T07:21:42.655174+00:00",
396
+ "source_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86"
397
+ },
398
+ "state": {
399
+ "supervisor_pid": 1630617,
400
+ "campaign": "/home/andy/ai/opensysone/runs/20260917T072142Z-24h",
401
+ "stage": "stopped",
402
+ "status": "interrupted",
403
+ "source_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86",
404
+ "child_pid": null,
405
+ "child_command": [
406
+ "/home/andy/ai/envs/opensysone/bin/python",
407
+ "/home/andy/ai/opensysone/source/expanded-24b8ccf/experiment.py",
408
+ "train",
409
+ "--output",
410
+ "/home/andy/ai/opensysone/runs/20260917T072142Z-24h/training",
411
+ "--resume",
412
+ "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts/checkpoint.pt",
413
+ "--deadline",
414
+ "2026-09-17T16:00:00+00:00",
415
+ "--model",
416
+ "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
417
+ "--dataset",
418
+ "/home/andy/ai/opensysone/data/public-decisions-v2-20260917",
419
+ "--epochs",
420
+ "3",
421
+ "--rank",
422
+ "8",
423
+ "--alpha",
424
+ "16.0",
425
+ "--lr",
426
+ "2e-05",
427
+ "--head-lr",
428
+ "2e-05",
429
+ "--seed",
430
+ "433",
431
+ "--effective-batch",
432
+ "4",
433
+ "--branch-batch-size",
434
+ "1",
435
+ "--max-tokens",
436
+ "512",
437
+ "--save-steps",
438
+ "250",
439
+ "--save-seconds",
440
+ "900",
441
+ "--eval-steps",
442
+ "500",
443
+ "--validation-per-family",
444
+ "128",
445
+ "--patience",
446
+ "8",
447
+ "--schedule-steps",
448
+ "3500",
449
+ "--selection-metric",
450
+ "crossfit_temperature_nll_v1",
451
+ "--two-pass"
452
+ ],
453
+ "stage_started_utc": "2026-09-17T07:21:43.386357+00:00",
454
+ "heartbeat_utc": "2026-09-17T07:52:16.276307+00:00",
455
+ "termination_reason": "requested_stop",
456
+ "training_exit_code": 0,
457
+ "training_output": "/home/andy/ai/opensysone/runs/20260917T072142Z-24h/training",
458
+ "resume_checkpoint": "/home/andy/ai/opensysone/runs/20260917T072142Z-24h/training/checkpoint.pt"
459
+ },
460
+ "supervisor_running": false,
461
+ "child_running": false,
462
+ "api_running": false
463
+ }
464
+ }
465
+ },
466
+ "heartbeat_utc": "2026-09-17T09:20:21.936006+00:00",
467
+ "selected": "gx10-4b-expanded",
468
+ "child_pid": null,
469
+ "child_command": [
470
+ "/home/andy/ai/envs/opensysone/bin/python",
471
+ "/home/andy/ai/opensysone/source/profile-07f10e7/jev_harness.py",
472
+ "--backend",
473
+ "local",
474
+ "--checkpoint",
475
+ "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/evaluation/model.pt",
476
+ "--request",
477
+ "/home/andy/ai/opensysone/source/profile-07f10e7/examples/jev_request.json",
478
+ "--max-tokens",
479
+ "1024"
480
+ ],
481
+ "stage_started_utc": "2026-09-17T09:20:07.916158+00:00",
482
+ "evaluation_exit_code": 0,
483
+ "harness_check_exit_code": 0,
484
+ "api_pid": 1716630,
485
+ "api_command": [
486
+ "/home/andy/ai/envs/opensysone/bin/python",
487
+ "/home/andy/ai/opensysone/source/profile-07f10e7/jev_harness.py",
488
+ "--backend",
489
+ "serve",
490
+ "--checkpoint",
491
+ "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/evaluation/model.pt",
492
+ "--port",
493
+ "18081",
494
+ "--max-tokens",
495
+ "1024"
496
+ ],
497
+ "url": "http://127.0.0.1:18081",
498
+ "api_ready": true,
499
+ "model": "/home/andy/ai/opensysone/runs/20260916T194403396250Z-fleet/evaluation/model.pt",
500
+ "finished_utc": "2026-09-17T09:20:23.727387+00:00"
501
+ }
final/20260916T194403396250Z-fleet/model.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e270e3da905604d97bf5a8f380ea308133403d1c4790a5c012cb1c12e9b6f348
3
+ size 66221579
final/20260916T194403396250Z-fleet/selected/best.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c4f781e80ade257544100b03c525709d0559eb2bf6a992c97708554c7d360aca
3
+ size 66220939
final/20260916T194403396250Z-fleet/selected/training_correctness.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "branch_chunks_1_probability_max_abs": 0.0,
3
+ "branch_chunks_2_probability_max_abs": 3.5762786865234375e-07,
4
+ "branch_chunks_4_probability_max_abs": 2.384185791015625e-07,
5
+ "question_isolation_probability_max_abs": 0.0,
6
+ "candidate_permutation_probability_max_abs": 0.0,
7
+ "repeat_probability_max_abs": 0.0,
8
+ "tolerance_probability_abs": 0.0001
9
+ }
final/20260916T194403396250Z-fleet/selected/training_manifest.json ADDED
@@ -0,0 +1,416 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "config": {
3
+ "command": "train",
4
+ "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
5
+ "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917",
6
+ "output": "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts",
7
+ "resume": null,
8
+ "warm_start": "/home/andy/ai/opensysone/runs/20260917T070415Z-expanded-parent/best.pt",
9
+ "allow_train_data_change": true,
10
+ "steps": 8,
11
+ "epochs": 3,
12
+ "rank": 8,
13
+ "alpha": 16.0,
14
+ "head_only": false,
15
+ "two_pass": true,
16
+ "lr": 2e-05,
17
+ "head_lr": 2e-05,
18
+ "seed": 433,
19
+ "effective_batch": 4,
20
+ "branch_batch_size": 1,
21
+ "max_tokens": 512,
22
+ "save_steps": 250,
23
+ "save_seconds": 900,
24
+ "eval_steps": 500,
25
+ "validation_per_family": 128,
26
+ "patience": 8,
27
+ "deadline": "2026-09-17T16:00:00Z",
28
+ "schedule_steps": 3500,
29
+ "selection_metric": "crossfit_temperature_nll_v1",
30
+ "adapters": true
31
+ },
32
+ "pid": 1630638,
33
+ "hostname": "gx10-9dd0",
34
+ "started_utc": "2026-09-17T07:21:45.071835+00:00",
35
+ "git_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86",
36
+ "git_status": "",
37
+ "source_sha256": {
38
+ "playground.py": "b10c400421dd8558a7fef8ddde632676cdfe7f63edf94184f299f9ed569c010a",
39
+ "selection.py": "be0a7a8496b5b830aa572ceba93606320f442063fd38180503fd6980dc1c578f",
40
+ "training_model.py": "d5b0aefeeb5290816bc0b669aa0a8cbbe27f6a12b9cb23c141ac9b9ae9ee4e65",
41
+ "decision_model.py": "a3d8aeb02a1ac765c6cc30ff175acad0664560f01ab5403e22cade924d17371e",
42
+ "data_transition.py": "93aaa89b4de3aa78c34f03a5643e31f91f738c832395334966368df9b902621c",
43
+ "smoke_data.py": "06b3cbac1c8c4a86b8aecbee4459073cc3e46d4ddcd576392f3cb4805924f815",
44
+ "jev_harness.py": "4d4e979cb7ae352bcdacaaa6d64045e6b5e550b1a9721d4bad545045bee6c67f",
45
+ "experiment.py": "c779c3936aa1c2c51052f035df7bc0895a2de79c9ffc6c50fb0ee848832e17c7",
46
+ "smoke_train.py": "8cdeb2b397177fc9c26638aaa871501ddab8e3871aa1573ecd98f66590f5c228",
47
+ "scripts/run_experiment.sh": "661a6309fc54a2a8aff918f14a553c72dcb21730bd6a3cfc55d6ccd4700d11d3",
48
+ "scripts/download_candidate.py": "d06a2c01be0cf6577f927fb37e3bc1eab014949fd934e4f4d9825674adba608e",
49
+ "scripts/prepare_public_data.py": "32ea84aa719818e1141b958b6ef27a85f7ddb86bfcd7c1585fc487b25253d977",
50
+ "scripts/verify_playground_layout.cjs": "dea3fa07c568c141cd58fead8c547db7196f4a48fe4ea4cea0312d2ba7bd8bb4",
51
+ "scripts/run_smoke.sh": "39d59f2120f362729d1c2e384391b82be1e580dcc1aca0eaa7ab231115225574",
52
+ "scripts/start_spark_candidate.sh": "c2ca18b008a144de7cb264c9fcca634d68c4a8a317db3638e8a70b3dbcff064b",
53
+ "scripts/launch_24h.py": "39c26dc10535d3adac09209b0743edf2ed384563e732512ad62f8c6f37161a83",
54
+ "scripts/verify_artifact.py": "9833350e9d72c0065b15206bb71c5a8b5a6b3185219db90074e369ede563a985",
55
+ "scripts/publish_hf_snapshot.py": "b5fe16a00fcbc5ab97121428c6ce750275ee193438e3ffe24aac5e3bb325c018",
56
+ "scripts/prepare_expanded_data.py": "5c05478b29c84218784690f3c7c3ec994615fec1c57826591607189f191c56b4",
57
+ "scripts/fleet_campaign.py": "e69fdff96f92c6943b7895be11df444f016d0b744a1b9441995a5f8bb7af9d54",
58
+ "scripts/run_precision.sh": "766d82b30cf3e83951f662685b4472ee053c7fbc171c4ff140823bf4d1c2782f",
59
+ "scripts/fleet_status.py": "2519ced157ef4ac4fa449eebabae5740d7527778d578b4ac6720583010fa5217",
60
+ "scripts/verify_playground.cjs": "a55aa0e945baeb9e536c2b9cce7a9aa2364244b587e90477190add96f78601dc",
61
+ "scripts/diagnose_parity.py": "08b5d66d316ebda98a2226251a4f952701f86a1d5726ce7a4d7e8fb22755da5a",
62
+ "scripts/publish_hf_final.py": "278efc5878d7ebc5d1171f3a735d6c7d78275d93650e5a8a16c3f55b8353a577",
63
+ "scripts/download_model.py": "72ad9a5de44d09e2ee4ed8afb7c3c0ff6fb48987410a3f7f0368353572bf1f4d",
64
+ "scripts/investigate_precision.py": "609b744a926d8a45b87ba8d225e5312ee0c71a7096b21bd3589c5846e1dc847c",
65
+ "scripts/campaign_status.py": "1500b5e24f06231c7aafd7277aefd840582e35997e265f79db93614828d34411"
66
+ },
67
+ "packages": {
68
+ "torch": "2.11.0+cu130",
69
+ "transformers": "5.15.0",
70
+ "pyarrow": "25.0.1",
71
+ "numpy": "2.5.2"
72
+ },
73
+ "cuda": "13.0",
74
+ "gpu": "NVIDIA GB10",
75
+ "capability": [
76
+ 12,
77
+ 1
78
+ ],
79
+ "cuda_cap_bytes": 17179869184,
80
+ "initial_mem_available_bytes": 106574667776,
81
+ "oom_score_adj": "0",
82
+ "parent_checkpoint_sha256": "cf4cd775165cc2bb02113d17b94facd0db7b94fcff87b27389b3280fbb712469",
83
+ "initialization": {
84
+ "kind": "resume",
85
+ "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260917T070758Z-train/artifacts/checkpoint.pt",
86
+ "parent_checkpoint_sha256": "cf4cd775165cc2bb02113d17b94facd0db7b94fcff87b27389b3280fbb712469",
87
+ "restores_optimizer": true,
88
+ "restores_rng": true,
89
+ "parent_step": 8,
90
+ "parent_source_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86",
91
+ "parent_initialization": {
92
+ "kind": "warm_start",
93
+ "parent_checkpoint": "/home/andy/ai/opensysone/runs/20260917T070415Z-expanded-parent/best.pt",
94
+ "parent_checkpoint_sha256": "5f57ec38796d132edfa23638fbce66131fd4e7dfe87ceeadaba2b0e0d5c78024",
95
+ "restores_optimizer": false,
96
+ "restores_rng": false,
97
+ "parent_step": 1500,
98
+ "parent_source_commit": "4a60423c39d70f8d50472ce4f4f7fa4a4bd9fce1",
99
+ "data_transition": {
100
+ "kind": "training_split_only",
101
+ "parent_dataset": "/home/andy/ai/opensysone/data/public-decisions-v1-20260916",
102
+ "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917",
103
+ "parent_data_signature": "c00527e505fba87aabbfa65ba5fb7a67cf628ac7d46d7598f270936a894e3884",
104
+ "data_signature": "76183c642668602f42b7f3e71a3fe03bd5bd76f064fce4ba92351d8703396207",
105
+ "parent_manifest_sha256": "adf5a8ca2bab60cf7429a82b1b7a2d1ae7d3cc812de04dd195a478dfd7c6f628",
106
+ "manifest_sha256": "fde6ee7ce2eca20cb22cdbbe4db0ddbdb29a8ea9d597906d88e545939a5b602c",
107
+ "protected_split_sha256": {
108
+ "validation": "411199524c930d33fed8e1afa24597c59d400a500195fbec33d96afafd7ce74f",
109
+ "calibration": "58fea4f180f16e5e0f2c9fd5f57487d3106415ec8e37bdd003e629ea944c51b4",
110
+ "test": "ea61477a192d0a7174bcf0536f7664b547fdefe0bca9fe02c8e13c621e380819",
111
+ "holdout": "0fb1bf6baf32374cf5dd8059428f058929884c903daa26be518ef14d33510cc3"
112
+ }
113
+ }
114
+ },
115
+ "selection_policy_change": {
116
+ "from": "crossfit_temperature_nll_v1",
117
+ "to": "crossfit_temperature_nll_v1",
118
+ "optimizer_and_rng_unchanged": true
119
+ }
120
+ },
121
+ "model_provenance": {
122
+ "model_id": "Qwen/Qwen3-4B-Instruct-2507",
123
+ "revision": "cdbee75f17c01a7cc42f958dc650907174af0554",
124
+ "license": "apache-2.0"
125
+ },
126
+ "data_signature": "76183c642668602f42b7f3e71a3fe03bd5bd76f064fce4ba92351d8703396207",
127
+ "total_parameters": 4038985729,
128
+ "trainable_parameters": 16517633,
129
+ "adapter_modules": [
130
+ "layers.0.self_attn.q_proj",
131
+ "layers.0.self_attn.k_proj",
132
+ "layers.0.self_attn.v_proj",
133
+ "layers.0.self_attn.o_proj",
134
+ "layers.0.mlp.gate_proj",
135
+ "layers.0.mlp.up_proj",
136
+ "layers.0.mlp.down_proj",
137
+ "layers.1.self_attn.q_proj",
138
+ "layers.1.self_attn.k_proj",
139
+ "layers.1.self_attn.v_proj",
140
+ "layers.1.self_attn.o_proj",
141
+ "layers.1.mlp.gate_proj",
142
+ "layers.1.mlp.up_proj",
143
+ "layers.1.mlp.down_proj",
144
+ "layers.2.self_attn.q_proj",
145
+ "layers.2.self_attn.k_proj",
146
+ "layers.2.self_attn.v_proj",
147
+ "layers.2.self_attn.o_proj",
148
+ "layers.2.mlp.gate_proj",
149
+ "layers.2.mlp.up_proj",
150
+ "layers.2.mlp.down_proj",
151
+ "layers.3.self_attn.q_proj",
152
+ "layers.3.self_attn.k_proj",
153
+ "layers.3.self_attn.v_proj",
154
+ "layers.3.self_attn.o_proj",
155
+ "layers.3.mlp.gate_proj",
156
+ "layers.3.mlp.up_proj",
157
+ "layers.3.mlp.down_proj",
158
+ "layers.4.self_attn.q_proj",
159
+ "layers.4.self_attn.k_proj",
160
+ "layers.4.self_attn.v_proj",
161
+ "layers.4.self_attn.o_proj",
162
+ "layers.4.mlp.gate_proj",
163
+ "layers.4.mlp.up_proj",
164
+ "layers.4.mlp.down_proj",
165
+ "layers.5.self_attn.q_proj",
166
+ "layers.5.self_attn.k_proj",
167
+ "layers.5.self_attn.v_proj",
168
+ "layers.5.self_attn.o_proj",
169
+ "layers.5.mlp.gate_proj",
170
+ "layers.5.mlp.up_proj",
171
+ "layers.5.mlp.down_proj",
172
+ "layers.6.self_attn.q_proj",
173
+ "layers.6.self_attn.k_proj",
174
+ "layers.6.self_attn.v_proj",
175
+ "layers.6.self_attn.o_proj",
176
+ "layers.6.mlp.gate_proj",
177
+ "layers.6.mlp.up_proj",
178
+ "layers.6.mlp.down_proj",
179
+ "layers.7.self_attn.q_proj",
180
+ "layers.7.self_attn.k_proj",
181
+ "layers.7.self_attn.v_proj",
182
+ "layers.7.self_attn.o_proj",
183
+ "layers.7.mlp.gate_proj",
184
+ "layers.7.mlp.up_proj",
185
+ "layers.7.mlp.down_proj",
186
+ "layers.8.self_attn.q_proj",
187
+ "layers.8.self_attn.k_proj",
188
+ "layers.8.self_attn.v_proj",
189
+ "layers.8.self_attn.o_proj",
190
+ "layers.8.mlp.gate_proj",
191
+ "layers.8.mlp.up_proj",
192
+ "layers.8.mlp.down_proj",
193
+ "layers.9.self_attn.q_proj",
194
+ "layers.9.self_attn.k_proj",
195
+ "layers.9.self_attn.v_proj",
196
+ "layers.9.self_attn.o_proj",
197
+ "layers.9.mlp.gate_proj",
198
+ "layers.9.mlp.up_proj",
199
+ "layers.9.mlp.down_proj",
200
+ "layers.10.self_attn.q_proj",
201
+ "layers.10.self_attn.k_proj",
202
+ "layers.10.self_attn.v_proj",
203
+ "layers.10.self_attn.o_proj",
204
+ "layers.10.mlp.gate_proj",
205
+ "layers.10.mlp.up_proj",
206
+ "layers.10.mlp.down_proj",
207
+ "layers.11.self_attn.q_proj",
208
+ "layers.11.self_attn.k_proj",
209
+ "layers.11.self_attn.v_proj",
210
+ "layers.11.self_attn.o_proj",
211
+ "layers.11.mlp.gate_proj",
212
+ "layers.11.mlp.up_proj",
213
+ "layers.11.mlp.down_proj",
214
+ "layers.12.self_attn.q_proj",
215
+ "layers.12.self_attn.k_proj",
216
+ "layers.12.self_attn.v_proj",
217
+ "layers.12.self_attn.o_proj",
218
+ "layers.12.mlp.gate_proj",
219
+ "layers.12.mlp.up_proj",
220
+ "layers.12.mlp.down_proj",
221
+ "layers.13.self_attn.q_proj",
222
+ "layers.13.self_attn.k_proj",
223
+ "layers.13.self_attn.v_proj",
224
+ "layers.13.self_attn.o_proj",
225
+ "layers.13.mlp.gate_proj",
226
+ "layers.13.mlp.up_proj",
227
+ "layers.13.mlp.down_proj",
228
+ "layers.14.self_attn.q_proj",
229
+ "layers.14.self_attn.k_proj",
230
+ "layers.14.self_attn.v_proj",
231
+ "layers.14.self_attn.o_proj",
232
+ "layers.14.mlp.gate_proj",
233
+ "layers.14.mlp.up_proj",
234
+ "layers.14.mlp.down_proj",
235
+ "layers.15.self_attn.q_proj",
236
+ "layers.15.self_attn.k_proj",
237
+ "layers.15.self_attn.v_proj",
238
+ "layers.15.self_attn.o_proj",
239
+ "layers.15.mlp.gate_proj",
240
+ "layers.15.mlp.up_proj",
241
+ "layers.15.mlp.down_proj",
242
+ "layers.16.self_attn.q_proj",
243
+ "layers.16.self_attn.k_proj",
244
+ "layers.16.self_attn.v_proj",
245
+ "layers.16.self_attn.o_proj",
246
+ "layers.16.mlp.gate_proj",
247
+ "layers.16.mlp.up_proj",
248
+ "layers.16.mlp.down_proj",
249
+ "layers.17.self_attn.q_proj",
250
+ "layers.17.self_attn.k_proj",
251
+ "layers.17.self_attn.v_proj",
252
+ "layers.17.self_attn.o_proj",
253
+ "layers.17.mlp.gate_proj",
254
+ "layers.17.mlp.up_proj",
255
+ "layers.17.mlp.down_proj",
256
+ "layers.18.self_attn.q_proj",
257
+ "layers.18.self_attn.k_proj",
258
+ "layers.18.self_attn.v_proj",
259
+ "layers.18.self_attn.o_proj",
260
+ "layers.18.mlp.gate_proj",
261
+ "layers.18.mlp.up_proj",
262
+ "layers.18.mlp.down_proj",
263
+ "layers.19.self_attn.q_proj",
264
+ "layers.19.self_attn.k_proj",
265
+ "layers.19.self_attn.v_proj",
266
+ "layers.19.self_attn.o_proj",
267
+ "layers.19.mlp.gate_proj",
268
+ "layers.19.mlp.up_proj",
269
+ "layers.19.mlp.down_proj",
270
+ "layers.20.self_attn.q_proj",
271
+ "layers.20.self_attn.k_proj",
272
+ "layers.20.self_attn.v_proj",
273
+ "layers.20.self_attn.o_proj",
274
+ "layers.20.mlp.gate_proj",
275
+ "layers.20.mlp.up_proj",
276
+ "layers.20.mlp.down_proj",
277
+ "layers.21.self_attn.q_proj",
278
+ "layers.21.self_attn.k_proj",
279
+ "layers.21.self_attn.v_proj",
280
+ "layers.21.self_attn.o_proj",
281
+ "layers.21.mlp.gate_proj",
282
+ "layers.21.mlp.up_proj",
283
+ "layers.21.mlp.down_proj",
284
+ "layers.22.self_attn.q_proj",
285
+ "layers.22.self_attn.k_proj",
286
+ "layers.22.self_attn.v_proj",
287
+ "layers.22.self_attn.o_proj",
288
+ "layers.22.mlp.gate_proj",
289
+ "layers.22.mlp.up_proj",
290
+ "layers.22.mlp.down_proj",
291
+ "layers.23.self_attn.q_proj",
292
+ "layers.23.self_attn.k_proj",
293
+ "layers.23.self_attn.v_proj",
294
+ "layers.23.self_attn.o_proj",
295
+ "layers.23.mlp.gate_proj",
296
+ "layers.23.mlp.up_proj",
297
+ "layers.23.mlp.down_proj",
298
+ "layers.24.self_attn.q_proj",
299
+ "layers.24.self_attn.k_proj",
300
+ "layers.24.self_attn.v_proj",
301
+ "layers.24.self_attn.o_proj",
302
+ "layers.24.mlp.gate_proj",
303
+ "layers.24.mlp.up_proj",
304
+ "layers.24.mlp.down_proj",
305
+ "layers.25.self_attn.q_proj",
306
+ "layers.25.self_attn.k_proj",
307
+ "layers.25.self_attn.v_proj",
308
+ "layers.25.self_attn.o_proj",
309
+ "layers.25.mlp.gate_proj",
310
+ "layers.25.mlp.up_proj",
311
+ "layers.25.mlp.down_proj",
312
+ "layers.26.self_attn.q_proj",
313
+ "layers.26.self_attn.k_proj",
314
+ "layers.26.self_attn.v_proj",
315
+ "layers.26.self_attn.o_proj",
316
+ "layers.26.mlp.gate_proj",
317
+ "layers.26.mlp.up_proj",
318
+ "layers.26.mlp.down_proj",
319
+ "layers.27.self_attn.q_proj",
320
+ "layers.27.self_attn.k_proj",
321
+ "layers.27.self_attn.v_proj",
322
+ "layers.27.self_attn.o_proj",
323
+ "layers.27.mlp.gate_proj",
324
+ "layers.27.mlp.up_proj",
325
+ "layers.27.mlp.down_proj",
326
+ "layers.28.self_attn.q_proj",
327
+ "layers.28.self_attn.k_proj",
328
+ "layers.28.self_attn.v_proj",
329
+ "layers.28.self_attn.o_proj",
330
+ "layers.28.mlp.gate_proj",
331
+ "layers.28.mlp.up_proj",
332
+ "layers.28.mlp.down_proj",
333
+ "layers.29.self_attn.q_proj",
334
+ "layers.29.self_attn.k_proj",
335
+ "layers.29.self_attn.v_proj",
336
+ "layers.29.self_attn.o_proj",
337
+ "layers.29.mlp.gate_proj",
338
+ "layers.29.mlp.up_proj",
339
+ "layers.29.mlp.down_proj",
340
+ "layers.30.self_attn.q_proj",
341
+ "layers.30.self_attn.k_proj",
342
+ "layers.30.self_attn.v_proj",
343
+ "layers.30.self_attn.o_proj",
344
+ "layers.30.mlp.gate_proj",
345
+ "layers.30.mlp.up_proj",
346
+ "layers.30.mlp.down_proj",
347
+ "layers.31.self_attn.q_proj",
348
+ "layers.31.self_attn.k_proj",
349
+ "layers.31.self_attn.v_proj",
350
+ "layers.31.self_attn.o_proj",
351
+ "layers.31.mlp.gate_proj",
352
+ "layers.31.mlp.up_proj",
353
+ "layers.31.mlp.down_proj",
354
+ "layers.32.self_attn.q_proj",
355
+ "layers.32.self_attn.k_proj",
356
+ "layers.32.self_attn.v_proj",
357
+ "layers.32.self_attn.o_proj",
358
+ "layers.32.mlp.gate_proj",
359
+ "layers.32.mlp.up_proj",
360
+ "layers.32.mlp.down_proj",
361
+ "layers.33.self_attn.q_proj",
362
+ "layers.33.self_attn.k_proj",
363
+ "layers.33.self_attn.v_proj",
364
+ "layers.33.self_attn.o_proj",
365
+ "layers.33.mlp.gate_proj",
366
+ "layers.33.mlp.up_proj",
367
+ "layers.33.mlp.down_proj",
368
+ "layers.34.self_attn.q_proj",
369
+ "layers.34.self_attn.k_proj",
370
+ "layers.34.self_attn.v_proj",
371
+ "layers.34.self_attn.o_proj",
372
+ "layers.34.mlp.gate_proj",
373
+ "layers.34.mlp.up_proj",
374
+ "layers.34.mlp.down_proj",
375
+ "layers.35.self_attn.q_proj",
376
+ "layers.35.self_attn.k_proj",
377
+ "layers.35.self_attn.v_proj",
378
+ "layers.35.self_attn.o_proj",
379
+ "layers.35.mlp.gate_proj",
380
+ "layers.35.mlp.up_proj",
381
+ "layers.35.mlp.down_proj"
382
+ ],
383
+ "load_and_tokenize_seconds": 4.576527460012585,
384
+ "final_validation": {
385
+ "status": "running",
386
+ "stage": "validating",
387
+ "pid": 1669972,
388
+ "command": [
389
+ "/home/andy/ai/envs/opensysone/bin/python",
390
+ "/home/andy/ai/opensysone/source/wrapup-validation-20260917/scripts/final_validation.py",
391
+ "--checkpoint",
392
+ "/home/andy/ai/opensysone/runs/20260917T075209Z-training-wrapup/gx10-expanded-snapshot/checkpoint.pt",
393
+ "--reference-predictions",
394
+ "/home/andy/ai/opensysone/runs/20260916T183823Z-train/artifacts/validation_step_000040_predictions.json",
395
+ "--reference-sha256",
396
+ "e671e1508185765552b0f933ba03f356be62143c531d8ef534457d34b1645c9b",
397
+ "--output",
398
+ "/home/andy/ai/opensysone/runs/20260917T075209Z-gx10-final-validation",
399
+ "--deadline",
400
+ "2026-09-17T09:00:00Z"
401
+ ],
402
+ "started_utc": "2026-09-17T07:58:09.650481+00:00",
403
+ "deadline": "2026-09-17T09:00:00Z",
404
+ "helper_sha256": "f5cea8bad330dd066f43b3dea5a977dbe3e8d5d69336d2850279bb743b628643",
405
+ "helper_source_commit": "18f2b39506d4f4e6bdbed0eb6cdcb08cd50dcb57",
406
+ "reserved_predictions_accessed": false,
407
+ "heartbeat_utc": "2026-09-17T08:04:10.665456+00:00",
408
+ "latest_step": 159,
409
+ "previous_best_step": 0,
410
+ "source_checkpoint_sha256": "dcd9a12c8e2a5d6d2812372fedb6df13953c607f810167bf193f93d675f0bda0",
411
+ "source_best_sha256": "c4f781e80ade257544100b03c525709d0559eb2bf6a992c97708554c7d360aca",
412
+ "completed_decisions": 512,
413
+ "source_manifest": "/home/andy/ai/opensysone/runs/20260917T075209Z-training-wrapup/gx10-expanded-snapshot/manifest.json",
414
+ "selected_training_source_commit": "24b8ccf60d388f9cbb184e03a6ae260a1f5a8b86"
415
+ }
416
+ }
final/20260916T194403396250Z-fleet/selected/validation_predictions.json ADDED
The diff for this file is too large to render. See raw diff
 
final/20260916T194403396250Z-fleet/source.tar.gz ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:aecd66cb0bea0e1e7f1f275d46ffb173f40bd0095898f2a197b409d6243ca52e
3
+ size 2115783