opensysone / source /results /20260917-expanded-data /tokenization-proof.json
andyshu's picture
Back up verified OpenSysOne training snapshot and pinned source
1a0a7fb verified
Raw History Blame
2.22 kB
{
"cuda_initialized": false,
"data_signature": "76183c642668602f42b7f3e71a3fe03bd5bd76f064fce4ba92351d8703396207",
"dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917",
"diagnostics": {
"dropped_ids": [
"piqa:train:5528"
],
"family_counts": {
"commonsenseqa": 128,
"hellaswag": 128,
"piqa": 127
},
"max_branch_tokens": 242,
"retained": 383
},
"max_tokens": 512,
"model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f",
"model_provenance": {
"license": "apache-2.0",
"model_id": "Qwen/Qwen3-4B-Instruct-2507",
"revision": "cdbee75f17c01a7cc42f958dc650907174af0554"
},
"original_training_replay_exact": true,
"original_training_replay_retained": 40915,
"production_sequences": "training_model.TrainableScorer.sequences (same method, tokenizer only)",
"protected_token_replay": {
"calibration": {
"exact_equal": true,
"retained": 510,
"retained_id_sha256": "8a0d4add2dd95717d34915195dc7100878b8f443bf714656d54b336b629f6476"
},
"holdout": {
"exact_equal": true,
"retained": 768,
"retained_id_sha256": "805387bd9156d12d4d40b33e5926f209451f323814ad876a3f04a8abece3ae0b"
},
"test": {
"exact_equal": true,
"retained": 2042,
"retained_id_sha256": "343960f63f954a0c05884459a13a3ef8560fd98c2caead3aa3c3b70022917ec5"
},
"validation": {
"exact_equal": true,
"retained": 512,
"retained_id_sha256": "b0ea1f550bee363d58849a00295771707fca92d9d8e630840d0f870a91a9f8a7"
}
},
"seconds": 60.95969182101544,
"token_cache": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917/tokens-76183c642668602f.pt",
"token_cache_bytes": 166924109,
"token_cache_sha256": "4513bf6dbe68aab0503f917d895eb3b2b644386d28876a9946045082f7835d42",
"tokenizer_helper_sha256": "f30b0dab7c917f74676fd1cffc988ffb175ddb9a843f2029ce715d7e44bc7f96",
"train_family_counts": {
"arc": 3345,
"banking": 9608,
"boolq": 7962,
"commonsenseqa": 9490,
"hellaswag": 16000,
"piqa": 14360,
"snli": 20000
},
"training_model_sha256": "d5b0aefeeb5290816bc0b669aa0a8cbbe27f6a12b9cb23c141ac9b9ae9ee4e65"
}