Download source/results/20260917-expanded-data/tokenization-proof.json from andyshu/opensysone: direct link, hf CLI and curl.
- Browser
- Download file 2.22 kB
-
https://huggingface.co/andyshu/opensysone/resolve/f2d6f8daa15bd21c316e61249f45ac16cbb79d45/source/results/20260917-expanded-data/tokenization-proof.json
- Command line
-
hf download hf://andyshu/opensysone@f2d6f8daa15bd21c316e61249f45ac16cbb79d45/source/results/20260917-expanded-data/tokenization-proof.json
-
curl -L -o tokenization-proof.json https://huggingface.co/andyshu/opensysone/resolve/f2d6f8daa15bd21c316e61249f45ac16cbb79d45/source/results/20260917-expanded-data/tokenization-proof.json
2.22 kB
| { | |
| "cuda_initialized": false, | |
| "data_signature": "76183c642668602f42b7f3e71a3fe03bd5bd76f064fce4ba92351d8703396207", | |
| "dataset": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917", | |
| "diagnostics": { | |
| "dropped_ids": [ | |
| "piqa:train:5528" | |
| ], | |
| "family_counts": { | |
| "commonsenseqa": 128, | |
| "hellaswag": 128, | |
| "piqa": 127 | |
| }, | |
| "max_branch_tokens": 242, | |
| "retained": 383 | |
| }, | |
| "max_tokens": 512, | |
| "model": "/home/andy/ai/models/opensysone/Qwen3-4B-Instruct-2507-cdbee75f", | |
| "model_provenance": { | |
| "license": "apache-2.0", | |
| "model_id": "Qwen/Qwen3-4B-Instruct-2507", | |
| "revision": "cdbee75f17c01a7cc42f958dc650907174af0554" | |
| }, | |
| "original_training_replay_exact": true, | |
| "original_training_replay_retained": 40915, | |
| "production_sequences": "training_model.TrainableScorer.sequences (same method, tokenizer only)", | |
| "protected_token_replay": { | |
| "calibration": { | |
| "exact_equal": true, | |
| "retained": 510, | |
| "retained_id_sha256": "8a0d4add2dd95717d34915195dc7100878b8f443bf714656d54b336b629f6476" | |
| }, | |
| "holdout": { | |
| "exact_equal": true, | |
| "retained": 768, | |
| "retained_id_sha256": "805387bd9156d12d4d40b33e5926f209451f323814ad876a3f04a8abece3ae0b" | |
| }, | |
| "test": { | |
| "exact_equal": true, | |
| "retained": 2042, | |
| "retained_id_sha256": "343960f63f954a0c05884459a13a3ef8560fd98c2caead3aa3c3b70022917ec5" | |
| }, | |
| "validation": { | |
| "exact_equal": true, | |
| "retained": 512, | |
| "retained_id_sha256": "b0ea1f550bee363d58849a00295771707fca92d9d8e630840d0f870a91a9f8a7" | |
| } | |
| }, | |
| "seconds": 60.95969182101544, | |
| "token_cache": "/home/andy/ai/opensysone/data/public-decisions-v2-20260917/tokens-76183c642668602f.pt", | |
| "token_cache_bytes": 166924109, | |
| "token_cache_sha256": "4513bf6dbe68aab0503f917d895eb3b2b644386d28876a9946045082f7835d42", | |
| "tokenizer_helper_sha256": "f30b0dab7c917f74676fd1cffc988ffb175ddb9a843f2029ce715d7e44bc7f96", | |
| "train_family_counts": { | |
| "arc": 3345, | |
| "banking": 9608, | |
| "boolq": 7962, | |
| "commonsenseqa": 9490, | |
| "hellaswag": 16000, | |
| "piqa": 14360, | |
| "snli": 20000 | |
| }, | |
| "training_model_sha256": "d5b0aefeeb5290816bc0b669aa0a8cbbe27f6a12b9cb23c141ac9b9ae9ee4e65" | |
| } | |