File size: 6,384 Bytes
99df2d0 8bda8c7 b9abe2f 99df2d0 8bda8c7 b9abe2f 8bda8c7 99df2d0 b9abe2f e780025 99df2d0 5e571cd e780025 5e571cd b9abe2f 5e571cd b9abe2f 5e571cd 698984b 5e571cd b9abe2f e780025 b9abe2f 5e571cd 698984b b9abe2f e780025 b9abe2f 5e571cd e780025 b9abe2f 698984b 5e571cd 8bda8c7 b9abe2f cd7e692 e780025 b9abe2f 8bda8c7 5e571cd b9abe2f e780025 5e571cd b9abe2f 5e571cd e780025 b9abe2f e780025 b9abe2f 5e571cd b9abe2f cd7e692 e780025 b9abe2f 5e571cd b9abe2f e780025 b9abe2f 99df2d0 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 | {
"schema_version": 3,
"revision": 6,
"revision_id": "r006",
"release": "Pollock 1.4",
"model_id": "SlayerLab/pollock-mini-lm-125m",
"publication": {
"weights_commit": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9",
"version_tag": "v1.4",
"publication_commit_pending": true
},
"source_checkpoint": {
"path_in_training_workspace": "runs/r006-corrected-corpus-v2-s1337-lr0.0004/checkpoints/ckpt-final.pt",
"sha256": "580fbb3c039ea1f349285a9bf7a67731fb6d455de931b6b7601d4ed09fce9437",
"iteration": 22003,
"tokens_seen": 10814914560,
"native_nanogpt_parameters": 126637952,
"native_unique_trainable_parameters": 127555456
},
"architecture": {
"n_layer": 12,
"n_head": 14,
"n_embd": 896,
"block_size": 1024,
"vocab_size": 12288,
"dropout": 0.0,
"bias": false,
"tied_word_embeddings": true
},
"training": {
"dataset": "SlayerLab/minimal-en-corpus-2.5b",
"dataset_revision": "d68d992622e9fc11f19e7d7fb8547c4e653439a4",
"training_tokens": 2689323439,
"validation_tokens": 5236486,
"tokenizer_sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51",
"native_tokenizer_sha256": "d0d126f0a59e51e8cb1b26d77b5527870be0e8ea1e84adefd730fff906c48234",
"init_from": "scratch",
"micro_batch_per_gpu": 12,
"gradient_accumulation_global": 40,
"gradient_accumulation_per_gpu": 20,
"ddp_world_size": 2,
"effective_batch_tokens": 491520,
"optimizer": "fused AdamW",
"learning_rate": 0.0004,
"min_learning_rate": 0.00004,
"schedule": "cosine",
"warmup_iters": 440,
"beta1": 0.9,
"beta2": 0.95,
"weight_decay": 0.1,
"grad_clip": 1.0,
"precision": "bfloat16",
"hardware": "2x NVIDIA GeForce RTX 4090",
"framework": "PyTorch 2.8.0+cu128",
"nanogpt_commit": "3adf61e154c3fe3fca428ad6bc3818b27a3b8291",
"data_pass_equivalent": 4.021425762020438,
"runtime_hours": 15.226956854563,
"mean_tokens_per_second": 199894.6379482684
},
"evaluation": {
"protocol": "fixed sampled subset",
"subset_tokens_per_split": 1228800,
"final_train_loss": 2.48974818944931,
"final_validation_loss": 2.536356544494629,
"best_validation_loss": 2.5362311387062073,
"best_validation_update": 22000,
"best_validation_tokens_seen": 10813440000,
"final_validation_perplexity": 12.633557211556399,
"benchmark_harness": "lm-evaluation-harness 0.4.12",
"benchmark_num_fewshot": 0,
"benchmark_batch_size": 8,
"truncated_benchmark_requests": 0,
"benchmark_checkpoint_sha256": "9339d20ea32d24bc9777518a2f0ec553e4a3871647d342db92d7c5b6c18c7b56",
"benchmark_tokenizer_sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51",
"results_file": "benchmarks/english.json"
},
"fixed_inference": {
"suite_id": "fixed-sampling-v1",
"comparison_file": "inference-samples/README.md",
"config_file": "inference-samples/config.json",
"generator_file": "inference-samples/generate.py",
"results_file": "inference-samples/results.json",
"operating_system": "macOS 26.2",
"architecture": "arm64",
"device": "cpu",
"dtype": "float32",
"seed": 1337,
"prompt_count": 4,
"revision_count": 6,
"local_review_artifact": false,
"exact_replay_verified": true,
"publication_commit_pending": true
},
"conversion": {
"target_class": "GPT2LMHeadModel",
"transformers_version": "5.15.1",
"checkpoint_sha256": "580fbb3c039ea1f349285a9bf7a67731fb6d455de931b6b7601d4ed09fce9437",
"model_sha256": "8e74aa4d34464229e86a1831fb34d442f91e22aa600e0cf2503891957fdc6a43",
"unique_serialized_parameters": 127674624,
"compatibility_zero_bias_parameters": 119168,
"validation_probe_shape": [2, 64],
"max_absolute_logit_error": 0.0
},
"artifacts": {
".gitattributes": {"sha256": "c759491a998899dbefaec4d51cc791e68c714a37dd9f9829020a368788fe3063"},
"CHANGELOG.md": {"sha256": "8422ad337b22be7adb561d1b2388cf53e5a6d1c6d49d628468498c72f078769a"},
"LICENSE.md": {"sha256": "46cbe928ed0aa24875f02f774313b27ec9d8abdf41f0c9ef67e0adb4e0614de4"},
"README.md": {"sha256": "875fa9cd748691e376df905e036a0c471db77aaa1272d71c5ea10d0f76fa888d"},
"assets/pollock-mini-lm-avatar-320.png": {"sha256": "7be10cc9d0f4aedeb298d9a5d722b2b2ac5b2e219b3884916dbca16198f70750"},
"benchmarks/english.json": {"sha256": "646a3aff7c97859e6974baa2c191702829a982af87675870dd1a339f8073a146"},
"config.json": {"sha256": "1529f8a8fc31b4fa68fc18fffbad7e155dc8f9e8663a40b4f3306b4012ea38c0"},
"generation_config.json": {"sha256": "435beb27be51f0ed054f4a011e5109d125cdadc118b8799b18b155cc798d94d2"},
"inference-samples/README.md": {"sha256": "6559c85e1ed70f47f2d2978f28151fdd9e3355e397c5591cbe0b8ab767fa5ac5"},
"inference-samples/config.json": {"sha256": "a1b306a1e1f3a70942706d00107a83dd98289a534d17af78adbb95d48c9b69c1"},
"inference-samples/generate.py": {"sha256": "807a9946472bd0087dd2b9260ee90d1dea09785d24d4f923f028667c53a23933"},
"inference-samples/requirements.txt": {"sha256": "584583335ffb3061aa62058aee725b0de25fed7523904fbd13fb4622f601943d"},
"inference-samples/results.json": {"sha256": "a279806ea225130b0e0aa36d9bd13653401e236395fe64b5f0aa7c825d445620"},
"model.safetensors": {"sha256": "8e74aa4d34464229e86a1831fb34d442f91e22aa600e0cf2503891957fdc6a43"},
"special_tokens_map.json": {"sha256": "8b2257a17ea997bb038f43b133aefec82344ad2b8abc2b8a02a6c0a994ed624e"},
"tokenizer.json": {"sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51"},
"tokenizer_config.json": {"sha256": "4cdabe37dbdc1adfcc017ee9a1f86ab89bdf184827d2d9f05181cae0f8af19bf"},
"training-history/r001.md": {"sha256": "5b0ac39265921ed3604e7dfe8181b00d07902640d8024f591462eeb508bcfece"},
"training-history/r002.md": {"sha256": "80031a84b2b67dc4cb4f81a178fa354feea15cb184d17368ee7e64ec3d0a2629"},
"training-history/r003.md": {"sha256": "7ea9e6b15a0af635ff238417b9fff2a98c12a9090800a0923d622253b7eab02d"},
"training-history/r004.md": {"sha256": "e260bfba69f154caf16f405d6944c2f963f467da9ed1e6c3c94e08932feae9b9"},
"training-history/r005.md": {"sha256": "a25c2932701695dd7402582510adbe71e50a6fcc2da0fb3c6224a846dfdb22df"},
"training-history/r006.md": {"sha256": "47d53f9aa8a233eb7a319f559209936df988e83d96f6f6517fa3670b5dd2cdd3"}
}
}
|