| { |
| "schema_version": 3, |
| "revision": 6, |
| "revision_id": "r006", |
| "release": "Pollock 1.4", |
| "model_id": "SlayerLab/pollock-mini-lm-125m", |
| "publication": { |
| "weights_commit": "a2e53d9f1b690ebfbea375cf8bd3b5bdf69dece9", |
| "version_tag": "v1.4", |
| "publication_commit_pending": true |
| }, |
| "source_checkpoint": { |
| "path_in_training_workspace": "runs/r006-corrected-corpus-v2-s1337-lr0.0004/checkpoints/ckpt-final.pt", |
| "sha256": "580fbb3c039ea1f349285a9bf7a67731fb6d455de931b6b7601d4ed09fce9437", |
| "iteration": 22003, |
| "tokens_seen": 10814914560, |
| "native_nanogpt_parameters": 126637952, |
| "native_unique_trainable_parameters": 127555456 |
| }, |
| "architecture": { |
| "n_layer": 12, |
| "n_head": 14, |
| "n_embd": 896, |
| "block_size": 1024, |
| "vocab_size": 12288, |
| "dropout": 0.0, |
| "bias": false, |
| "tied_word_embeddings": true |
| }, |
| "training": { |
| "dataset": "SlayerLab/minimal-en-corpus-2.5b", |
| "dataset_revision": "d68d992622e9fc11f19e7d7fb8547c4e653439a4", |
| "training_tokens": 2689323439, |
| "validation_tokens": 5236486, |
| "tokenizer_sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51", |
| "native_tokenizer_sha256": "d0d126f0a59e51e8cb1b26d77b5527870be0e8ea1e84adefd730fff906c48234", |
| "init_from": "scratch", |
| "micro_batch_per_gpu": 12, |
| "gradient_accumulation_global": 40, |
| "gradient_accumulation_per_gpu": 20, |
| "ddp_world_size": 2, |
| "effective_batch_tokens": 491520, |
| "optimizer": "fused AdamW", |
| "learning_rate": 0.0004, |
| "min_learning_rate": 0.00004, |
| "schedule": "cosine", |
| "warmup_iters": 440, |
| "beta1": 0.9, |
| "beta2": 0.95, |
| "weight_decay": 0.1, |
| "grad_clip": 1.0, |
| "precision": "bfloat16", |
| "hardware": "2x NVIDIA GeForce RTX 4090", |
| "framework": "PyTorch 2.8.0+cu128", |
| "nanogpt_commit": "3adf61e154c3fe3fca428ad6bc3818b27a3b8291", |
| "data_pass_equivalent": 4.021425762020438, |
| "runtime_hours": 15.226956854563, |
| "mean_tokens_per_second": 199894.6379482684 |
| }, |
| "evaluation": { |
| "protocol": "fixed sampled subset", |
| "subset_tokens_per_split": 1228800, |
| "final_train_loss": 2.48974818944931, |
| "final_validation_loss": 2.536356544494629, |
| "best_validation_loss": 2.5362311387062073, |
| "best_validation_update": 22000, |
| "best_validation_tokens_seen": 10813440000, |
| "final_validation_perplexity": 12.633557211556399, |
| "benchmark_harness": "lm-evaluation-harness 0.4.12", |
| "benchmark_num_fewshot": 0, |
| "benchmark_batch_size": 8, |
| "truncated_benchmark_requests": 0, |
| "benchmark_checkpoint_sha256": "9339d20ea32d24bc9777518a2f0ec553e4a3871647d342db92d7c5b6c18c7b56", |
| "benchmark_tokenizer_sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51", |
| "results_file": "benchmarks/english.json" |
| }, |
| "fixed_inference": { |
| "suite_id": "fixed-sampling-v1", |
| "comparison_file": "inference-samples/README.md", |
| "config_file": "inference-samples/config.json", |
| "generator_file": "inference-samples/generate.py", |
| "results_file": "inference-samples/results.json", |
| "operating_system": "macOS 26.2", |
| "architecture": "arm64", |
| "device": "cpu", |
| "dtype": "float32", |
| "seed": 1337, |
| "prompt_count": 4, |
| "revision_count": 6, |
| "local_review_artifact": false, |
| "exact_replay_verified": true, |
| "publication_commit_pending": true |
| }, |
| "conversion": { |
| "target_class": "GPT2LMHeadModel", |
| "transformers_version": "5.15.1", |
| "checkpoint_sha256": "580fbb3c039ea1f349285a9bf7a67731fb6d455de931b6b7601d4ed09fce9437", |
| "model_sha256": "8e74aa4d34464229e86a1831fb34d442f91e22aa600e0cf2503891957fdc6a43", |
| "unique_serialized_parameters": 127674624, |
| "compatibility_zero_bias_parameters": 119168, |
| "validation_probe_shape": [2, 64], |
| "max_absolute_logit_error": 0.0 |
| }, |
| "artifacts": { |
| ".gitattributes": {"sha256": "c759491a998899dbefaec4d51cc791e68c714a37dd9f9829020a368788fe3063"}, |
| "CHANGELOG.md": {"sha256": "8422ad337b22be7adb561d1b2388cf53e5a6d1c6d49d628468498c72f078769a"}, |
| "LICENSE.md": {"sha256": "46cbe928ed0aa24875f02f774313b27ec9d8abdf41f0c9ef67e0adb4e0614de4"}, |
| "README.md": {"sha256": "875fa9cd748691e376df905e036a0c471db77aaa1272d71c5ea10d0f76fa888d"}, |
| "assets/pollock-mini-lm-avatar-320.png": {"sha256": "7be10cc9d0f4aedeb298d9a5d722b2b2ac5b2e219b3884916dbca16198f70750"}, |
| "benchmarks/english.json": {"sha256": "646a3aff7c97859e6974baa2c191702829a982af87675870dd1a339f8073a146"}, |
| "config.json": {"sha256": "1529f8a8fc31b4fa68fc18fffbad7e155dc8f9e8663a40b4f3306b4012ea38c0"}, |
| "generation_config.json": {"sha256": "435beb27be51f0ed054f4a011e5109d125cdadc118b8799b18b155cc798d94d2"}, |
| "inference-samples/README.md": {"sha256": "6559c85e1ed70f47f2d2978f28151fdd9e3355e397c5591cbe0b8ab767fa5ac5"}, |
| "inference-samples/config.json": {"sha256": "a1b306a1e1f3a70942706d00107a83dd98289a534d17af78adbb95d48c9b69c1"}, |
| "inference-samples/generate.py": {"sha256": "807a9946472bd0087dd2b9260ee90d1dea09785d24d4f923f028667c53a23933"}, |
| "inference-samples/requirements.txt": {"sha256": "584583335ffb3061aa62058aee725b0de25fed7523904fbd13fb4622f601943d"}, |
| "inference-samples/results.json": {"sha256": "a279806ea225130b0e0aa36d9bd13653401e236395fe64b5f0aa7c825d445620"}, |
| "model.safetensors": {"sha256": "8e74aa4d34464229e86a1831fb34d442f91e22aa600e0cf2503891957fdc6a43"}, |
| "special_tokens_map.json": {"sha256": "8b2257a17ea997bb038f43b133aefec82344ad2b8abc2b8a02a6c0a994ed624e"}, |
| "tokenizer.json": {"sha256": "6cda4e5ec8293f3b821e02253f9c0e88e43ad4f7c6e1324763e1fb91749fef51"}, |
| "tokenizer_config.json": {"sha256": "4cdabe37dbdc1adfcc017ee9a1f86ab89bdf184827d2d9f05181cae0f8af19bf"}, |
| "training-history/r001.md": {"sha256": "5b0ac39265921ed3604e7dfe8181b00d07902640d8024f591462eeb508bcfece"}, |
| "training-history/r002.md": {"sha256": "80031a84b2b67dc4cb4f81a178fa354feea15cb184d17368ee7e64ec3d0a2629"}, |
| "training-history/r003.md": {"sha256": "7ea9e6b15a0af635ff238417b9fff2a98c12a9090800a0923d622253b7eab02d"}, |
| "training-history/r004.md": {"sha256": "e260bfba69f154caf16f405d6944c2f963f467da9ed1e6c3c94e08932feae9b9"}, |
| "training-history/r005.md": {"sha256": "a25c2932701695dd7402582510adbe71e50a6fcc2da0fb3c6224a846dfdb22df"}, |
| "training-history/r006.md": {"sha256": "47d53f9aa8a233eb7a319f559209936df988e83d96f6f6517fa3670b5dd2cdd3"} |
| } |
| } |
|
|
