| { | |
| "model_type": "gpt-custom", | |
| "vocab_size": 50257, | |
| "d_model": 768, | |
| "num_heads": 12, | |
| "num_layers": 12, | |
| "ffn_hidden_dim": 3072, | |
| "context_length": 512, | |
| "dropout": 0.1, | |
| "ffn_type": "SwiGLU", | |
| "positional_encoding": "RoPE", | |
| "total_parameters": 151881984, | |
| "training_steps": 6000, | |
| "total_tokens_seen": 197000000, | |
| "dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)", | |
| "tokenizer": "gpt2", | |
| "framework": "PyTorch 2.9.0" | |
| } |