config.json

{
  "model_type": "gpt-custom",
  "vocab_size": 50257,
  "d_model": 768,
  "num_heads": 12,
  "num_layers": 12,
  "ffn_hidden_dim": 3072,
  "context_length": 512,
  "dropout": 0.1,
  "ffn_type": "SwiGLU",
  "positional_encoding": "RoPE",
  "total_parameters": 151881984,
  "training_steps": 6000,
  "total_tokens_seen": 197000000,
  "dataset": "HuggingFaceFW/fineweb-edu (sample-10BT)",
  "tokenizer": "gpt2",
  "framework": "PyTorch 2.9.0"
}