| from transformers import PretrainedConfig | |
| class LightningConfig(PretrainedConfig): | |
| model_type = "lightning" | |
| def __init__( | |
| self, | |
| vocab_size=75000, | |
| d_model=256, | |
| nhead=4, | |
| num_layers=6, | |
| dropout=0.1, | |
| max_seq_len=200, | |
| **kwargs | |
| ): | |
| kwargs.setdefault("tie_word_embeddings", False) | |
| super().__init__(**kwargs) | |
| self.vocab_size = vocab_size | |
| self.d_model = d_model | |
| self.nhead = nhead | |
| self.num_layers = num_layers | |
| self.dropout = dropout | |
| self.max_seq_len = max_seq_len | |
| self.num_hidden_layers = num_layers | |
| self.num_attention_heads = nhead | |
| self.hidden_size = d_model |