Upload folder using huggingface_hub

config.json

@@ -0,0 +1,18 @@
+{
+  "architectures": [
+    "LightningForCausalLM"
+  ],
+  "d_model": 256,
+  "dropout": 0.1,
+  "dtype": "float32",
+  "max_seq_len": 160,
+  "model_type": "lightning",
+  "nhead": 4,
+  "num_layers": 4,
+  "transformers_version": "4.57.3",
+  "vocab_size": 50000,
+  "auto_map": {
+    "AutoConfig": "configuration_lightning.LightningConfig",
+    "AutoModelForCausalLM": "modeling_lightning.LightningForCausalLM"
+  }
+}

configuration_lightning.py

@@ -0,0 +1,27 @@
+from transformers import PretrainedConfig
+
+
+class LightningConfig(PretrainedConfig):
+    model_type = "lightning"
+
+    def __init__(
+        self,
+        vocab_size=50000,
+        d_model=256,
+        nhead=4,
+        num_layers=4,
+        dropout=0.1,
+        max_seq_len=160,
+        **kwargs
+    ):
+        super().__init__(
+            tie_word_embeddings=False,
+            **kwargs
+        )
+
+        self.vocab_size = vocab_size
+        self.d_model = d_model
+        self.nhead = nhead
+        self.num_layers = num_layers
+        self.dropout = dropout
+        self.max_seq_len = max_seq_len

modeling_lightning.py

@@ -0,0 +1,61 @@
+import torch
+import torch.nn as nn
+
+from transformers import PreTrainedModel
+from transformers.modeling_outputs import CausalLMOutput
+
+from configuration_lightning import LightningConfig
+from model import TransformerLanguageModel
+
+
+class LightningForCausalLM(PreTrainedModel):
+
+    config_class = LightningConfig
+    base_model_prefix = "lightning"
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.lightning = TransformerLanguageModel(
+            vocab_size=config.vocab_size,
+            d_model=config.d_model,
+            nhead=config.nhead,
+            num_layers=config.num_layers,
+            dropout=config.dropout,
+            max_seq_len=config.max_seq_len
+        )
+
+        self.post_init()
+
+    def forward(self, input_ids=None, labels=None, **kwargs):
+        logits = self.lightning(input_ids)
+
+        loss = None
+
+        if labels is not None:
+            shift_logits = logits[..., :-1, :].contiguous()
+            shift_labels = labels[..., 1:].contiguous()
+
+            loss_fn = nn.CrossEntropyLoss()
+
+            loss = loss_fn(
+                shift_logits.view(-1, shift_logits.size(-1)),
+                shift_labels.view(-1)
+            )
+
+        return CausalLMOutput(
+            loss=loss,
+            logits=logits
+        )
+
+    def get_input_embeddings(self):
+        return self.lightning.token_embedding
+
+    def set_input_embeddings(self, value):
+        self.lightning.token_embedding = value
+
+    def get_output_embeddings(self):
+        return self.lightning.output_layer
+
+    def set_output_embeddings(self, new_embeddings):
+        self.lightning.output_layer = new_embeddings

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f13c912ae94bf1109ab3662c42fd4358a020dfa5e03bfc9f7d9571e9e925582
+size 115219558