V9 Lightning Upload

README.md

@@ -0,0 +1,11 @@
+---
+library_name: transformers
+tags:
+- chess
+- llm-course
+- chess-challenge
+license: mit
+---
+# Chess Submission V9
+**User:** MDaytek
+**Model:** Transformer from Scratch

config.json

@@ -1,15 +1,19 @@
 {
-  "architectures": [
-    "ChessForCausalLM"
-  ],
-  "dtype": "float32",
-  "model_type": "chess_lm",
-  "n_ctx": 256,
-  "n_embd": 96,
-  "n_head": 8,
-  "n_layer": 8,
-  "n_positions": 256,
-  "tie_word_embeddings": false,
-  "transformers_version": "4.57.6",
-  "vocab_size": 354
-}
+    "architectures": [
+        "ChessForCausalLM"
+    ],
+    "dtype": "float32",
+    "model_type": "chess_lm",
+    "n_ctx": 256,
+    "n_embd": 96,
+    "n_head": 8,
+    "n_layer": 8,
+    "n_positions": 256,
+    "tie_word_embeddings": false,
+    "transformers_version": "4.57.6",
+    "vocab_size": 354,
+    "auto_map": {
+        "AutoConfig": "model.ChessConfig",
+        "AutoModelForCausalLM": "model.ChessForCausalLM"
+    }
+}

model.py

@@ -0,0 +1,72 @@
+import torch
+import torch.nn as nn
+from transformers import PreTrainedModel, PretrainedConfig
+from transformers.modeling_outputs import CausalLMOutputWithPast
+
+class ChessConfig(PretrainedConfig):
+    model_type = "chess_lm"
+
+    def __init__(
+        self,
+        vocab_size=354,
+        n_positions=256,
+        n_embd=96,
+        n_layer=8,
+        n_head=8,
+        tie_word_embeddings=False,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.vocab_size = vocab_size
+        self.n_positions = n_positions
+        self.n_embd = n_embd
+        self.n_layer = n_layer
+        self.n_head = n_head
+        self.tie_word_embeddings = tie_word_embeddings
+
+class ChessForCausalLM(PreTrainedModel):
+    config_class = ChessConfig
+
+    def __init__(self, config):
+        super().__init__(config)
+
+        self.token_emb = nn.Embedding(config.vocab_size, config.n_embd)
+        self.pos_emb = nn.Embedding(config.n_positions, config.n_embd)
+
+        encoder_layer = nn.TransformerEncoderLayer(
+            d_model=config.n_embd,
+            nhead=config.n_head,
+            dim_feedforward=4 * config.n_embd,
+            batch_first=True,
+            norm_first=True
+        )
+        self.transformer = nn.TransformerEncoder(encoder_layer, config.n_layer)
+
+        self.ln_f = nn.LayerNorm(config.n_embd)
+        self.lm_head = nn.Linear(config.n_embd, config.vocab_size, bias=False)
+
+        if config.tie_word_embeddings:
+            self.lm_head.weight = self.token_emb.weight
+
+        self.post_init()
+
+    def forward(self, input_ids, labels=None):
+        B, T = input_ids.shape
+        pos = torch.arange(T, device=input_ids.device)
+
+        x = self.token_emb(input_ids) + self.pos_emb(pos)
+        mask = torch.triu(torch.ones(T, T, device=x.device), diagonal=1).bool()
+
+        x = self.transformer(x, mask=mask)
+        x = self.ln_f(x)
+        logits = self.lm_head(x)
+
+        loss = None
+        if labels is not None:
+            loss_fct = nn.CrossEntropyLoss()
+            loss = loss_fct(
+                logits[:, :-1].reshape(-1, logits.size(-1)),
+                labels[:, 1:].reshape(-1)
+            )
+
+        return CausalLMOutputWithPast(loss=loss, logits=logits)

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e22c6b4c46c726d51af026bf6fad1ba7cc765bac558a03864f68a48e6c727cd
+oid sha256:d91cc00c285c8b97c03a50223cab9ab54ff39937492306dcc88ed8b6c75f793b
 size 3960136

tokenizer.py

@@ -0,0 +1,33 @@
+import json, os, torch
+
+class ChessTokenizer:
+    def __init__(self, vocab):
+        self.vocab = vocab
+        self.id_to_token = {v: k for k, v in vocab.items()}
+        self.pad_token_id = vocab["[PAD]"]
+        self.bos_token_id = vocab["[BOS]"]
+        self.eos_token_id = vocab["[EOS]"]
+        self.unk_token_id = vocab["[UNK]"]
+
+    @property
+    def vocab_size(self):
+        return len(self.vocab)
+
+    def _convert_token_to_id(self, token):
+        return self.vocab.get(token, self.unk_token_id)
+
+    def pad(self, encoded_inputs, padding=True, max_length=None, pad_to_multiple_of=None, return_tensors=None, **kwargs):
+
+        max_len = max(len(x["input_ids"]) for x in encoded_inputs)
+        padded = [
+            x["input_ids"] + [self.pad_token_id] * (max_len - len(x["input_ids"]))
+            for x in encoded_inputs
+        ]
+        if return_tensors == "pt":
+            return {"input_ids": torch.tensor(padded, dtype=torch.long)}
+        return {"input_ids": padded}
+
+    def save_pretrained(self, path):
+        os.makedirs(path, exist_ok=True)
+        with open(os.path.join(path, "vocab.json"), "w") as f:
+            json.dump(self.vocab, f, indent=2)