Publish native Llama MatFormer M conversion from cb222bb9

README.md

@@ -0,0 +1,46 @@
+---
+license: cc-by-nc-4.0
+language:
+- en
+datasets:
+- nvidia/Nemotron-ClimbMix
+tags:
+- llama
+- matformer
+- matformer-m
+- vllm
+- causal-lm
+- pretraining
+- climbmix
+---
+
+# atrost/climbmix-matformer-353m-1p2b-h100-m-vllm
+
+Native `LlamaForCausalLM` conversion of `atrost/climbmix-matformer-353m-1p2b-h100-m`.
+
+The source checkpoint was an extracted fixed-size MatFormer `M` submodel.
+Because that submodel has fixed attention and MLP dimensions, its tensors can be
+remapped losslessly into the standard Hugging Face Llama layout. This repo does
+not require remote code and is intended to load through vLLM's native Llama path.
+
+## Source
+
+- Source checkpoint: `atrost/climbmix-matformer-353m-1p2b-h100-m`
+- Source revision: `cb222bb9353c30c372c7478533a5cc6f9feac1d7`
+- Local parity max logits diff before save: `0.000e+00`
+
+## Loading
+
+```python
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+repo_id = "atrost/climbmix-matformer-353m-1p2b-h100-m-vllm"
+tokenizer = AutoTokenizer.from_pretrained(repo_id)
+model = AutoModelForCausalLM.from_pretrained(repo_id, torch_dtype="auto")
+```
+
+```python
+from vllm import LLM
+
+llm = LLM(model="atrost/climbmix-matformer-353m-1p2b-h100-m-vllm")
+```

config.json

@@ -0,0 +1,32 @@
+{
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 50256,
+  "dtype": "float32",
+  "eos_token_id": 50256,
+  "head_dim": 80,
+  "hidden_act": "silu",
+  "hidden_size": 1280,
+  "initializer_range": 0.02,
+  "intermediate_size": 1024,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 4,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 2,
+  "pad_token_id": 50256,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "tie_word_embeddings": false,
+  "transformers_version": "5.7.0",
+  "use_cache": true,
+  "vocab_size": 50304
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 50256,
+  "eos_token_id": 50256,
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 50256,
+  "transformers_version": "5.7.0",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f89bb749685f1e0cce1e96dbcd85acd7d4365eb5850dc7e3224bb546685e16c
+size 762979472

tokenizer_config.json

@@ -0,0 +1,13 @@
+{
+  "add_prefix_space": false,
+  "backend": "tokenizers",
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "errors": "replace",
+  "is_local": false,
+  "local_files_only": false,
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "tokenizer_class": "GPT2Tokenizer",
+  "unk_token": "<|endoftext|>"
+}