Training in progress, step 500

README.md

@@ -0,0 +1,58 @@
+---
+base_model: SurjoLabs/Blaze-SFT
+library_name: transformers
+model_name: Blaze-Title
+tags:
+- generated_from_trainer
+- trl
+- sft
+licence: license
+---
+
+# Model Card for Blaze-Title
+
+This model is a fine-tuned version of [SurjoLabs/Blaze-SFT](https://huggingface.co/SurjoLabs/Blaze-SFT).
+It has been trained using [TRL](https://github.com/huggingface/trl).
+
+## Quick start
+
+```python
+from transformers import pipeline
+
+question = "If you had a time machine, but could only go to the past or the future once and never return, which would you choose and why?"
+generator = pipeline("text-generation", model="SurjoLabs/Blaze-Title", device="cuda")
+output = generator([{"role": "user", "content": question}], max_new_tokens=128, return_full_text=False)[0]
+print(output["generated_text"])
+```
+
+## Training procedure
+
+ 
+
+
+
+This model was trained with SFT.
+
+### Framework versions
+
+- TRL: 1.12.0
+- Transformers: 5.16.1
+- Pytorch: 2.14.0
+- Datasets: 5.0.1
+- Tokenizers: 0.23.2
+
+## Citations
+
+
+
+Cite TRL as:
+    
+```bibtex
+@software{vonwerra2020trl,
+  title   = {{TRL: Transformers Reinforcement Learning}},
+  author  = {von Werra, Leandro and Belkada, Younes and Tunstall, Lewis and Beeching, Edward and Thrush, Tristan and Lambert, Nathan and Huang, Shengyi and Rasul, Kashif and Gallouédec, Quentin},
+  license = {Apache-2.0},
+  url     = {https://github.com/huggingface/trl},
+  year    = {2020}
+}
+```

chat_template.jinja

@@ -0,0 +1 @@
+{{ bos_token }}{% for message in messages %}{% if message['role'] == 'assistant' %}{{ '<|im_start|>assistant\n' }}{% generation %}{{ message['content'] + '<|im_end|>\n' }}{% endgeneration %}{% else %}{{ '<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>\n' }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}

config.json

@@ -0,0 +1,44 @@
+{
+  "architectures": [
+    "BlazeForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "configuration_blaze.BlazeConfig",
+    "AutoModelForCausalLM": "modeling_blaze.BlazeForCausalLM"
+  },
+  "bos_token_id": 2,
+  "coda_layers": 1,
+  "dtype": "bfloat16",
+  "eos_token_id": 6,
+  "gradient_checkpointing": false,
+  "head_dim": 64,
+  "hidden_act": "silu",
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "max_position_embeddings": 2048,
+  "mlp_bias": false,
+  "model_type": "blaze",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 14,
+  "num_key_value_heads": 4,
+  "pad_token_id": 1,
+  "prelude_layers": 1,
+  "pretraining_tp": 1,
+  "recurrent_layers": 12,
+  "recurrent_passes": 2,
+  "rms_norm_eps": 1e-05,
+  "rope_parameters": {
+    "rope_theta": 10000.0,
+    "rope_type": "default"
+  },
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.16.1",
+  "use_cache": false,
+  "use_flash_attn": false,
+  "vocab_size": 8192,
+  "xsa_projection": true
+}

generation_config.json

@@ -0,0 +1,14 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 2,
+  "eos_token_id": [
+    3,
+    2,
+    6
+  ],
+  "output_attentions": false,
+  "output_hidden_states": false,
+  "pad_token_id": 1,
+  "transformers_version": "5.16.1",
+  "use_cache": true
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b55a7a80924d89cb1a5fa681af2eec6f748207fed02202bce3fb0f9787aef94
+size 96519360

tokenizer_config.json

@@ -0,0 +1,85 @@
+{
+  "backend": "tokenizers",
+  "bos_token": "<|bos|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|im_end|>",
+  "extra_special_tokens": [
+    "<|unk|>",
+    "<|pad|>",
+    "<|bos|>",
+    "<|eos|>",
+    "<|mask|>",
+    "<|im_start|>",
+    "<|im_end|>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<think>",
+    "</think>",
+    "<|begin_of_thought|>",
+    "<|end_of_thought|>",
+    "<answer>",
+    "</answer>",
+    "<|step|>",
+    "<|/step|>",
+    "<context>",
+    "</context>",
+    "<|doc_start|>",
+    "<|doc_end|>",
+    "<|search|>",
+    "<|search_results|>",
+    "<|tool_list_start|>",
+    "<|tool_list_end|>",
+    "<tools>",
+    "</tools>",
+    "<|tool_call_start|>",
+    "<|tool_call_end|>",
+    "<|tool_call|>",
+    "<|/tool_call|>",
+    "<|tool_response_start|>",
+    "<|tool_response_end|>",
+    "<|tool_response|>",
+    "<|/tool_response|>",
+    "<|image|>",
+    "<|image_pad|>",
+    "<|image_placeholder|>",
+    "<|audio|>",
+    "<|audio_pad|>",
+    "<|audio_placeholder|>",
+    "<|video|>",
+    "<|video_pad|>",
+    "<|fim_prefix|>",
+    "<|fim_suffix|>",
+    "<|fim_middle|>",
+    "<|repo_name|>",
+    "<|file_separator|>",
+    "<|reward|>",
+    "<|reserved_0|>",
+    "<|reserved_1|>",
+    "<|reserved_2|>",
+    "<|reserved_3|>",
+    "<|reserved_4|>",
+    "<|reserved_5|>",
+    "<|reserved_6|>",
+    "<|reserved_7|>",
+    "<|reserved_8|>",
+    "<|reserved_9|>",
+    "<|reserved_10|>",
+    "<|reserved_11|>",
+    "<|reserved_12|>",
+    "<|reserved_13|>",
+    "<|reserved_14|>",
+    "<|reserved_15|>",
+    "<|reserved_16|>",
+    "<|reserved_17|>",
+    "<|reserved_18|>",
+    "<|reserved_19|>"
+  ],
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "<|mask|>",
+  "model_max_length": 10000000,
+  "pad_token": "<|pad|>",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "<|unk|>"
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e03aae144474a03f8ecb82b049a21b4dd10f88c0bdcf9a88d1d04379f9c18f32
+size 5841