Hugging Face's logo

N8Programs
/
lil-bard-checkpts

Transformers
Safetensors
English
spark-gpt
qwen3-moe
from-scratch
stories
checkpoints
Model card Files
xet
 Community
N8Programs commited on
Commit
5f1f69f
·
verified ·
1 Parent(s): 7fcca91

Add files using upload-large-folder tool

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. step-0001000/config.json +36 -0
  2. step-0001000/generation_config.json +5 -0
  3. step-0001000/model.safetensors +3 -0
  4. step-0001000/tokenizer.json +0 -0
  5. step-0001000/tokenizer_config.json +9 -0
  6. step-0002000/config.json +36 -0
  7. step-0002000/generation_config.json +5 -0
  8. step-0002000/model.safetensors +3 -0
  9. step-0002000/tokenizer.json +0 -0
  10. step-0003000/model.safetensors +3 -0
  11. step-0004000/config.json +36 -0
  12. step-0004000/generation_config.json +5 -0
  13. step-0004000/model.safetensors +3 -0
  14. step-0004000/tokenizer.json +0 -0
  15. step-0004000/tokenizer_config.json +9 -0
  16. step-0005000/model.safetensors +3 -0
  17. step-0006000/config.json +36 -0
  18. step-0006000/generation_config.json +5 -0
  19. step-0006000/tokenizer.json +0 -0
  20. step-0006000/tokenizer_config.json +9 -0
  21. step-0007000/model.safetensors +3 -0
  22. step-0008000/model.safetensors +3 -0
  23. step-0009000/config.json +36 -0
  24. step-0009000/generation_config.json +5 -0
  25. step-0009000/model.safetensors +3 -0
  26. step-0009000/tokenizer.json +0 -0
  27. step-0009000/tokenizer_config.json +9 -0
  28. step-0010000/model.safetensors +3 -0
  29. step-0011000/model.safetensors +3 -0
  30. step-0012000/config.json +36 -0
  31. step-0012000/generation_config.json +5 -0
  32. step-0012000/model.safetensors +3 -0
  33. step-0012000/tokenizer.json +0 -0
  34. step-0012000/tokenizer_config.json +9 -0
  35. step-0013000/model.safetensors +3 -0
  36. step-0014000/config.json +36 -0
  37. step-0014000/generation_config.json +5 -0
  38. step-0014000/model.safetensors +3 -0
  39. step-0014000/tokenizer.json +0 -0
  40. step-0014000/tokenizer_config.json +9 -0
  41. step-0015000/model.safetensors +3 -0
  42. step-0016000/config.json +36 -0
  43. step-0016000/generation_config.json +5 -0
  44. step-0016000/model.safetensors +3 -0
  45. step-0016000/tokenizer.json +0 -0
  46. step-0016000/tokenizer_config.json +9 -0
  47. step-0017000/model.safetensors +3 -0
  48. step-0018000/model.safetensors +3 -0
  49. step-0019000/config.json +36 -0
  50. step-0019000/generation_config.json +5 -0
step-0001000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0001000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0001000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9a7f2f4475df90826cd6765f1e1237b685970d9296344e46224ca2050ac98f26
3
+ size 688231784
step-0001000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0001000/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|beginoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "model_max_length": 32768,
7
+ "clean_up_tokenization_spaces": false,
8
+ "unk_token": "<|unk|>"
9
+ }
step-0002000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0002000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0002000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0c6ad479b924e43538a385627c4a9768ac6ed3f8263b891e449f0ceb418a34f7
3
+ size 688231784
step-0002000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0003000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b810e08b773b0180e70893a722c5450a32c86e8ca786298623a1856241da2bfc
3
+ size 688231784
step-0004000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0004000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0004000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6c7bfb6103f5e543ce00308f907bfaefaf8baec5beac52a9e765601504be51de
3
+ size 688231784
step-0004000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0004000/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|beginoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "model_max_length": 32768,
7
+ "clean_up_tokenization_spaces": false,
8
+ "unk_token": "<|unk|>"
9
+ }
step-0005000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:09f37044160554ef32ae2a9d1df16ce391589c49a02163e7530c3c811d9c9101
3
+ size 688231784
step-0006000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0006000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0006000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0006000/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|beginoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "model_max_length": 32768,
7
+ "clean_up_tokenization_spaces": false,
8
+ "unk_token": "<|unk|>"
9
+ }
step-0007000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:62b9ec55e18b3e326111c0c38e31b44a75ca45277d5cb44ae1cbbc23148c6336
3
+ size 688231784
step-0008000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c01f137151d124097b0464d85bf1398511f899c7d9a3f8960c847a54d721a651
3
+ size 688231784
step-0009000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0009000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0009000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5c9511e52f367dc4b2fe22cd06f71ac25686adafad5a50ffbbb8d9b1e5e60494
3
+ size 688231784
step-0009000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0009000/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|beginoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "model_max_length": 32768,
7
+ "clean_up_tokenization_spaces": false,
8
+ "unk_token": "<|unk|>"
9
+ }
step-0010000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2c8482efb20482603b566a73a2d615a1bd3bc1119f537c268f72cdb7d13eda4e
3
+ size 688231784
step-0011000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:377b4546324028e92a0966be88a8767de6513852a3904207775d3e4e3eb558ed
3
+ size 688231784
step-0012000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0012000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0012000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:53b5f73649faa4f823fbcd9e72f60b8580b8f288a4da3b9d4fa16d548eccfed5
3
+ size 688231784
step-0012000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0012000/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|beginoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "model_max_length": 32768,
7
+ "clean_up_tokenization_spaces": false,
8
+ "unk_token": "<|unk|>"
9
+ }
step-0013000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c0f68d34c706671efad0d0b60234cfdb881b1c55564cb61776618c11f3873610
3
+ size 688231784
step-0014000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0014000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0014000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c981c57dc299d615dc4273696a4664625116e86d094631fdaacae73c28345f55
3
+ size 688231784
step-0014000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0014000/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|beginoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "model_max_length": 32768,
7
+ "clean_up_tokenization_spaces": false,
8
+ "unk_token": "<|unk|>"
9
+ }
step-0015000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:44c99e0bee2f829e06e97c8f66fb79cefc72f5524170a6b9945e29b707550666
3
+ size 688231784
step-0016000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0016000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }
step-0016000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f01749110b226e12287d4ba7a486766848c8650d79ae1a7478868e8f19a46a99
3
+ size 688231784
step-0016000/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
step-0016000/tokenizer_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "tokenizer_class": "PreTrainedTokenizerFast",
3
+ "bos_token": "<|beginoftext|>",
4
+ "eos_token": "<|endoftext|>",
5
+ "pad_token": "<|pad|>",
6
+ "model_max_length": 32768,
7
+ "clean_up_tokenization_spaces": false,
8
+ "unk_token": "<|unk|>"
9
+ }
step-0017000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a2a6705f4ba722bef88e3fa5325bb867dc474ed1388f67c7e2cc82852f2eb610
3
+ size 688231784
step-0018000/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9ba2a721464ce4cbfe989c9cfc2504c9506a3750f120810855023153f3286bfd
3
+ size 688231784
step-0019000/config.json ADDED
@@ -0,0 +1,36 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen3MoeForCausalLM"
4
+ ],
5
+ "model_type": "qwen3_moe",
6
+ "vocab_size": 8192,
7
+ "hidden_size": 512,
8
+ "num_hidden_layers": 16,
9
+ "intermediate_size": 1536,
10
+ "num_attention_heads": 4,
11
+ "num_key_value_heads": 2,
12
+ "head_dim": 128,
13
+ "hidden_act": "silu",
14
+ "rms_norm_eps": 1e-06,
15
+ "max_position_embeddings": 32768,
16
+ "rope_theta": 1000000.0,
17
+ "rope_scaling": null,
18
+ "attention_bias": false,
19
+ "attention_dropout": 0.0,
20
+ "tie_word_embeddings": false,
21
+ "use_cache": true,
22
+ "bos_token_id": 8190,
23
+ "eos_token_id": 0,
24
+ "pad_token_id": 8191,
25
+ "torch_dtype": "float32",
26
+ "decoder_sparse_step": 1,
27
+ "moe_intermediate_size": 768,
28
+ "num_experts": 8,
29
+ "num_experts_per_tok": 2,
30
+ "norm_topk_prob": true,
31
+ "output_router_logits": false,
32
+ "router_aux_loss_coef": 0.001,
33
+ "mlp_only_layers": [],
34
+ "use_sliding_window": false,
35
+ "sliding_window": null
36
+ }
step-0019000/generation_config.json ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ {
2
+ "bos_token_id": 8190,
3
+ "eos_token_id": 0,
4
+ "pad_token_id": 8191
5
+ }