Upload model baru

.gitattributes

@@ -0,0 +1 @@
+model.safetensors filter=lfs diff=lfs merge=lfs -text

config.json

@@ -0,0 +1,31 @@
+{
+  "pad_token_id": 0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "tie_word_embeddings": true,
+  "use_cache": false,
+  "vocab_size": 50257,
+  "n_embd": 512,
+  "n_layer": 8,
+  "n_head": 8,
+  "recall_ratio": 0.5,
+  "latent_dim": 128,
+  "attn_drop": 0.0,
+  "use_rope": false,
+  "rope_head_dim": 32,
+  "rope_base": 10000.0,
+  "ffn_mode": "moe",
+  "ff_mult": 2.6666666667,
+  "ffn_dropout": 0.0,
+  "num_experts": 4,
+  "active_experts": 2,
+  "num_shared_experts": 1,
+  "fine_grained_factor": 1,
+  "aux_loss_coef": 0.01,
+  "device_balance_coef": 0.001,
+  "embd_drop": 0.0,
+  "resid_drop": 0.0,
+  "initializer_range": 0.006,
+  "return_dict": true,
+  "model_type": "kilat_transformer"
+}

config.yaml

@@ -0,0 +1,28 @@
+pad_token_id: 0
+bos_token_id: 1
+eos_token_id: 2
+tie_word_embeddings: true
+use_cache: false
+vocab_size: 50257
+n_embd: 512
+n_layer: 8
+n_head: 8
+recall_ratio: 0.5
+latent_dim: 128
+attn_drop: 0.0
+use_rope: false
+rope_head_dim: 32
+rope_base: 10000.0
+ffn_mode: moe
+ff_mult: 2.6666666667
+ffn_dropout: 0.0
+num_experts: 4
+active_experts: 2
+num_shared_experts: 1
+fine_grained_factor: 1
+aux_loss_coef: 0.01
+device_balance_coef: 0.001
+embd_drop: 0.0
+resid_drop: 0.0
+initializer_range: 0.006
+return_dict: true

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8bf671f26e74f7f6d3ebfb87e20d6b927dc1f8dd06a6da7227e45b7b1f5cd52c
+size 563432760

tokenizer_config.json

@@ -0,0 +1,7 @@
+{
+  "tokenizer_type": "auto",
+  "tokenizer_name_or_path": "gpt2",
+  "tokenizer_model_path": null,
+  "use_fast": true,
+  "local_files_only": false
+}