Upload config.yaml to L56-D1920-qwen_mamba2_qwen2-e1-i1920-s256-hd64-gn6-A0-S4096-step1-rand2b

config.yaml

@@ -0,0 +1,125 @@
+!!python/object/new:configs.Config
+dictitems:
+  model: !!python/object/new:configs.Config
+    dictitems:
+      __type__: configs.Transformer_Config
+      alibi: null
+      attention_type: mamba2
+      attn_classes_path: transformers.models.qwen2.modeling_qwen2.QWEN2_ATTENTION_CLASSES
+      attn_path: rwkv6attn.RWKV6Attention
+      balance_state: 0
+      brope: null
+      classname: qwen2
+      cmix: x060
+      cmix2: x060
+      ctx_len: 4096
+      dim_att: 0
+      dim_ffn: 4800
+      dropout: 0.0
+      eos_token_id: 2
+      gate_rank_type: 0
+      gdn: !!python/object/new:configs.Config
+        dictitems:
+          __type__: configs.GDN_Config
+          expand_v: 2
+          head_dim: 64
+          num_heads: 6
+          num_v_heads: 6
+      groupnorm_att: 0
+      head_size: 64
+      head_size_divisor: 8
+      hf_cfg: null
+      hf_path: ''
+      inv_other_layer_ratio: 1
+      kv_cache_compression_ratio: 16
+      lora_rank_decay: 0
+      lora_rank_gate: 0
+      lora_rank_iclr: 0
+      lora_rank_tokenshift: 0
+      lora_rank_value_residual_mix: 0
+      mamba: !!python/object/new:configs.Config
+        dictitems:
+          __type__: configs.Mamba_Config
+          attn_layer_indices: []
+          d_conv: 4
+          d_inner: 1920
+          d_state: 256
+          expand: 1
+          headdim: 64
+          ngroups: 6
+      n_embd: 1920
+      n_layer: 56
+      num_key_value_heads: 6
+      parallel: 0
+      preserve_last_n_layers: 0
+      reinit_att: 0
+      rms_norm_eps: 1.0e-06
+      rope: !!python/object/new:configs.Config
+        dictitems:
+          __type__: configs.RoPE_Config
+          base: 490000.0
+          rebase: 1.0
+          rescale: 1.0
+      tie_word_embeddings: false
+      tmix: qwen_mamba2
+      tmix2: qwen2
+      use_pos_emb: 1
+      use_tokenshift: 1
+      vocab_padding_idx: 102
+      vocab_size: 99000
+  runtime: !!python/object:configs.Runtime_Config
+    epoch_count: 999999999
+    epoch_global_steps: 999999999
+    global_step_bsz: 32
+    my_pile_prev_p: 0
+    my_timestamp: 2025-09-24-15-53-42
+    proj_path: out/L56-D1920-qwen_mamba2_qwen2-e1-i1920-s256-hd64-gn6-A0-S4096--step1-rand2b
+    run_name: 'qwen_mamba2_qwen2 L56 D1920 ctx4096 '
+  train: !!python/object/new:configs.Config
+    dictitems:
+      accelerator: gpu
+      accumulate_grad_batches: 1
+      adam_eps: 1.0e-08
+      attention_distillation_stage: 1
+      beta1: 0.9
+      beta2: 0.95
+      check_val_every_n_epoch: 1
+      data_file: ../cache/datasets/huggingface/Teaven/Selection/yulan-random2b/YuLan-Mini/data
+      data_type: binidx
+      devices: 8
+      ds_bucket_mb: 200
+      epoch_begin: 0
+      epoch_save: 5
+      grad_cp: 0
+      gradient_clip_val: 1.0
+      layerwise_lr: 1
+      load_model: out/YuLanMini-2.4B.pth
+      load_partial: 1
+      log_every_n_steps: 50
+      lr2_final: -1
+      lr2_init: -1
+      lr_decay_type: cos
+      lr_endpoint: 1.0
+      lr_final: 1.0e-05
+      lr_init: 0.001
+      lr_wait: 0.0
+      magic_prime: 451961
+      micro_bsz: 4
+      my_exit_tokens: 1851236987
+      num_nodes: 1
+      optimizer: dsfusedadamw
+      precision: bf16
+      proj_dir: out
+      proj_name: L56-D1920-qwen_mamba2_qwen2-e1-i1920-s256-hd64-gn6-A0-S4096
+      proj_suffix: step1-rand2b
+      proj_suffix0: ''
+      seed_everything: 1337
+      strategy: deepspeed_stage_1
+      teacher: null
+      train_stage: 3
+      val_check_interval: null
+      validation_data_file: ''
+      wandb: block-distill
+      warmup_steps: 50
+      weight_decay: 0
+      weight_decay_final: -1.0