config.json

{
  "hf_repo_id": "aixk/BAAR3-embed-49M",
  "hf_dataset_repo_id": "aixk/b-embed-dataset",
  "hf_vocab_dataset": "aixk/L20",
  "local_checkpoint_dir": "./checkpoints_embed_49m",
  "local_dataset_cache_dir": "./cached_embed_dataset_49m",
  "local_tokenizer_cache_dir": "./cached_embed_tokenizer_49m",
  "export_dir": "./baar2_embed_export_49m",
  "baar2_filename_int8": "model_embed_pure_int8.baar2",
  "baar2_filename_int4": "model_embed_pure_int4.baar2",
  "safetensors_filename": "model.safetensors",
  "max_steps": 50000,
  "batch_size": 64,
  "accum_steps": 1,
  "log_interval": 10,
  "val_interval": 500,
  "keep_last_n_ckpts": 2,
  "learning_rate": 0.0003,
  "min_lr": 3e-05,
  "warmup_steps": 500,
  "decay_start_ratio": 0.5,
  "weight_decay": 0.01,
  "max_seq_len": 128,
  "concept_size": 1024,
  "grapheme_pool_size": 1024,
  "vocab_size": 2048,
  "hidden_dim": 384,
  "expansion_dim": 2048,
  "attn_dim": 64,
  "num_layers": 20,
  "temperature": 0.05,
  "logit_soft_capping": 0.0
}