{
  "expected_vocab_size": 768,
  "cache": {
    "cache_dir": "/media/kurogane/backup/cache/cache_dir",
    "hf_home": "/media/kurogane/backup/cache/hf_home",
    "hub_cache_dir": "/media/kurogane/backup/cache/hub_cache_dir",
    "datasets_cache_dir": "/media/kurogane/backup/cache/datasets_cache_dir",
    "model_cache_dir": "/media/kurogane/backup/cache/model_cache_dir",
    "tokenizer_cache_dir": "/media/kurogane/backup/cache/tokenizer_cache_dir",
    "modules_cache_dir": "/media/kurogane/backup/cache/modules_cache_dir",
    "assets_cache_dir": "/media/kurogane/backup/cache/assets_cache_dir"
  },
  "tokenizer": {
    "path": "tokenizers/tinystories_spm768",
    "use_fast": true,
    "padding_side": "right"
  },
  "dataset": {
    "name": "roneneldan/TinyStories",
    "config_name": null,
    "train_split": "train",
    "eval_split": "validation",
    "text_column": "text",
    "num_proc": 2,
    "shuffle": true,
    "seed": 42,
    "data_files": null,
    "data_dir": null,
    "revision": null,
    "streaming": null
  },
  "model": {
    "config_path": "configs/multiscreen_154m_768_10x_model/config.json",
    "trust_remote_code": false,
    "from_pretrained": false,
    "pretrained_model_name_or_path": null,
    "attn_implementation": null,
    "tie_word_embeddings": true
  },
  "sft_config": {
    "output_dir": "outputs/multiscreen_154m_768_10x_model",
    "max_steps": 10000,
    "per_device_train_batch_size": 128,
    "per_device_eval_batch_size": 8,
    "gradient_accumulation_steps": 8,
    "learning_rate": 0.0003,
    "lr_scheduler_type": "cosine",
    "warmup_steps": 1000,
    "weight_decay": 0.1,
    "max_grad_norm": 1.0,
    "optim": "adamw_torch_fused",
    "bf16": true,
    "fp16": false,
    "tf32": true,
    "gradient_checkpointing": true,
    "use_cache": false,
    "dataset_text_field": "text",
    "dataset_num_proc": 2,
    "max_length": 512,
    "packing": false,
    "eval_packing": false,
    "completion_only_loss": false,
    "eos_token": "</s>",
    "eval_strategy": "steps",
    "eval_steps": 1000,
    "save_strategy": "steps",
    "save_steps": 2000,
    "save_total_limit": 3,
    "logging_steps": 20,
    "logging_first_step": true,
    "report_to": "none",
    "seed": 42,
    "data_seed": 42
  }
}