| model_name: meta-llama/Llama-3.2-1B |
| glp_kwargs: |
| normalizer_config: |
| rep_statistic: ./data/llama1b-dynamic-fineweb-1B/layer_07/rep_statistics.pt |
| denoiser_config: |
| d_input: 2048 |
| d_model: 4096 |
| d_mlp: 8192 |
| n_layers: 3 |
| multi_layer_n_layers: null |
| tracedict_config: |
| layer_prefix: model.layers |
| layers: |
| - 7 |
| retain: output |
| cold_diffusion_kwargs: |
| degradation: vector_addition |
| degradation_kwargs: |
| scale: 5.2087 |
| alpha_mode: constant |
| alpha: 1.0 |
| steering_vectors_path: ./data/steering_vectors/llama3.2-1b-layer07-spar-concepts/steering_vectors.pt |
| steering_combination_min: 1 |
| steering_combination_max: 4 |
| steering_coefficient_min: 1.0 |
| steering_coefficient_max: 1.0 |
| normalize_steering_combinations: true |
| loss_type: l2 |
| shuffle: true |
| train_dataset: ./data/llama1b-dynamic-fineweb-1B/layer_07 |
| rep_statistic: ./data/llama1b-dynamic-fineweb-1B/layer_07/rep_statistics.pt |
| dynamic_dataset: true |
| use_bf16: true |
| num_epochs: 1 |
| epoch_size: 1000000000 |
| batch_size: 4096 |
| learning_rate: 5.0e-05 |
| lr_scheduler: |
| scheduler_cls: cosine_scheduler_with_warmup |
| warmup_ratio: 0.01 |
| initial_factor: 0.01 |
| final_factor: 0.1 |
| gradient_accumulation_steps: 1 |
| gradient_clipping_threshold: 1.0 |
| log_every_n_steps: 10 |
| save_every_n_steps: 7812 |
| save_epochs: |
| - 1 |
| save_opt_state: true |
| output_path: !!python/object/apply:pathlib.PosixPath |
| - runs |
| - cold-vector-combinations-llama1b-d3-dynamic-1B |
| wandb_enabled: true |
| wandb_entity: null |
| wandb_project: glp |
| wandb_run_name: cold-vector-combinations-llama1b-d3-dynamic-1B |
| save_root: . |
| run_name: cold-vector-combinations-llama1b-d3-dynamic-1B |
|
|
