pretraining_config/optimizer_config.json

{"model_lr": 0.001, "model_weight_decay": 5e-05, "projection_head_lr": 0.001, "projection_head_weight_decay": 0.0001, "max_weight_decay": -1, "loss_fn": "dcl", "clipnorm": 5, "warmup_steps": 2000.0}