| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 3.8070456365092076, |
| "eval_steps": 500, |
| "global_step": 300, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.012810248198558846, |
| "gen/advantages_abs_mean": 0.052109457552433014, |
| "gen/advantages_max": 0.13163888454437256, |
| "gen/advantages_mean": 3.026798367500305e-09, |
| "gen/advantages_min": -0.2830061912536621, |
| "gen/advantages_std": 0.06655973941087723, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 88.0, |
| "gen/prompt/min_length": 33.0, |
| "gen/reward": 0.7407832741737366, |
| "gen/reward_std": 0.07185085862874985, |
| "grad_norm": 0.004070668946951628, |
| "learning_rate": 0.0, |
| "loss": -0.0, |
| "rewards/perceptual_score_reward_func/mean": 0.7447192668914795, |
| "rewards/perceptual_score_reward_func/std": 0.06898166239261627, |
| "step": 1, |
| "time_profile/curr_logprobs_and_update": 0.28362782069598325, |
| "time_profile/image_rollout": 95.28615356422961, |
| "time_profile/old_logps": 35.66134166903794, |
| "time_profile/ref_logps": 35.66421937197447, |
| "time_profile/reward": 4.256949637085199, |
| "train/gen_step": 64.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.0, |
| "unified/loss": -1.0550138540565968e-08 |
| }, |
| { |
| "epoch": 0.025620496397117692, |
| "grad_norm": 0.0038007174152880907, |
| "learning_rate": 1e-05, |
| "loss": -0.0, |
| "step": 2, |
| "time_profile/curr_logprobs_and_update": 0.2893671114288736, |
| "train/gen_step": 128.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.0, |
| "unified/loss": -1.1525116860866547e-08 |
| }, |
| { |
| "epoch": 0.03843074459567654, |
| "gen/advantages_abs_mean": 0.06179381534457207, |
| "gen/advantages_max": 0.2224750518798828, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.27374815940856934, |
| "gen/advantages_std": 0.07891622185707092, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 182.5, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.4988939166069031, |
| "gen/reward_std": 0.23086503148078918, |
| "grad_norm": 0.004844950512051582, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "rewards/perceptual_score_reward_func/mean": 0.4858173727989197, |
| "rewards/perceptual_score_reward_func/std": 0.2394656240940094, |
| "step": 3, |
| "time_profile/curr_logprobs_and_update": 0.2888016360811889, |
| "time_profile/image_rollout": 85.37720386311412, |
| "time_profile/old_logps": 35.70834754779935, |
| "time_profile/ref_logps": 35.81470891647041, |
| "time_profile/reward": 1.1268615759909153, |
| "train/gen_step": 192.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.0011604064757193555, |
| "unified/loss": 4.6893979742890224e-05 |
| }, |
| { |
| "epoch": 0.051240992794235385, |
| "grad_norm": 0.00484450301155448, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "step": 4, |
| "time_profile/curr_logprobs_and_update": 0.2927547112049069, |
| "train/gen_step": 256.0, |
| "unified/clip_ratio/high_mean": 0.0007498142977055977, |
| "unified/clip_ratio/low_mean": 7.392094084934797e-05, |
| "unified/clip_ratio/region_mean": 0.0008237352385549457, |
| "unified/kl": 0.0015762942311994266, |
| "unified/loss": -1.2426604371285066e-05 |
| }, |
| { |
| "epoch": 0.06405124099279423, |
| "gen/advantages_abs_mean": 0.05059637129306793, |
| "gen/advantages_max": 0.29003840684890747, |
| "gen/advantages_mean": -4.94765117764473e-10, |
| "gen/advantages_min": -0.2061920166015625, |
| "gen/advantages_std": 0.07026641815900803, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 133.5, |
| "gen/prompt/min_length": 39.0, |
| "gen/reward": 0.5134599208831787, |
| "gen/reward_std": 0.2526681423187256, |
| "grad_norm": 0.0043946485966444016, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/perceptual_score_reward_func/mean": 0.5262137055397034, |
| "rewards/perceptual_score_reward_func/std": 0.2509992718696594, |
| "step": 5, |
| "time_profile/curr_logprobs_and_update": 0.2874946145748254, |
| "time_profile/image_rollout": 93.83373009413481, |
| "time_profile/old_logps": 35.67024562694132, |
| "time_profile/ref_logps": 35.715081388130784, |
| "time_profile/reward": 5.443569419905543, |
| "train/gen_step": 320.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.0018907583944383077, |
| "unified/loss": 7.613089474034496e-05 |
| }, |
| { |
| "epoch": 0.07686148919135308, |
| "grad_norm": 0.004030726384371519, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "step": 6, |
| "time_profile/curr_logprobs_and_update": 0.2908289354527369, |
| "train/gen_step": 384.0, |
| "unified/clip_ratio/high_mean": 0.0009645656609791331, |
| "unified/clip_ratio/low_mean": 0.00019809185232588788, |
| "unified/clip_ratio/region_mean": 0.0011626575096670422, |
| "unified/kl": 0.002805404281389201, |
| "unified/loss": 0.00010875017323996872 |
| }, |
| { |
| "epoch": 0.08967173738991192, |
| "gen/advantages_abs_mean": 0.05144927650690079, |
| "gen/advantages_max": 0.15652674436569214, |
| "gen/advantages_mean": 1.1641532182693481e-09, |
| "gen/advantages_min": -0.3314337134361267, |
| "gen/advantages_std": 0.06792663037776947, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 88.0, |
| "gen/prompt/min_length": 33.0, |
| "gen/reward": 0.7259606122970581, |
| "gen/reward_std": 0.11048998683691025, |
| "grad_norm": 0.004164206329733133, |
| "learning_rate": 1e-05, |
| "loss": 0.0004, |
| "rewards/perceptual_score_reward_func/mean": 0.7273693084716797, |
| "rewards/perceptual_score_reward_func/std": 0.10635492205619812, |
| "step": 7, |
| "time_profile/curr_logprobs_and_update": 0.28579148260178044, |
| "time_profile/image_rollout": 93.78808394446969, |
| "time_profile/old_logps": 35.66581052169204, |
| "time_profile/ref_logps": 35.73940089531243, |
| "time_profile/reward": 1.449136609211564, |
| "train/gen_step": 448.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.010006293836340774, |
| "unified/loss": 0.0003717760555446148 |
| }, |
| { |
| "epoch": 0.10248198558847077, |
| "grad_norm": 0.003952718339860439, |
| "learning_rate": 1e-05, |
| "loss": 0.0006, |
| "step": 8, |
| "time_profile/curr_logprobs_and_update": 0.2963303836877458, |
| "train/gen_step": 512.0, |
| "unified/clip_ratio/high_mean": 0.0006130620704425382, |
| "unified/clip_ratio/low_mean": 0.002367428182878939, |
| "unified/clip_ratio/region_mean": 0.0029804902487740037, |
| "unified/kl": 0.015264490411937004, |
| "unified/loss": 0.0006416451851691818 |
| }, |
| { |
| "epoch": 0.11529223378702963, |
| "gen/advantages_abs_mean": 0.08506780862808228, |
| "gen/advantages_max": 0.3090728223323822, |
| "gen/advantages_mean": 1.6880221664905548e-09, |
| "gen/advantages_min": -0.27506160736083984, |
| "gen/advantages_std": 0.10490497946739197, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 137.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.25868576765060425, |
| "gen/reward_std": 0.11191420257091522, |
| "grad_norm": 0.006360508035868406, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/perceptual_score_reward_func/mean": 0.25572726130485535, |
| "rewards/perceptual_score_reward_func/std": 0.11308468878269196, |
| "step": 9, |
| "time_profile/curr_logprobs_and_update": 0.2922193466220051, |
| "time_profile/image_rollout": 77.53796414472163, |
| "time_profile/old_logps": 35.67887348122895, |
| "time_profile/ref_logps": 35.77796713076532, |
| "time_profile/reward": 1.6198791358619928, |
| "train/gen_step": 576.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.003742291562957689, |
| "unified/loss": 0.00015398636605823413 |
| }, |
| { |
| "epoch": 0.12810248198558846, |
| "grad_norm": 0.006031422410160303, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "step": 10, |
| "time_profile/curr_logprobs_and_update": 0.28741057447041385, |
| "train/gen_step": 640.0, |
| "unified/clip_ratio/high_mean": 0.00038488462996610906, |
| "unified/clip_ratio/low_mean": 4.0186700971389655e-05, |
| "unified/clip_ratio/region_mean": 0.0004250713309374987, |
| "unified/kl": 0.004935910466883797, |
| "unified/loss": 0.00012804145080735907 |
| }, |
| { |
| "epoch": 0.14091273018414732, |
| "gen/advantages_abs_mean": 0.06213773041963577, |
| "gen/advantages_max": 0.23558755218982697, |
| "gen/advantages_mean": -1.4260876923799515e-09, |
| "gen/advantages_min": -0.23738695681095123, |
| "gen/advantages_std": 0.0792107805609703, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 30.0, |
| "gen/prompt/mean_length": 30.0, |
| "gen/prompt/min_length": 30.0, |
| "gen/reward": 0.2623980641365051, |
| "gen/reward_std": 0.10127276927232742, |
| "grad_norm": 0.00370898493565619, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/perceptual_score_reward_func/mean": 0.27432122826576233, |
| "rewards/perceptual_score_reward_func/std": 0.0917019173502922, |
| "step": 11, |
| "time_profile/curr_logprobs_and_update": 0.29560886629042216, |
| "time_profile/image_rollout": 94.6510800216347, |
| "time_profile/old_logps": 35.63860863447189, |
| "time_profile/ref_logps": 35.675177259370685, |
| "time_profile/reward": 1.475969910621643, |
| "train/gen_step": 704.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.0035910366859752685, |
| "unified/loss": 0.00014697932192575536 |
| }, |
| { |
| "epoch": 0.15372297838270615, |
| "grad_norm": 0.00393424229696393, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "step": 12, |
| "time_profile/curr_logprobs_and_update": 0.28807597383274697, |
| "train/gen_step": 768.0, |
| "unified/clip_ratio/high_mean": 0.0004313259105401812, |
| "unified/clip_ratio/low_mean": 2.7469435735838488e-05, |
| "unified/clip_ratio/region_mean": 0.0004587953462760197, |
| "unified/kl": 0.0045709875666943844, |
| "unified/loss": 0.00014194081450114027 |
| }, |
| { |
| "epoch": 0.16653322658126501, |
| "gen/advantages_abs_mean": 0.05380730703473091, |
| "gen/advantages_max": 0.32686302065849304, |
| "gen/advantages_mean": 3.259629011154175e-09, |
| "gen/advantages_min": -0.1513519287109375, |
| "gen/advantages_std": 0.0732535719871521, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 36.0, |
| "gen/prompt/mean_length": 32.0, |
| "gen/prompt/min_length": 28.0, |
| "gen/reward": 0.4303140640258789, |
| "gen/reward_std": 0.35259875655174255, |
| "grad_norm": 0.003100387519225478, |
| "learning_rate": 1e-05, |
| "loss": 0.0002, |
| "rewards/perceptual_score_reward_func/mean": 0.43855977058410645, |
| "rewards/perceptual_score_reward_func/std": 0.3576069474220276, |
| "step": 13, |
| "time_profile/curr_logprobs_and_update": 0.2939714658714365, |
| "time_profile/image_rollout": 90.3282467238605, |
| "time_profile/old_logps": 35.644943645223975, |
| "time_profile/ref_logps": 35.6841149572283, |
| "time_profile/reward": 1.4403626210987568, |
| "train/gen_step": 832.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.00593089139147196, |
| "unified/loss": 0.00022935521064937348 |
| }, |
| { |
| "epoch": 0.17934347477982385, |
| "grad_norm": 0.0031230556778609753, |
| "learning_rate": 1e-05, |
| "loss": 0.0003, |
| "step": 14, |
| "time_profile/curr_logprobs_and_update": 0.28601749025983736, |
| "train/gen_step": 896.0, |
| "unified/clip_ratio/high_mean": 0.00032966266735456884, |
| "unified/clip_ratio/low_mean": 1.3754401152255014e-05, |
| "unified/clip_ratio/region_mean": 0.00034341706850682385, |
| "unified/kl": 0.00694357078828034, |
| "unified/loss": 0.00020494235377555015 |
| }, |
| { |
| "epoch": 0.1921537229783827, |
| "gen/advantages_abs_mean": 0.035464800894260406, |
| "gen/advantages_max": 0.18103191256523132, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.16873770952224731, |
| "gen/advantages_std": 0.047589611262083054, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 32.0, |
| "gen/prompt/mean_length": 30.5, |
| "gen/prompt/min_length": 29.0, |
| "gen/reward": 0.4555138349533081, |
| "gen/reward_std": 0.3522937297821045, |
| "grad_norm": 0.0038101491518318653, |
| "learning_rate": 1e-05, |
| "loss": 0.0003, |
| "rewards/perceptual_score_reward_func/mean": 0.4537283182144165, |
| "rewards/perceptual_score_reward_func/std": 0.3564547300338745, |
| "step": 15, |
| "time_profile/curr_logprobs_and_update": 0.2867734569008462, |
| "time_profile/image_rollout": 93.33716715313494, |
| "time_profile/old_logps": 35.64802477508783, |
| "time_profile/ref_logps": 35.67770854756236, |
| "time_profile/reward": 1.3251771815121174, |
| "train/gen_step": 960.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.007522514912125189, |
| "unified/loss": 0.00029024387777099037 |
| }, |
| { |
| "epoch": 0.20496397117694154, |
| "grad_norm": 0.003709500189870596, |
| "learning_rate": 1e-05, |
| "loss": 0.0004, |
| "step": 16, |
| "time_profile/curr_logprobs_and_update": 0.2836496669333428, |
| "train/gen_step": 1024.0, |
| "unified/clip_ratio/high_mean": 0.0001654277248235303, |
| "unified/clip_ratio/low_mean": 0.00014177265802572947, |
| "unified/clip_ratio/region_mean": 0.00030720038193976507, |
| "unified/kl": 0.009284940926590934, |
| "unified/loss": 0.0003907864602297195 |
| }, |
| { |
| "epoch": 0.2177742193755004, |
| "gen/advantages_abs_mean": 0.06693125516176224, |
| "gen/advantages_max": 0.2859118580818176, |
| "gen/advantages_mean": -2.3283064365386963e-09, |
| "gen/advantages_min": -0.248762309551239, |
| "gen/advantages_std": 0.08490201085805893, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 26.0, |
| "gen/prompt/mean_length": 25.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.3253200650215149, |
| "gen/reward_std": 0.12534543871879578, |
| "grad_norm": 0.003804264822974801, |
| "learning_rate": 1e-05, |
| "loss": 0.0004, |
| "rewards/perceptual_score_reward_func/mean": 0.30608081817626953, |
| "rewards/perceptual_score_reward_func/std": 0.1369846761226654, |
| "step": 17, |
| "time_profile/curr_logprobs_and_update": 0.28391146194189787, |
| "time_profile/image_rollout": 91.10511220991611, |
| "time_profile/old_logps": 35.609973045066, |
| "time_profile/ref_logps": 35.6577035933733, |
| "time_profile/reward": 1.3727597426623106, |
| "train/gen_step": 1088.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.00982376610772917, |
| "unified/loss": 0.00041090739432547707 |
| }, |
| { |
| "epoch": 0.23058446757405926, |
| "grad_norm": 0.0037233256734907627, |
| "learning_rate": 1e-05, |
| "loss": 0.0004, |
| "step": 18, |
| "time_profile/curr_logprobs_and_update": 0.2903437889472116, |
| "train/gen_step": 1152.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.01129970328474883, |
| "unified/loss": 0.00045823437358194496 |
| }, |
| { |
| "epoch": 0.2433947157726181, |
| "gen/advantages_abs_mean": 0.055955708026885986, |
| "gen/advantages_max": 0.2999653220176697, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.25927937030792236, |
| "gen/advantages_std": 0.07066643238067627, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 34.0, |
| "gen/prompt/mean_length": 33.0, |
| "gen/prompt/min_length": 32.0, |
| "gen/reward": 0.5881402492523193, |
| "gen/reward_std": 0.16526837646961212, |
| "grad_norm": 0.004073821473866701, |
| "learning_rate": 1e-05, |
| "loss": 0.0005, |
| "rewards/perceptual_score_reward_func/mean": 0.5861481428146362, |
| "rewards/perceptual_score_reward_func/std": 0.16784866154193878, |
| "step": 19, |
| "time_profile/curr_logprobs_and_update": 0.2822369273926597, |
| "time_profile/image_rollout": 88.6840718369931, |
| "time_profile/old_logps": 35.64870150946081, |
| "time_profile/ref_logps": 35.687422059476376, |
| "time_profile/reward": 1.4169904831796885, |
| "train/gen_step": 1216.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.013576237324741669, |
| "unified/loss": 0.0005496463236340787 |
| }, |
| { |
| "epoch": 0.2562049639711769, |
| "grad_norm": 0.0040005045011639595, |
| "learning_rate": 1e-05, |
| "loss": 0.0006, |
| "step": 20, |
| "time_profile/curr_logprobs_and_update": 0.28987074297037907, |
| "train/gen_step": 1280.0, |
| "unified/clip_ratio/high_mean": 0.0002034804583672667, |
| "unified/clip_ratio/low_mean": 9.962243984773522e-05, |
| "unified/clip_ratio/region_mean": 0.0003031028982150019, |
| "unified/kl": 0.014395568068721332, |
| "unified/loss": 0.0005391188487919862 |
| }, |
| { |
| "epoch": 0.2690152121697358, |
| "gen/advantages_abs_mean": 0.06234389543533325, |
| "gen/advantages_max": 0.23057347536087036, |
| "gen/advantages_mean": -1.862645149230957e-09, |
| "gen/advantages_min": -0.2955120801925659, |
| "gen/advantages_std": 0.08112785965204239, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 185.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.5955153703689575, |
| "gen/reward_std": 0.19560492038726807, |
| "grad_norm": 0.006979112047702074, |
| "learning_rate": 1e-05, |
| "loss": 0.0062, |
| "rewards/perceptual_score_reward_func/mean": 0.5835608839988708, |
| "rewards/perceptual_score_reward_func/std": 0.1863791048526764, |
| "step": 21, |
| "time_profile/curr_logprobs_and_update": 0.28707319442764856, |
| "time_profile/image_rollout": 89.13379067741334, |
| "time_profile/old_logps": 35.708857618272305, |
| "time_profile/ref_logps": 35.791965547949076, |
| "time_profile/reward": 1.1867827828973532, |
| "train/gen_step": 1344.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.15566039714030921, |
| "unified/loss": 0.005685318652922433 |
| }, |
| { |
| "epoch": 0.28182546036829464, |
| "grad_norm": 0.00766932126134634, |
| "learning_rate": 1e-05, |
| "loss": 0.0052, |
| "step": 22, |
| "time_profile/curr_logprobs_and_update": 0.2838307637430262, |
| "train/gen_step": 1408.0, |
| "unified/clip_ratio/high_mean": 0.010980058965287753, |
| "unified/clip_ratio/low_mean": 0.00033267143953708, |
| "unified/clip_ratio/region_mean": 0.011312730418467254, |
| "unified/kl": 0.13134970283135772, |
| "unified/loss": 0.005792835259853746 |
| }, |
| { |
| "epoch": 0.2946357085668535, |
| "gen/advantages_abs_mean": 0.05016437917947769, |
| "gen/advantages_max": 0.2511361837387085, |
| "gen/advantages_mean": -2.0372681319713593e-10, |
| "gen/advantages_min": -0.2357330471277237, |
| "gen/advantages_std": 0.06782188266515732, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 49.0, |
| "gen/prompt/mean_length": 38.0, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.48785874247550964, |
| "gen/reward_std": 0.313375324010849, |
| "grad_norm": 0.003058127360418439, |
| "learning_rate": 1e-05, |
| "loss": 0.0007, |
| "rewards/perceptual_score_reward_func/mean": 0.4932803511619568, |
| "rewards/perceptual_score_reward_func/std": 0.3043711483478546, |
| "step": 23, |
| "time_profile/curr_logprobs_and_update": 0.2894766298995819, |
| "time_profile/image_rollout": 90.673310758546, |
| "time_profile/old_logps": 35.6381083894521, |
| "time_profile/ref_logps": 35.683547265827656, |
| "time_profile/reward": 1.3350308034569025, |
| "train/gen_step": 1472.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.018112926394678652, |
| "unified/loss": 0.0007296701951418072 |
| }, |
| { |
| "epoch": 0.3074459567654123, |
| "grad_norm": 0.003112577134743333, |
| "learning_rate": 1e-05, |
| "loss": 0.0007, |
| "step": 24, |
| "time_profile/curr_logprobs_and_update": 0.28356376514420845, |
| "train/gen_step": 1536.0, |
| "unified/clip_ratio/high_mean": 0.00036169634040561505, |
| "unified/clip_ratio/low_mean": 1.2056327250320464e-05, |
| "unified/clip_ratio/region_mean": 0.0003737526712939143, |
| "unified/kl": 0.018430211246595718, |
| "unified/loss": 0.0007105697004590183 |
| }, |
| { |
| "epoch": 0.32025620496397117, |
| "gen/advantages_abs_mean": 0.06575670838356018, |
| "gen/advantages_max": 0.19591188430786133, |
| "gen/advantages_mean": -3.4924596548080444e-09, |
| "gen/advantages_min": -0.37458205223083496, |
| "gen/advantages_std": 0.08331269770860672, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 90.5, |
| "gen/prompt/min_length": 38.0, |
| "gen/reward": 0.7573187351226807, |
| "gen/reward_std": 0.08858481794595718, |
| "grad_norm": 0.004597505554556847, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "rewards/perceptual_score_reward_func/mean": 0.7637672424316406, |
| "rewards/perceptual_score_reward_func/std": 0.07734499126672745, |
| "step": 25, |
| "time_profile/curr_logprobs_and_update": 0.28622239985270426, |
| "time_profile/image_rollout": 86.45065759681165, |
| "time_profile/old_logps": 35.67832253314555, |
| "time_profile/ref_logps": 35.702019242569804, |
| "time_profile/reward": 1.359033975750208, |
| "train/gen_step": 1600.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.029347103234613314, |
| "unified/loss": 0.0012196608440717682 |
| }, |
| { |
| "epoch": 0.33306645316253003, |
| "grad_norm": 0.0043387808836996555, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 26, |
| "time_profile/curr_logprobs_and_update": 0.28386400899034925, |
| "train/gen_step": 1664.0, |
| "unified/clip_ratio/high_mean": 0.01954531196224707, |
| "unified/clip_ratio/low_mean": 0.0003240542173443828, |
| "unified/clip_ratio/region_mean": 0.019869366309649195, |
| "unified/kl": 0.023536839376902208, |
| "unified/loss": 0.0010066187054320608 |
| }, |
| { |
| "epoch": 0.3458767013610889, |
| "gen/advantages_abs_mean": 0.057586055248975754, |
| "gen/advantages_max": 0.21371035277843475, |
| "gen/advantages_mean": -1.1641532182693481e-09, |
| "gen/advantages_min": -0.2889488935470581, |
| "gen/advantages_std": 0.08002809435129166, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 40.0, |
| "gen/prompt/mean_length": 33.0, |
| "gen/prompt/min_length": 26.0, |
| "gen/reward": 0.5128387212753296, |
| "gen/reward_std": 0.3075273931026459, |
| "grad_norm": 0.0035108160227537155, |
| "learning_rate": 1e-05, |
| "loss": 0.0007, |
| "rewards/perceptual_score_reward_func/mean": 0.5264440774917603, |
| "rewards/perceptual_score_reward_func/std": 0.3053143322467804, |
| "step": 27, |
| "time_profile/curr_logprobs_and_update": 0.28258850090787746, |
| "time_profile/image_rollout": 93.14971325173974, |
| "time_profile/old_logps": 35.633352022618055, |
| "time_profile/ref_logps": 35.67252997867763, |
| "time_profile/reward": 1.3629055880010128, |
| "train/gen_step": 1728.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.017757456327672116, |
| "unified/loss": 0.0006818042947998038 |
| }, |
| { |
| "epoch": 0.3586869495596477, |
| "grad_norm": 0.003482175525277853, |
| "learning_rate": 1e-05, |
| "loss": 0.0007, |
| "step": 28, |
| "time_profile/curr_logprobs_and_update": 0.29045128886355087, |
| "train/gen_step": 1792.0, |
| "unified/clip_ratio/high_mean": 0.003981999492680188, |
| "unified/clip_ratio/low_mean": 6.156700783321867e-05, |
| "unified/clip_ratio/region_mean": 0.004043566495965933, |
| "unified/kl": 0.016798593336716294, |
| "unified/loss": 0.0006475578447862063 |
| }, |
| { |
| "epoch": 0.37149719775820655, |
| "gen/advantages_abs_mean": 0.020567672327160835, |
| "gen/advantages_max": 0.0832895040512085, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.09890902042388916, |
| "gen/advantages_std": 0.0276055708527565, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 35.0, |
| "gen/prompt/mean_length": 34.5, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.8026696443557739, |
| "gen/reward_std": 0.035780273377895355, |
| "grad_norm": 0.001524417893961072, |
| "learning_rate": 1e-05, |
| "loss": 0.0008, |
| "rewards/perceptual_score_reward_func/mean": 0.7995024919509888, |
| "rewards/perceptual_score_reward_func/std": 0.039139553904533386, |
| "step": 29, |
| "time_profile/curr_logprobs_and_update": 0.28697867854498327, |
| "time_profile/image_rollout": 94.98372022993863, |
| "time_profile/old_logps": 35.64544406160712, |
| "time_profile/ref_logps": 35.6720716599375, |
| "time_profile/reward": 1.554461432620883, |
| "train/gen_step": 1856.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.020413192978594452, |
| "unified/loss": 0.0008108863465707827 |
| }, |
| { |
| "epoch": 0.3843074459567654, |
| "grad_norm": 0.001496512326411903, |
| "learning_rate": 1e-05, |
| "loss": 0.0008, |
| "step": 30, |
| "time_profile/curr_logprobs_and_update": 0.28794672252843156, |
| "train/gen_step": 1920.0, |
| "unified/clip_ratio/high_mean": 0.00028818798364227405, |
| "unified/clip_ratio/low_mean": 0.0001083762999769533, |
| "unified/clip_ratio/region_mean": 0.00039656428452872206, |
| "unified/kl": 0.020366095268400386, |
| "unified/loss": 0.000806004970286267 |
| }, |
| { |
| "epoch": 0.3971176941553243, |
| "gen/advantages_abs_mean": 0.07551919668912888, |
| "gen/advantages_max": 0.24253734946250916, |
| "gen/advantages_mean": -2.561137080192566e-09, |
| "gen/advantages_min": -0.31094181537628174, |
| "gen/advantages_std": 0.09551571309566498, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 89.0, |
| "gen/prompt/min_length": 35.0, |
| "gen/reward": 0.4983167052268982, |
| "gen/reward_std": 0.2392866164445877, |
| "grad_norm": 0.004808772820979357, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.5010837316513062, |
| "rewards/perceptual_score_reward_func/std": 0.2483944445848465, |
| "step": 31, |
| "time_profile/curr_logprobs_and_update": 0.2927919552021194, |
| "time_profile/image_rollout": 92.86078766733408, |
| "time_profile/old_logps": 35.645628007128835, |
| "time_profile/ref_logps": 35.68816146440804, |
| "time_profile/reward": 1.1220357697457075, |
| "train/gen_step": 1984.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.028540802421048284, |
| "unified/loss": 0.00118683417849752 |
| }, |
| { |
| "epoch": 0.4099279423538831, |
| "grad_norm": 0.004993099253624678, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 32, |
| "time_profile/curr_logprobs_and_update": 0.2836689332325477, |
| "train/gen_step": 2048.0, |
| "unified/clip_ratio/high_mean": 0.004601992815878475, |
| "unified/clip_ratio/low_mean": 8.766420523897978e-05, |
| "unified/clip_ratio/region_mean": 0.004689657042035833, |
| "unified/kl": 0.0318972848035628, |
| "unified/loss": 0.0012409869095790782 |
| }, |
| { |
| "epoch": 0.42273819055244194, |
| "gen/advantages_abs_mean": 0.056175872683525085, |
| "gen/advantages_max": 0.2621963918209076, |
| "gen/advantages_mean": 1.3969838619232178e-09, |
| "gen/advantages_min": -0.3464314341545105, |
| "gen/advantages_std": 0.07949691265821457, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 85.5, |
| "gen/prompt/min_length": 28.0, |
| "gen/reward": 0.5262117385864258, |
| "gen/reward_std": 0.21805663406848907, |
| "grad_norm": 0.003093303646892309, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "rewards/perceptual_score_reward_func/mean": 0.5296328663825989, |
| "rewards/perceptual_score_reward_func/std": 0.2171928584575653, |
| "step": 33, |
| "time_profile/curr_logprobs_and_update": 0.28117132117040455, |
| "time_profile/image_rollout": 94.87426543608308, |
| "time_profile/old_logps": 35.6621759980917, |
| "time_profile/ref_logps": 35.68978282995522, |
| "time_profile/reward": 1.123130239546299, |
| "train/gen_step": 2112.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.03187064320081845, |
| "unified/loss": 0.0012426345692801988 |
| }, |
| { |
| "epoch": 0.4355484387510008, |
| "grad_norm": 0.0030948214698582888, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 34, |
| "time_profile/curr_logprobs_and_update": 0.28984188855974935, |
| "train/gen_step": 2176.0, |
| "unified/clip_ratio/high_mean": 0.0015250134501911816, |
| "unified/clip_ratio/low_mean": 0.00027515896545082796, |
| "unified/clip_ratio/region_mean": 0.0018001724210989778, |
| "unified/kl": 0.032954427908407524, |
| "unified/loss": 0.0013315071737451945 |
| }, |
| { |
| "epoch": 0.44835868694955966, |
| "gen/advantages_abs_mean": 0.06641550362110138, |
| "gen/advantages_max": 0.2938459515571594, |
| "gen/advantages_mean": -2.9103830456733704e-11, |
| "gen/advantages_min": -0.1641228049993515, |
| "gen/advantages_std": 0.0816919133067131, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 126.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.08744116127490997, |
| "gen/reward_std": 0.09168319404125214, |
| "grad_norm": 0.004929918795824051, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.08148351311683655, |
| "rewards/perceptual_score_reward_func/std": 0.09414726495742798, |
| "step": 35, |
| "time_profile/curr_logprobs_and_update": 0.2904790450411383, |
| "time_profile/image_rollout": 92.59001582488418, |
| "time_profile/old_logps": 35.86070014163852, |
| "time_profile/ref_logps": 35.71136962622404, |
| "time_profile/reward": 2.4510285947471857, |
| "train/gen_step": 2240.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.028016642027068883, |
| "unified/loss": 0.0010704244341468439 |
| }, |
| { |
| "epoch": 0.4611689351481185, |
| "grad_norm": 0.004974815063178539, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "step": 36, |
| "time_profile/curr_logprobs_and_update": 0.2918431573198177, |
| "train/gen_step": 2304.0, |
| "unified/clip_ratio/high_mean": 0.00020790819235116942, |
| "unified/clip_ratio/low_mean": 2.3113905626814812e-05, |
| "unified/clip_ratio/region_mean": 0.00023102209797798423, |
| "unified/kl": 0.029212650420959108, |
| "unified/loss": 0.0011148997673444683 |
| }, |
| { |
| "epoch": 0.4739791833466773, |
| "gen/advantages_abs_mean": 0.019589366391301155, |
| "gen/advantages_max": 0.16522341966629028, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.06063012778759003, |
| "gen/advantages_std": 0.032727889716625214, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.28125, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 132.0, |
| "gen/prompt/min_length": 36.0, |
| "gen/reward": 0.015548557974398136, |
| "gen/reward_std": 0.03653242811560631, |
| "grad_norm": 0.0015641744248569012, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.019582821056246758, |
| "rewards/perceptual_score_reward_func/std": 0.04001577943563461, |
| "step": 37, |
| "time_profile/curr_logprobs_and_update": 0.2838708157360088, |
| "time_profile/image_rollout": 90.96994621679187, |
| "time_profile/old_logps": 35.68323879875243, |
| "time_profile/ref_logps": 35.691848032176495, |
| "time_profile/reward": 1.3393241744488478, |
| "train/gen_step": 2368.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.026530865143286064, |
| "unified/loss": 0.0010725572483352153 |
| }, |
| { |
| "epoch": 0.4867894315452362, |
| "grad_norm": 0.001707454095594585, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "step": 38, |
| "time_profile/curr_logprobs_and_update": 0.28927692130673677, |
| "train/gen_step": 2432.0, |
| "unified/clip_ratio/high_mean": 2.3696002244832925e-05, |
| "unified/clip_ratio/low_mean": 1.4909351193637121e-05, |
| "unified/clip_ratio/region_mean": 3.860535343847005e-05, |
| "unified/kl": 0.026749580312753096, |
| "unified/loss": 0.0010843233976629563 |
| }, |
| { |
| "epoch": 0.49959967974379504, |
| "gen/advantages_abs_mean": 0.06686871498823166, |
| "gen/advantages_max": 0.2528344690799713, |
| "gen/advantages_mean": 2.3283064365386963e-09, |
| "gen/advantages_min": -0.27862676978111267, |
| "gen/advantages_std": 0.08488749712705612, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 160.5, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.29595229029655457, |
| "gen/reward_std": 0.09182636439800262, |
| "grad_norm": 0.003406939096748829, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.3082679212093353, |
| "rewards/perceptual_score_reward_func/std": 0.08436276763677597, |
| "step": 39, |
| "time_profile/curr_logprobs_and_update": 0.28643454433768056, |
| "time_profile/image_rollout": 98.25927837193012, |
| "time_profile/old_logps": 35.69824261032045, |
| "time_profile/ref_logps": 35.77457028999925, |
| "time_profile/reward": 6.389842351898551, |
| "train/gen_step": 2496.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.0277218354458455, |
| "unified/loss": 0.0011168291480316839 |
| }, |
| { |
| "epoch": 0.5124099279423538, |
| "grad_norm": 0.0038018571212887764, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "step": 40, |
| "time_profile/curr_logprobs_and_update": 0.2840952040278353, |
| "train/gen_step": 2560.0, |
| "unified/clip_ratio/high_mean": 0.00011745172923838254, |
| "unified/clip_ratio/low_mean": 2.7233115361013915e-05, |
| "unified/clip_ratio/region_mean": 0.00014468484368990175, |
| "unified/kl": 0.028168133198050782, |
| "unified/loss": 0.001051058910888969 |
| }, |
| { |
| "epoch": 0.5252201761409128, |
| "gen/advantages_abs_mean": 0.06853524595499039, |
| "gen/advantages_max": 0.1936076283454895, |
| "gen/advantages_mean": -6.984919309616089e-09, |
| "gen/advantages_min": -0.3802328109741211, |
| "gen/advantages_std": 0.08733128011226654, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.653662919998169, |
| "gen/reward_std": 0.09350146353244781, |
| "grad_norm": 0.007019991986453533, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "rewards/perceptual_score_reward_func/mean": 0.6532255411148071, |
| "rewards/perceptual_score_reward_func/std": 0.08254469931125641, |
| "step": 41, |
| "time_profile/curr_logprobs_and_update": 0.2885236190923024, |
| "time_profile/image_rollout": 96.83855919353664, |
| "time_profile/old_logps": 35.68985055014491, |
| "time_profile/ref_logps": 35.7790841050446, |
| "time_profile/reward": 1.0563220214098692, |
| "train/gen_step": 2624.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.08251963119255379, |
| "unified/loss": 0.003462206368567422 |
| }, |
| { |
| "epoch": 0.5380304243394716, |
| "grad_norm": 0.004827939905226231, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "step": 42, |
| "time_profile/curr_logprobs_and_update": 0.28419688864960335, |
| "train/gen_step": 2688.0, |
| "unified/clip_ratio/high_mean": 0.001810626625228906, |
| "unified/clip_ratio/low_mean": 0.002413051798612287, |
| "unified/clip_ratio/region_mean": 0.004223678405651299, |
| "unified/kl": 0.07354312663665041, |
| "unified/loss": 0.00292131217156566 |
| }, |
| { |
| "epoch": 0.5508406725380304, |
| "gen/advantages_abs_mean": 0.03801333159208298, |
| "gen/advantages_max": 0.18501146137714386, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.17977194488048553, |
| "gen/advantages_std": 0.052959226071834564, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 38.0, |
| "gen/prompt/mean_length": 31.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.48641788959503174, |
| "gen/reward_std": 0.30649667978286743, |
| "grad_norm": 0.0027381409890949726, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/perceptual_score_reward_func/mean": 0.4760347306728363, |
| "rewards/perceptual_score_reward_func/std": 0.3147166967391968, |
| "step": 43, |
| "time_profile/curr_logprobs_and_update": 0.2863878221542109, |
| "time_profile/image_rollout": 93.18704553879797, |
| "time_profile/old_logps": 35.647348675876856, |
| "time_profile/ref_logps": 35.67785409279168, |
| "time_profile/reward": 1.3080625962466002, |
| "train/gen_step": 2752.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.022025975049473345, |
| "unified/loss": 0.0009260941587854177 |
| }, |
| { |
| "epoch": 0.5636509207365893, |
| "grad_norm": 0.0029822327196598053, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "step": 44, |
| "time_profile/curr_logprobs_and_update": 0.2840305963472929, |
| "train/gen_step": 2816.0, |
| "unified/clip_ratio/high_mean": 3.7968629840179347e-05, |
| "unified/clip_ratio/low_mean": 6.098106496210676e-05, |
| "unified/clip_ratio/region_mean": 9.89496948022861e-05, |
| "unified/kl": 0.02211095401435159, |
| "unified/loss": 0.0009122132069023792 |
| }, |
| { |
| "epoch": 0.5764611689351481, |
| "gen/advantages_abs_mean": 0.03591323643922806, |
| "gen/advantages_max": 0.09316939115524292, |
| "gen/advantages_mean": -4.190951585769653e-09, |
| "gen/advantages_min": -0.20308387279510498, |
| "gen/advantages_std": 0.047014325857162476, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 85.5, |
| "gen/prompt/min_length": 28.0, |
| "gen/reward": 0.7401652336120605, |
| "gen/reward_std": 0.07190340012311935, |
| "grad_norm": 0.0034790660720318556, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "rewards/perceptual_score_reward_func/mean": 0.7485611438751221, |
| "rewards/perceptual_score_reward_func/std": 0.06707578152418137, |
| "step": 45, |
| "time_profile/curr_logprobs_and_update": 0.2859906947123818, |
| "time_profile/image_rollout": 91.15467045269907, |
| "time_profile/old_logps": 35.6747612785548, |
| "time_profile/ref_logps": 35.70546899549663, |
| "time_profile/reward": 1.08854722045362, |
| "train/gen_step": 2880.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.03551414527464658, |
| "unified/loss": 0.0014457265460805502 |
| }, |
| { |
| "epoch": 0.589271417133707, |
| "grad_norm": 0.002829823410138488, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "step": 46, |
| "time_profile/curr_logprobs_and_update": 0.2882837516372092, |
| "train/gen_step": 2944.0, |
| "unified/clip_ratio/high_mean": 0.0007083340506142122, |
| "unified/clip_ratio/low_mean": 0.00391641097212414, |
| "unified/clip_ratio/region_mean": 0.004624744986358564, |
| "unified/kl": 0.035664693103171885, |
| "unified/loss": 0.0014884003412589664 |
| }, |
| { |
| "epoch": 0.6020816653322658, |
| "gen/advantages_abs_mean": 0.05315067991614342, |
| "gen/advantages_max": 0.2091759443283081, |
| "gen/advantages_mean": 4.889443516731262e-09, |
| "gen/advantages_min": -0.3550701141357422, |
| "gen/advantages_std": 0.06988511979579926, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.7280530333518982, |
| "gen/reward_std": 0.07879945635795593, |
| "grad_norm": 0.0043497150763869286, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "rewards/perceptual_score_reward_func/mean": 0.7486039400100708, |
| "rewards/perceptual_score_reward_func/std": 0.056908342987298965, |
| "step": 47, |
| "time_profile/curr_logprobs_and_update": 0.2845378862693906, |
| "time_profile/image_rollout": 93.7537659406662, |
| "time_profile/old_logps": 35.71550615131855, |
| "time_profile/ref_logps": 35.81924728676677, |
| "time_profile/reward": 0.9302971065044403, |
| "train/gen_step": 3008.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.05950034986017272, |
| "unified/loss": 0.0025013487907017407 |
| }, |
| { |
| "epoch": 0.6148919135308246, |
| "grad_norm": 0.0044915638864040375, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "step": 48, |
| "time_profile/curr_logprobs_and_update": 0.28826455789385363, |
| "train/gen_step": 3072.0, |
| "unified/clip_ratio/high_mean": 0.0014778335389564745, |
| "unified/clip_ratio/low_mean": 0.009821245124840061, |
| "unified/clip_ratio/region_mean": 0.011299078611045843, |
| "unified/kl": 0.06371590058552101, |
| "unified/loss": 0.002573426672597634 |
| }, |
| { |
| "epoch": 0.6277021617293835, |
| "gen/advantages_abs_mean": 0.03862232714891434, |
| "gen/advantages_max": 0.24333956837654114, |
| "gen/advantages_mean": 1.1641532182693481e-09, |
| "gen/advantages_min": -0.1539299339056015, |
| "gen/advantages_std": 0.05260282754898071, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 38.0, |
| "gen/prompt/mean_length": 33.5, |
| "gen/prompt/min_length": 29.0, |
| "gen/reward": 0.4815671443939209, |
| "gen/reward_std": 0.33772698044776917, |
| "grad_norm": 0.00316769746132195, |
| "learning_rate": 1e-05, |
| "loss": 0.0008, |
| "rewards/perceptual_score_reward_func/mean": 0.490455687046051, |
| "rewards/perceptual_score_reward_func/std": 0.3341163992881775, |
| "step": 49, |
| "time_profile/curr_logprobs_and_update": 0.28858013937133364, |
| "time_profile/image_rollout": 92.26549053192139, |
| "time_profile/old_logps": 35.63793566450477, |
| "time_profile/ref_logps": 35.693480079993606, |
| "time_profile/reward": 2.303144110366702, |
| "train/gen_step": 3136.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.020838526252191514, |
| "unified/loss": 0.0008064835201366805 |
| }, |
| { |
| "epoch": 0.6405124099279423, |
| "grad_norm": 0.0031122236978262663, |
| "learning_rate": 1e-05, |
| "loss": 0.0008, |
| "step": 50, |
| "time_profile/curr_logprobs_and_update": 0.2807842147012707, |
| "train/gen_step": 3200.0, |
| "unified/clip_ratio/high_mean": 0.00040497850750398356, |
| "unified/clip_ratio/low_mean": 5.118025183037389e-05, |
| "unified/clip_ratio/region_mean": 0.00045615876297233626, |
| "unified/kl": 0.02007605423568748, |
| "unified/loss": 0.0008150048051902559 |
| }, |
| { |
| "epoch": 0.6533226581265013, |
| "gen/advantages_abs_mean": 0.045201025903224945, |
| "gen/advantages_max": 0.2404613345861435, |
| "gen/advantages_mean": -2.5320332497358322e-09, |
| "gen/advantages_min": -0.16845963895320892, |
| "gen/advantages_std": 0.06456216424703598, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 36.0, |
| "gen/prompt/mean_length": 33.0, |
| "gen/prompt/min_length": 30.0, |
| "gen/reward": 0.48360586166381836, |
| "gen/reward_std": 0.3726156949996948, |
| "grad_norm": 0.004150118213146925, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "rewards/perceptual_score_reward_func/mean": 0.4762686491012573, |
| "rewards/perceptual_score_reward_func/std": 0.38303321599960327, |
| "step": 51, |
| "time_profile/curr_logprobs_and_update": 0.2836454862554092, |
| "time_profile/image_rollout": 90.03667972795665, |
| "time_profile/old_logps": 35.62950346246362, |
| "time_profile/ref_logps": 35.67210088297725, |
| "time_profile/reward": 1.1959608290344477, |
| "train/gen_step": 3264.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.023927463014842942, |
| "unified/loss": 0.000979490663667093 |
| }, |
| { |
| "epoch": 0.6661329063250601, |
| "grad_norm": 0.00425776606425643, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "step": 52, |
| "time_profile/curr_logprobs_and_update": 0.2833061977289617, |
| "train/gen_step": 3328.0, |
| "unified/clip_ratio/high_mean": 0.0007207577100416529, |
| "unified/clip_ratio/low_mean": 7.231485324155074e-05, |
| "unified/clip_ratio/region_mean": 0.0007930725596452248, |
| "unified/kl": 0.022969681856920943, |
| "unified/loss": 0.0008664954584673978 |
| }, |
| { |
| "epoch": 0.6789431545236189, |
| "gen/advantages_abs_mean": 0.06953415274620056, |
| "gen/advantages_max": 0.22283008694648743, |
| "gen/advantages_mean": 1.3969838619232178e-09, |
| "gen/advantages_min": -0.2646213173866272, |
| "gen/advantages_std": 0.08586404472589493, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 215.5, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.4072887897491455, |
| "gen/reward_std": 0.23019105195999146, |
| "grad_norm": 0.003899160074070096, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "rewards/perceptual_score_reward_func/mean": 0.42112410068511963, |
| "rewards/perceptual_score_reward_func/std": 0.23615583777427673, |
| "step": 53, |
| "time_profile/curr_logprobs_and_update": 0.2833262274216395, |
| "time_profile/image_rollout": 78.63981351442635, |
| "time_profile/old_logps": 35.7274684663862, |
| "time_profile/ref_logps": 35.797853803262115, |
| "time_profile/reward": 1.171833161264658, |
| "train/gen_step": 3392.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.03973888815380633, |
| "unified/loss": 0.0015896564364084043 |
| }, |
| { |
| "epoch": 0.6917534027221778, |
| "grad_norm": 0.0038946911226958036, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 54, |
| "time_profile/curr_logprobs_and_update": 0.28796653434983455, |
| "train/gen_step": 3456.0, |
| "unified/clip_ratio/high_mean": 0.00015383093978016404, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.00015383093978016404, |
| "unified/kl": 0.03877779032336548, |
| "unified/loss": 0.0015223519985738676 |
| }, |
| { |
| "epoch": 0.7045636509207366, |
| "gen/advantages_abs_mean": 0.09249347448348999, |
| "gen/advantages_max": 0.3164886236190796, |
| "gen/advantages_mean": 4.3655745685100555e-10, |
| "gen/advantages_min": -0.258684903383255, |
| "gen/advantages_std": 0.11256097257137299, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 28.0, |
| "gen/prompt/mean_length": 26.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.2795722782611847, |
| "gen/reward_std": 0.13351306319236755, |
| "grad_norm": 0.004936882294714451, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/perceptual_score_reward_func/mean": 0.2909090518951416, |
| "rewards/perceptual_score_reward_func/std": 0.13314305245876312, |
| "step": 55, |
| "time_profile/curr_logprobs_and_update": 0.28386063207290135, |
| "time_profile/image_rollout": 97.24604052305222, |
| "time_profile/old_logps": 35.62250469997525, |
| "time_profile/ref_logps": 35.67051147669554, |
| "time_profile/reward": 1.362681845203042, |
| "train/gen_step": 3520.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.0227790946373716, |
| "unified/loss": 0.0008780551270319847 |
| }, |
| { |
| "epoch": 0.7173738991192954, |
| "grad_norm": 0.00479149492457509, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "step": 56, |
| "time_profile/curr_logprobs_and_update": 0.28903222834924236, |
| "train/gen_step": 3584.0, |
| "unified/clip_ratio/high_mean": 0.000115000895675621, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.000115000895675621, |
| "unified/kl": 0.024405151838436723, |
| "unified/loss": 0.0010101461557496805 |
| }, |
| { |
| "epoch": 0.7301841473178543, |
| "gen/advantages_abs_mean": 0.058377232402563095, |
| "gen/advantages_max": 0.24894979596138, |
| "gen/advantages_mean": -2.561137080192566e-09, |
| "gen/advantages_min": -0.2390967607498169, |
| "gen/advantages_std": 0.07458589226007462, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 37.0, |
| "gen/prompt/mean_length": 32.0, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.5388422012329102, |
| "gen/reward_std": 0.1490672528743744, |
| "grad_norm": 0.0033019648399204016, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/perceptual_score_reward_func/mean": 0.5304601192474365, |
| "rewards/perceptual_score_reward_func/std": 0.15523570775985718, |
| "step": 57, |
| "time_profile/curr_logprobs_and_update": 0.28454931010492146, |
| "time_profile/image_rollout": 89.89481943659484, |
| "time_profile/old_logps": 35.618444772437215, |
| "time_profile/ref_logps": 35.67245299369097, |
| "time_profile/reward": 1.3785167206078768, |
| "train/gen_step": 3648.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.021809650992508978, |
| "unified/loss": 0.0008770865549649898 |
| }, |
| { |
| "epoch": 0.7429943955164131, |
| "grad_norm": 0.0033817507792264223, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "step": 58, |
| "time_profile/curr_logprobs_and_update": 0.28436136766686104, |
| "train/gen_step": 3712.0, |
| "unified/clip_ratio/high_mean": 0.00040959819307317957, |
| "unified/clip_ratio/low_mean": 2.537267664592946e-05, |
| "unified/clip_ratio/region_mean": 0.00043497086971910903, |
| "unified/kl": 0.023068611073540524, |
| "unified/loss": 0.0008947431465458067 |
| }, |
| { |
| "epoch": 0.755804643714972, |
| "gen/advantages_abs_mean": 0.06415918469429016, |
| "gen/advantages_max": 0.3439922332763672, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.25892210006713867, |
| "gen/advantages_std": 0.08247826993465424, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 28.0, |
| "gen/prompt/mean_length": 27.0, |
| "gen/prompt/min_length": 26.0, |
| "gen/reward": 0.23676720261573792, |
| "gen/reward_std": 0.08878710865974426, |
| "grad_norm": 0.002476656809449196, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.23685407638549805, |
| "rewards/perceptual_score_reward_func/std": 0.09089451283216476, |
| "step": 59, |
| "time_profile/curr_logprobs_and_update": 0.2890419715840835, |
| "time_profile/image_rollout": 91.48738839104772, |
| "time_profile/old_logps": 35.63207217492163, |
| "time_profile/ref_logps": 35.67341075837612, |
| "time_profile/reward": 1.305101539939642, |
| "train/gen_step": 3776.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.027545947610633448, |
| "unified/loss": 0.0011357355069776531 |
| }, |
| { |
| "epoch": 0.7686148919135308, |
| "grad_norm": 0.002608807757496834, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "step": 60, |
| "time_profile/curr_logprobs_and_update": 0.28312389296479523, |
| "train/gen_step": 3840.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 1.302083364862483e-05, |
| "unified/clip_ratio/region_mean": 1.302083364862483e-05, |
| "unified/kl": 0.027554543688893318, |
| "unified/loss": 0.0010848145793715958 |
| }, |
| { |
| "epoch": 0.7814251401120896, |
| "gen/advantages_abs_mean": 0.03390387073159218, |
| "gen/advantages_max": 0.13214147090911865, |
| "gen/advantages_mean": -2.561137080192566e-09, |
| "gen/advantages_min": -0.3367220163345337, |
| "gen/advantages_std": 0.04783477634191513, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 87.0, |
| "gen/prompt/min_length": 31.0, |
| "gen/reward": 0.6992359757423401, |
| "gen/reward_std": 0.09428206086158752, |
| "grad_norm": 0.0026902405079454184, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "rewards/perceptual_score_reward_func/mean": 0.6967364549636841, |
| "rewards/perceptual_score_reward_func/std": 0.09561172127723694, |
| "step": 61, |
| "time_profile/curr_logprobs_and_update": 0.2933660880953539, |
| "time_profile/image_rollout": 90.9877971354872, |
| "time_profile/old_logps": 35.695311322808266, |
| "time_profile/ref_logps": 35.70469231158495, |
| "time_profile/reward": 1.2580257393419743, |
| "train/gen_step": 3904.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.04532510880380869, |
| "unified/loss": 0.001845876574407157 |
| }, |
| { |
| "epoch": 0.7942353883106485, |
| "grad_norm": 0.002456333488225937, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "step": 62, |
| "time_profile/curr_logprobs_and_update": 0.289033788634697, |
| "train/gen_step": 3968.0, |
| "unified/clip_ratio/high_mean": 0.0044582416012417525, |
| "unified/clip_ratio/low_mean": 0.00011452383205323713, |
| "unified/clip_ratio/region_mean": 0.004572765432385495, |
| "unified/kl": 0.04306245065527037, |
| "unified/loss": 0.0016709059446498031 |
| }, |
| { |
| "epoch": 0.8070456365092074, |
| "gen/advantages_abs_mean": 0.04794721305370331, |
| "gen/advantages_max": 0.16062045097351074, |
| "gen/advantages_mean": -2.3283064365386963e-09, |
| "gen/advantages_min": -0.2264614701271057, |
| "gen/advantages_std": 0.06226880103349686, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 33.0, |
| "gen/prompt/mean_length": 31.5, |
| "gen/prompt/min_length": 30.0, |
| "gen/reward": 0.7169865965843201, |
| "gen/reward_std": 0.08426859974861145, |
| "grad_norm": 0.0035603002179414034, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "rewards/perceptual_score_reward_func/mean": 0.7329443693161011, |
| "rewards/perceptual_score_reward_func/std": 0.07624883949756622, |
| "step": 63, |
| "time_profile/curr_logprobs_and_update": 0.2918489087896887, |
| "time_profile/image_rollout": 90.09776932001114, |
| "time_profile/old_logps": 35.6436334438622, |
| "time_profile/ref_logps": 35.685858277603984, |
| "time_profile/reward": 1.3784125726670027, |
| "train/gen_step": 4032.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.024762474262388423, |
| "unified/loss": 0.000963904687523609 |
| }, |
| { |
| "epoch": 0.8198558847077662, |
| "grad_norm": 0.003421743866056204, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 64, |
| "time_profile/curr_logprobs_and_update": 0.28897311072796583, |
| "train/gen_step": 4096.0, |
| "unified/clip_ratio/high_mean": 0.0002471591642461135, |
| "unified/clip_ratio/low_mean": 0.00021569021282630274, |
| "unified/clip_ratio/region_mean": 0.00046284937798191095, |
| "unified/kl": 0.024078681715764105, |
| "unified/loss": 0.0009506012538622599 |
| }, |
| { |
| "epoch": 0.8326661329063251, |
| "gen/advantages_abs_mean": 0.03795839846134186, |
| "gen/advantages_max": 0.19034850597381592, |
| "gen/advantages_mean": 2.3283064365386963e-09, |
| "gen/advantages_min": -0.11913624405860901, |
| "gen/advantages_std": 0.05202922970056534, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 90.5, |
| "gen/prompt/min_length": 44.0, |
| "gen/reward": 0.4587080478668213, |
| "gen/reward_std": 0.3764707148075104, |
| "grad_norm": 0.0019860498141497374, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/perceptual_score_reward_func/mean": 0.45268112421035767, |
| "rewards/perceptual_score_reward_func/std": 0.3850635290145874, |
| "step": 65, |
| "time_profile/curr_logprobs_and_update": 0.2939559489605017, |
| "time_profile/image_rollout": 87.89024894498289, |
| "time_profile/old_logps": 35.630313439294696, |
| "time_profile/ref_logps": 35.68208645284176, |
| "time_profile/reward": 1.5224378574639559, |
| "train/gen_step": 4160.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.023230015503941104, |
| "unified/loss": 0.0009148021113105642 |
| }, |
| { |
| "epoch": 0.8454763811048839, |
| "grad_norm": 0.0018545385682955384, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "step": 66, |
| "time_profile/curr_logprobs_and_update": 0.2903638135176152, |
| "train/gen_step": 4224.0, |
| "unified/clip_ratio/high_mean": 0.00017140993986686226, |
| "unified/clip_ratio/low_mean": 0.00045493132347473875, |
| "unified/clip_ratio/region_mean": 0.0006263412597036222, |
| "unified/kl": 0.02263786207186058, |
| "unified/loss": 0.0009286232896101865 |
| }, |
| { |
| "epoch": 0.8582866293034428, |
| "gen/advantages_abs_mean": 0.06805029511451721, |
| "gen/advantages_max": 0.33124828338623047, |
| "gen/advantages_mean": -3.055902197957039e-09, |
| "gen/advantages_min": -0.2604416608810425, |
| "gen/advantages_std": 0.08757373690605164, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 34.0, |
| "gen/prompt/mean_length": 30.5, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.4795072078704834, |
| "gen/reward_std": 0.2444414645433426, |
| "grad_norm": 0.00417951587587595, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.48338961601257324, |
| "rewards/perceptual_score_reward_func/std": 0.2435249537229538, |
| "step": 67, |
| "time_profile/curr_logprobs_and_update": 0.2845795691537205, |
| "time_profile/image_rollout": 94.08473618142307, |
| "time_profile/old_logps": 35.653244607150555, |
| "time_profile/ref_logps": 35.68242741189897, |
| "time_profile/reward": 1.570898663252592, |
| "train/gen_step": 4288.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.027502871002070606, |
| "unified/loss": 0.001087280929368717 |
| }, |
| { |
| "epoch": 0.8710968775020016, |
| "grad_norm": 0.004231288097798824, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 68, |
| "time_profile/curr_logprobs_and_update": 0.28916991435107775, |
| "train/gen_step": 4352.0, |
| "unified/clip_ratio/high_mean": 0.000318356032948941, |
| "unified/clip_ratio/low_mean": 0.000416163186855556, |
| "unified/clip_ratio/region_mean": 0.0007345192207139917, |
| "unified/kl": 0.026457387197297066, |
| "unified/loss": 0.0010340961512156355 |
| }, |
| { |
| "epoch": 0.8839071257005604, |
| "gen/advantages_abs_mean": 0.04543104022741318, |
| "gen/advantages_max": 0.23743367195129395, |
| "gen/advantages_mean": -3.026798367500305e-09, |
| "gen/advantages_min": -0.31966233253479004, |
| "gen/advantages_std": 0.06301993876695633, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 166.5, |
| "gen/prompt/min_length": 39.0, |
| "gen/reward": 0.6233011484146118, |
| "gen/reward_std": 0.24330009520053864, |
| "grad_norm": 0.00291136815212667, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "rewards/perceptual_score_reward_func/mean": 0.6394872665405273, |
| "rewards/perceptual_score_reward_func/std": 0.22583803534507751, |
| "step": 69, |
| "time_profile/curr_logprobs_and_update": 0.29470567259704694, |
| "time_profile/image_rollout": 83.45683548785746, |
| "time_profile/old_logps": 35.69185835123062, |
| "time_profile/ref_logps": 35.72108877636492, |
| "time_profile/reward": 1.4233777895569801, |
| "train/gen_step": 4416.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.043804147600894794, |
| "unified/loss": 0.0017766948712960584 |
| }, |
| { |
| "epoch": 0.8967173738991193, |
| "grad_norm": 0.003228710265830159, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 70, |
| "time_profile/curr_logprobs_and_update": 0.28153255736106075, |
| "train/gen_step": 4480.0, |
| "unified/clip_ratio/high_mean": 0.00013945894897915423, |
| "unified/clip_ratio/low_mean": 0.000410210202971939, |
| "unified/clip_ratio/region_mean": 0.0005496691501321038, |
| "unified/kl": 0.04113790258998051, |
| "unified/loss": 0.001632484994843253 |
| }, |
| { |
| "epoch": 0.9095276220976781, |
| "gen/advantages_abs_mean": 0.04988550767302513, |
| "gen/advantages_max": 0.25346121191978455, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.20711418986320496, |
| "gen/advantages_std": 0.06387721002101898, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 165.0, |
| "gen/prompt/min_length": 36.0, |
| "gen/reward": 0.510562539100647, |
| "gen/reward_std": 0.2098800390958786, |
| "grad_norm": 0.002879665931686759, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.5041624903678894, |
| "rewards/perceptual_score_reward_func/std": 0.20727090537548065, |
| "step": 71, |
| "time_profile/curr_logprobs_and_update": 0.2848870683228597, |
| "time_profile/image_rollout": 89.13787977769971, |
| "time_profile/old_logps": 35.68902938440442, |
| "time_profile/ref_logps": 35.7388895843178, |
| "time_profile/reward": 1.2322177048772573, |
| "train/gen_step": 4544.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.026298649958334863, |
| "unified/loss": 0.0010413604650238995 |
| }, |
| { |
| "epoch": 0.922337870296237, |
| "grad_norm": 0.002842222573235631, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 72, |
| "time_profile/curr_logprobs_and_update": 0.287293476780178, |
| "train/gen_step": 4608.0, |
| "unified/clip_ratio/high_mean": 0.00032580364859313704, |
| "unified/clip_ratio/low_mean": 0.0001650192261877237, |
| "unified/clip_ratio/region_mean": 0.0004908228756903554, |
| "unified/kl": 0.025436352443648502, |
| "unified/loss": 0.0009967807036446175 |
| }, |
| { |
| "epoch": 0.9351481184947958, |
| "gen/advantages_abs_mean": 0.06981154531240463, |
| "gen/advantages_max": 0.2795065641403198, |
| "gen/advantages_mean": 9.313225746154785e-10, |
| "gen/advantages_min": -0.2961052358150482, |
| "gen/advantages_std": 0.08908019214868546, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 129.0, |
| "gen/prompt/min_length": 30.0, |
| "gen/reward": 0.3487090468406677, |
| "gen/reward_std": 0.10480765253305435, |
| "grad_norm": 0.0054478757083415985, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "rewards/perceptual_score_reward_func/mean": 0.34865331649780273, |
| "rewards/perceptual_score_reward_func/std": 0.09679755568504333, |
| "step": 73, |
| "time_profile/curr_logprobs_and_update": 0.2976631856581662, |
| "time_profile/image_rollout": 93.59027141705155, |
| "time_profile/old_logps": 35.697798715904355, |
| "time_profile/ref_logps": 35.81994019635022, |
| "time_profile/reward": 1.8824999630451202, |
| "train/gen_step": 4672.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.030644683924037963, |
| "unified/loss": 0.0012316222382651176 |
| }, |
| { |
| "epoch": 0.9479583666933546, |
| "grad_norm": 0.005177585408091545, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 74, |
| "time_profile/curr_logprobs_and_update": 0.28828327282099053, |
| "train/gen_step": 4736.0, |
| "unified/clip_ratio/high_mean": 0.00013896780728828162, |
| "unified/clip_ratio/low_mean": 0.00019588950817706063, |
| "unified/clip_ratio/region_mean": 0.00033485731455584755, |
| "unified/kl": 0.03049655826180242, |
| "unified/loss": 0.0014046990941096738 |
| }, |
| { |
| "epoch": 0.9607686148919136, |
| "gen/advantages_abs_mean": 0.05187247693538666, |
| "gen/advantages_max": 0.17589330673217773, |
| "gen/advantages_mean": -1.3969838619232178e-09, |
| "gen/advantages_min": -0.3043435215950012, |
| "gen/advantages_std": 0.07163926959037781, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 88.0, |
| "gen/prompt/min_length": 33.0, |
| "gen/reward": 0.7588982582092285, |
| "gen/reward_std": 0.09124431014060974, |
| "grad_norm": 0.008605101145803928, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "rewards/perceptual_score_reward_func/mean": 0.7651864290237427, |
| "rewards/perceptual_score_reward_func/std": 0.07482659071683884, |
| "step": 75, |
| "time_profile/curr_logprobs_and_update": 0.28617306941305287, |
| "time_profile/image_rollout": 95.36229601316154, |
| "time_profile/old_logps": 35.670409236103296, |
| "time_profile/ref_logps": 35.708794893696904, |
| "time_profile/reward": 1.2329577188938856, |
| "train/gen_step": 4800.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.07468988915206864, |
| "unified/loss": 0.0028182062202404268 |
| }, |
| { |
| "epoch": 0.9735788630904724, |
| "grad_norm": 0.003986031282693148, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "step": 76, |
| "time_profile/curr_logprobs_and_update": 0.29819186983513646, |
| "train/gen_step": 4864.0, |
| "unified/clip_ratio/high_mean": 0.003613574607697956, |
| "unified/clip_ratio/low_mean": 0.0007360392692135065, |
| "unified/clip_ratio/region_mean": 0.0043496139069247874, |
| "unified/kl": 0.06856736988993362, |
| "unified/loss": 0.0028428112407254957 |
| }, |
| { |
| "epoch": 0.9863891112890312, |
| "gen/advantages_abs_mean": 0.04775109142065048, |
| "gen/advantages_max": 0.19789379835128784, |
| "gen/advantages_mean": 3.4924596548080444e-09, |
| "gen/advantages_min": -0.39248591661453247, |
| "gen/advantages_std": 0.06537375599145889, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 87.5, |
| "gen/prompt/min_length": 32.0, |
| "gen/reward": 0.6827777028083801, |
| "gen/reward_std": 0.09045189619064331, |
| "grad_norm": 0.005181001964956522, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "rewards/perceptual_score_reward_func/mean": 0.6811036467552185, |
| "rewards/perceptual_score_reward_func/std": 0.09361086785793304, |
| "step": 77, |
| "time_profile/curr_logprobs_and_update": 0.29179857825511135, |
| "time_profile/image_rollout": 83.27612288482487, |
| "time_profile/old_logps": 35.658640841022134, |
| "time_profile/ref_logps": 35.68707458116114, |
| "time_profile/reward": 1.1409052349627018, |
| "train/gen_step": 4928.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.05598265668959357, |
| "unified/loss": 0.0022124095494291396 |
| }, |
| { |
| "epoch": 0.9991993594875901, |
| "grad_norm": 0.005045454483479261, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "step": 78, |
| "time_profile/curr_logprobs_and_update": 0.2866454735340085, |
| "train/gen_step": 4992.0, |
| "unified/clip_ratio/high_mean": 0.001705825293356611, |
| "unified/clip_ratio/low_mean": 0.002124744443790405, |
| "unified/clip_ratio/region_mean": 0.0038305697316900478, |
| "unified/kl": 0.04944189221714623, |
| "unified/loss": 0.0019907314253941877 |
| }, |
| { |
| "epoch": 1.0, |
| "gen/advantages_abs_mean": 0.05695275217294693, |
| "gen/advantages_max": 0.29835107922554016, |
| "gen/advantages_mean": -1.862645149230957e-09, |
| "gen/advantages_min": -0.24371516704559326, |
| "gen/advantages_std": 0.07456254214048386, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 45.0, |
| "gen/prompt/mean_length": 38.0, |
| "gen/prompt/min_length": 31.0, |
| "gen/reward": 0.5770522356033325, |
| "gen/reward_std": 0.21481432020664215, |
| "grad_norm": 0.0033948994241654873, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "rewards/perceptual_score_reward_func/mean": 0.5808098316192627, |
| "rewards/perceptual_score_reward_func/std": 0.21286322176456451, |
| "step": 79, |
| "time_profile/curr_logprobs_and_update": 0.28097593411803246, |
| "time_profile/image_rollout": 86.5219391360879, |
| "time_profile/old_logps": 35.649459190666676, |
| "time_profile/ref_logps": 35.691718278452754, |
| "time_profile/reward": 1.29204579629004, |
| "train/gen_step": 4996.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.025911119766533375, |
| "unified/loss": 0.008625521790236235 |
| }, |
| { |
| "epoch": 1.0128102481985588, |
| "grad_norm": 0.002015733392909169, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 80, |
| "time_profile/curr_logprobs_and_update": 0.2885962183645461, |
| "train/gen_step": 5060.0, |
| "unified/clip_ratio/high_mean": 0.0003266334970248863, |
| "unified/clip_ratio/low_mean": 0.00028056274277332705, |
| "unified/clip_ratio/region_mean": 0.000607196237979224, |
| "unified/kl": 0.02535266784252599, |
| "unified/loss": 0.00100777412262687 |
| }, |
| { |
| "epoch": 1.0256204963971176, |
| "gen/advantages_abs_mean": 0.04768690466880798, |
| "gen/advantages_max": 0.23502177000045776, |
| "gen/advantages_mean": -1.6298145055770874e-09, |
| "gen/advantages_min": -0.17850369215011597, |
| "gen/advantages_std": 0.06418629735708237, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 39.0, |
| "gen/prompt/mean_length": 32.0, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.5296216011047363, |
| "gen/reward_std": 0.24720948934555054, |
| "grad_norm": 0.002834862098097801, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "rewards/perceptual_score_reward_func/mean": 0.5289547443389893, |
| "rewards/perceptual_score_reward_func/std": 0.24988876283168793, |
| "step": 81, |
| "time_profile/curr_logprobs_and_update": 0.28801219374872744, |
| "time_profile/image_rollout": 86.72076600790024, |
| "time_profile/old_logps": 35.62672356516123, |
| "time_profile/ref_logps": 35.670366356149316, |
| "time_profile/reward": 1.4512761607766151, |
| "train/gen_step": 5124.0, |
| "unified/clip_ratio/high_mean": 0.0009658800263423473, |
| "unified/clip_ratio/low_mean": 0.0008977456272987183, |
| "unified/clip_ratio/region_mean": 0.001863625655460055, |
| "unified/kl": 0.025290006422437727, |
| "unified/loss": -0.0006758762265235418 |
| }, |
| { |
| "epoch": 1.0384307445956766, |
| "grad_norm": 0.003795901546254754, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 82, |
| "time_profile/curr_logprobs_and_update": 0.29509411737672053, |
| "train/gen_step": 5188.0, |
| "unified/clip_ratio/high_mean": 0.0003557014515536139, |
| "unified/clip_ratio/low_mean": 0.00017478797963121906, |
| "unified/clip_ratio/region_mean": 0.0005304894348228117, |
| "unified/kl": 0.03130401810631156, |
| "unified/loss": 0.000909826174163797 |
| }, |
| { |
| "epoch": 1.0512409927942354, |
| "gen/advantages_abs_mean": 0.050498440861701965, |
| "gen/advantages_max": 0.19424709677696228, |
| "gen/advantages_mean": 3.026798367500305e-09, |
| "gen/advantages_min": -0.28340137004852295, |
| "gen/advantages_std": 0.06539153307676315, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 32.0, |
| "gen/prompt/mean_length": 28.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.4895917773246765, |
| "gen/reward_std": 0.17331120371818542, |
| "grad_norm": 0.003306619357317686, |
| "learning_rate": 1e-05, |
| "loss": -0.0001, |
| "rewards/perceptual_score_reward_func/mean": 0.497758686542511, |
| "rewards/perceptual_score_reward_func/std": 0.1741664856672287, |
| "step": 83, |
| "time_profile/curr_logprobs_and_update": 0.28760485889506526, |
| "time_profile/image_rollout": 88.75942911952734, |
| "time_profile/old_logps": 35.62598751485348, |
| "time_profile/ref_logps": 35.6756409779191, |
| "time_profile/reward": 1.3580076191574335, |
| "train/gen_step": 5252.0, |
| "unified/clip_ratio/high_mean": 0.0012764136354235234, |
| "unified/clip_ratio/low_mean": 0.001044765193000785, |
| "unified/clip_ratio/region_mean": 0.0023211788338812767, |
| "unified/kl": 0.021809721365571022, |
| "unified/loss": 0.0047350469174034515 |
| }, |
| { |
| "epoch": 1.0640512409927942, |
| "grad_norm": 0.004482661839574575, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 84, |
| "time_profile/curr_logprobs_and_update": 0.28422601052443497, |
| "train/gen_step": 5316.0, |
| "unified/clip_ratio/high_mean": 0.00010350447519158479, |
| "unified/clip_ratio/low_mean": 6.248103727557464e-05, |
| "unified/clip_ratio/region_mean": 0.00016598551337665413, |
| "unified/kl": 0.02575801726197824, |
| "unified/loss": 0.0009556097375025274 |
| }, |
| { |
| "epoch": 1.076861489191353, |
| "gen/advantages_abs_mean": 0.07610887289047241, |
| "gen/advantages_max": 0.279594749212265, |
| "gen/advantages_mean": 4.656612873077393e-10, |
| "gen/advantages_min": -0.2552163898944855, |
| "gen/advantages_std": 0.09314155578613281, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 28.0, |
| "gen/prompt/mean_length": 26.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.16894452273845673, |
| "gen/reward_std": 0.12381462752819061, |
| "grad_norm": 0.005054818466305733, |
| "learning_rate": 1e-05, |
| "loss": 0.0005, |
| "rewards/perceptual_score_reward_func/mean": 0.1732819378376007, |
| "rewards/perceptual_score_reward_func/std": 0.11441599577665329, |
| "step": 85, |
| "time_profile/curr_logprobs_and_update": 0.2935908046492841, |
| "time_profile/image_rollout": 80.12255467288196, |
| "time_profile/old_logps": 35.63555760681629, |
| "time_profile/ref_logps": 35.66839297674596, |
| "time_profile/reward": 1.3038444705307484, |
| "train/gen_step": 5380.0, |
| "unified/clip_ratio/high_mean": 0.0005621371064989944, |
| "unified/clip_ratio/low_mean": 0.0003427702713452163, |
| "unified/clip_ratio/region_mean": 0.0009049073787537054, |
| "unified/kl": 0.02579947459162213, |
| "unified/loss": -0.0009912145637827052 |
| }, |
| { |
| "epoch": 1.0896717373899119, |
| "grad_norm": 0.00472763879224658, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "step": 86, |
| "time_profile/curr_logprobs_and_update": 0.2931293906294741, |
| "train/gen_step": 5444.0, |
| "unified/clip_ratio/high_mean": 2.467482136125909e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 2.467482136125909e-05, |
| "unified/kl": 0.023300431785173714, |
| "unified/loss": 0.0010191962392127607 |
| }, |
| { |
| "epoch": 1.1024819855884709, |
| "gen/advantages_abs_mean": 0.059635188430547714, |
| "gen/advantages_max": 0.23380088806152344, |
| "gen/advantages_mean": 2.0954757928848267e-09, |
| "gen/advantages_min": -0.3752295970916748, |
| "gen/advantages_std": 0.0778651088476181, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.4627537727355957, |
| "gen/reward_std": 0.2263425588607788, |
| "grad_norm": 0.005005163140594959, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "rewards/perceptual_score_reward_func/mean": 0.46124348044395447, |
| "rewards/perceptual_score_reward_func/std": 0.2309817224740982, |
| "step": 87, |
| "time_profile/curr_logprobs_and_update": 0.2806922975287307, |
| "time_profile/image_rollout": 86.29171478375793, |
| "time_profile/old_logps": 35.68952482007444, |
| "time_profile/ref_logps": 35.76360684260726, |
| "time_profile/reward": 1.391890512779355, |
| "train/gen_step": 5508.0, |
| "unified/clip_ratio/high_mean": 0.000248023759922944, |
| "unified/clip_ratio/low_mean": 5.1719837756536435e-05, |
| "unified/clip_ratio/region_mean": 0.00029974359676998574, |
| "unified/kl": 0.02937340398784727, |
| "unified/loss": -0.0005479482424561866 |
| }, |
| { |
| "epoch": 1.1152922337870297, |
| "grad_norm": 0.004413130227476358, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "step": 88, |
| "time_profile/curr_logprobs_and_update": 0.281406976893777, |
| "train/gen_step": 5572.0, |
| "unified/clip_ratio/high_mean": 0.0038563079042432946, |
| "unified/clip_ratio/low_mean": 9.112708903558087e-05, |
| "unified/clip_ratio/region_mean": 0.003947434987821907, |
| "unified/kl": 0.07321834639878944, |
| "unified/loss": 0.002794792841086746 |
| }, |
| { |
| "epoch": 1.1281024819855885, |
| "gen/advantages_abs_mean": 0.08485985547304153, |
| "gen/advantages_max": 0.37464210391044617, |
| "gen/advantages_mean": 9.604264050722122e-10, |
| "gen/advantages_min": -0.2651618421077728, |
| "gen/advantages_std": 0.10387808084487915, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 161.0, |
| "gen/prompt/min_length": 28.0, |
| "gen/reward": 0.3185094892978668, |
| "gen/reward_std": 0.18651913106441498, |
| "grad_norm": 0.006158351898193359, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "rewards/perceptual_score_reward_func/mean": 0.3045373857021332, |
| "rewards/perceptual_score_reward_func/std": 0.19984664022922516, |
| "step": 89, |
| "time_profile/curr_logprobs_and_update": 0.2874557306349743, |
| "time_profile/image_rollout": 89.15939953178167, |
| "time_profile/old_logps": 35.65223306231201, |
| "time_profile/ref_logps": 35.69842333719134, |
| "time_profile/reward": 1.4332980029284954, |
| "train/gen_step": 5636.0, |
| "unified/clip_ratio/high_mean": 0.034317739496145805, |
| "unified/clip_ratio/low_mean": 0.0007901403296273202, |
| "unified/clip_ratio/region_mean": 0.03510787995310238, |
| "unified/kl": 0.057693745475262403, |
| "unified/loss": 0.006083310350277316 |
| }, |
| { |
| "epoch": 1.1409127301841473, |
| "grad_norm": 0.006367347203195095, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 90, |
| "time_profile/curr_logprobs_and_update": 0.2880125379888341, |
| "train/gen_step": 5700.0, |
| "unified/clip_ratio/high_mean": 0.001398666523527936, |
| "unified/clip_ratio/low_mean": 0.002986333717672096, |
| "unified/clip_ratio/region_mean": 0.004385000211186707, |
| "unified/kl": 0.03243849166028667, |
| "unified/loss": 0.0013116523468852392 |
| }, |
| { |
| "epoch": 1.153722978382706, |
| "gen/advantages_abs_mean": 0.05420481413602829, |
| "gen/advantages_max": 0.20414644479751587, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.27419042587280273, |
| "gen/advantages_std": 0.07089695334434509, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 85.0, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.5731308460235596, |
| "gen/reward_std": 0.16429921984672546, |
| "grad_norm": 0.005784476175904274, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/perceptual_score_reward_func/mean": 0.5847713947296143, |
| "rewards/perceptual_score_reward_func/std": 0.1503916084766388, |
| "step": 91, |
| "time_profile/curr_logprobs_and_update": 0.29128942391253076, |
| "time_profile/image_rollout": 91.42569714412093, |
| "time_profile/old_logps": 35.68290564417839, |
| "time_profile/ref_logps": 35.72888129577041, |
| "time_profile/reward": 1.2796872407197952, |
| "train/gen_step": 5764.0, |
| "unified/clip_ratio/high_mean": 0.003734815680218162, |
| "unified/clip_ratio/low_mean": 0.009340950593468733, |
| "unified/clip_ratio/region_mean": 0.01307576622639317, |
| "unified/kl": 0.03101163225073833, |
| "unified/loss": 0.00124256549815982 |
| }, |
| { |
| "epoch": 1.1665332265812651, |
| "grad_norm": 0.004937492776662111, |
| "learning_rate": 1e-05, |
| "loss": 0.005, |
| "step": 92, |
| "time_profile/curr_logprobs_and_update": 0.2811797432950698, |
| "train/gen_step": 5828.0, |
| "unified/clip_ratio/high_mean": 0.007572715853711998, |
| "unified/clip_ratio/low_mean": 3.8088402106950525e-05, |
| "unified/clip_ratio/region_mean": 0.007610804260366422, |
| "unified/kl": 0.12355734975426458, |
| "unified/loss": 0.004843149222779175 |
| }, |
| { |
| "epoch": 1.179343474779824, |
| "gen/advantages_abs_mean": 0.03607391566038132, |
| "gen/advantages_max": 0.2704744338989258, |
| "gen/advantages_mean": -2.3283064365386963e-10, |
| "gen/advantages_min": -0.10774393379688263, |
| "gen/advantages_std": 0.0513719767332077, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.015625, |
| "gen/prompt/max_length": 46.0, |
| "gen/prompt/mean_length": 35.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.43351101875305176, |
| "gen/reward_std": 0.38293835520744324, |
| "grad_norm": 0.004236552864313126, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "rewards/perceptual_score_reward_func/mean": 0.43139415979385376, |
| "rewards/perceptual_score_reward_func/std": 0.3872426748275757, |
| "step": 93, |
| "time_profile/curr_logprobs_and_update": 0.28935843985527754, |
| "time_profile/image_rollout": 90.9723764192313, |
| "time_profile/old_logps": 35.630882170051336, |
| "time_profile/ref_logps": 35.66354006715119, |
| "time_profile/reward": 1.4163836408406496, |
| "train/gen_step": 5892.0, |
| "unified/clip_ratio/high_mean": 0.02233631252693158, |
| "unified/clip_ratio/low_mean": 0.00026049184361909283, |
| "unified/clip_ratio/region_mean": 0.022596804475142562, |
| "unified/kl": 0.11128485442895908, |
| "unified/loss": 0.0008558991967220209 |
| }, |
| { |
| "epoch": 1.1921537229783827, |
| "grad_norm": 0.003623092547059059, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 94, |
| "time_profile/curr_logprobs_and_update": 0.29402141278842464, |
| "train/gen_step": 5956.0, |
| "unified/clip_ratio/high_mean": 0.0002511533584765857, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0002511533584765857, |
| "unified/kl": 0.024947728699771687, |
| "unified/loss": 0.0009469758740578982 |
| }, |
| { |
| "epoch": 1.2049639711769415, |
| "gen/advantages_abs_mean": 0.061276111751794815, |
| "gen/advantages_max": 0.27673643827438354, |
| "gen/advantages_mean": -1.3969838619232178e-09, |
| "gen/advantages_min": -0.24275538325309753, |
| "gen/advantages_std": 0.07711324840784073, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 126.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.29714375734329224, |
| "gen/reward_std": 0.09595662355422974, |
| "grad_norm": 0.004491707310080528, |
| "learning_rate": 1e-05, |
| "loss": -0.0, |
| "rewards/perceptual_score_reward_func/mean": 0.2941877245903015, |
| "rewards/perceptual_score_reward_func/std": 0.10239575803279877, |
| "step": 95, |
| "time_profile/curr_logprobs_and_update": 0.2845701384358108, |
| "time_profile/image_rollout": 94.30383717268705, |
| "time_profile/old_logps": 35.69289446435869, |
| "time_profile/ref_logps": 35.735097793862224, |
| "time_profile/reward": 1.5108983255922794, |
| "train/gen_step": 6020.0, |
| "unified/clip_ratio/high_mean": 0.0010795590387715492, |
| "unified/clip_ratio/low_mean": 2.6937110305880196e-05, |
| "unified/clip_ratio/region_mean": 0.0011064961481679347, |
| "unified/kl": 0.024600972144980915, |
| "unified/loss": 0.0015992923219982913 |
| }, |
| { |
| "epoch": 1.2177742193755003, |
| "grad_norm": 0.004685665015131235, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "step": 96, |
| "time_profile/curr_logprobs_and_update": 0.2889473946997896, |
| "train/gen_step": 6084.0, |
| "unified/clip_ratio/high_mean": 0.00010233268585579935, |
| "unified/clip_ratio/low_mean": 2.49623572017299e-05, |
| "unified/clip_ratio/region_mean": 0.00012729504305752926, |
| "unified/kl": 0.02420059047290124, |
| "unified/loss": 0.0009602482255104405 |
| }, |
| { |
| "epoch": 1.2305844675740594, |
| "gen/advantages_abs_mean": 0.07217675447463989, |
| "gen/advantages_max": 0.22240379452705383, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.36782652139663696, |
| "gen/advantages_std": 0.09157998859882355, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.606522798538208, |
| "gen/reward_std": 0.1440057009458542, |
| "grad_norm": 0.0029499332886189222, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "rewards/perceptual_score_reward_func/mean": 0.6199157238006592, |
| "rewards/perceptual_score_reward_func/std": 0.13665904104709625, |
| "step": 97, |
| "time_profile/curr_logprobs_and_update": 0.29385396288125776, |
| "time_profile/image_rollout": 97.1500741597265, |
| "time_profile/old_logps": 35.72174118645489, |
| "time_profile/ref_logps": 35.81614971533418, |
| "time_profile/reward": 1.2525407243520021, |
| "train/gen_step": 6148.0, |
| "unified/clip_ratio/high_mean": 0.0007662127018193132, |
| "unified/clip_ratio/low_mean": 0.00015251044169417582, |
| "unified/clip_ratio/region_mean": 0.0009187231389660155, |
| "unified/kl": 0.026670520936022513, |
| "unified/loss": -0.0007936091969895642 |
| }, |
| { |
| "epoch": 1.2433947157726182, |
| "grad_norm": 0.004023274406790733, |
| "learning_rate": 1e-05, |
| "loss": 0.0041, |
| "step": 98, |
| "time_profile/curr_logprobs_and_update": 0.286594680743292, |
| "train/gen_step": 6212.0, |
| "unified/clip_ratio/high_mean": 0.001166129409284622, |
| "unified/clip_ratio/low_mean": 2.7221584787184838e-05, |
| "unified/clip_ratio/region_mean": 0.001193350993162312, |
| "unified/kl": 0.1031805292586796, |
| "unified/loss": 0.0034758291503749206 |
| }, |
| { |
| "epoch": 1.256204963971177, |
| "gen/advantages_abs_mean": 0.05734759569168091, |
| "gen/advantages_max": 0.1668534278869629, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.24680942296981812, |
| "gen/advantages_std": 0.0720229223370552, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.4030495882034302, |
| "gen/reward_std": 0.30972352623939514, |
| "grad_norm": 0.005159635562449694, |
| "learning_rate": 1e-05, |
| "loss": 0.0054, |
| "rewards/perceptual_score_reward_func/mean": 0.40351754426956177, |
| "rewards/perceptual_score_reward_func/std": 0.30575186014175415, |
| "step": 99, |
| "time_profile/curr_logprobs_and_update": 0.2843075899290852, |
| "time_profile/image_rollout": 90.91732196509838, |
| "time_profile/old_logps": 35.70482533983886, |
| "time_profile/ref_logps": 35.787240678444505, |
| "time_profile/reward": 1.3005148079246283, |
| "train/gen_step": 6276.0, |
| "unified/clip_ratio/high_mean": 0.005653069780237274, |
| "unified/clip_ratio/low_mean": 0.0001404002250637859, |
| "unified/clip_ratio/region_mean": 0.005793470003482071, |
| "unified/kl": 0.10310261632548645, |
| "unified/loss": 0.008152539892307686 |
| }, |
| { |
| "epoch": 1.2690152121697358, |
| "grad_norm": 0.005166908260434866, |
| "learning_rate": 1e-05, |
| "loss": 0.0044, |
| "step": 100, |
| "time_profile/curr_logprobs_and_update": 0.28935047722188756, |
| "train/gen_step": 6340.0, |
| "unified/clip_ratio/high_mean": 0.0013726951256103348, |
| "unified/clip_ratio/low_mean": 0.00010761726116470527, |
| "unified/clip_ratio/region_mean": 0.0014803123895035242, |
| "unified/kl": 0.11148487494210713, |
| "unified/loss": 0.005707373344193911 |
| }, |
| { |
| "epoch": 1.2818254603682946, |
| "gen/advantages_abs_mean": 0.05996637046337128, |
| "gen/advantages_max": 0.22481316328048706, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.28761735558509827, |
| "gen/advantages_std": 0.0771557167172432, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 37.0, |
| "gen/prompt/mean_length": 33.5, |
| "gen/prompt/min_length": 30.0, |
| "gen/reward": 0.4563029110431671, |
| "gen/reward_std": 0.1505608707666397, |
| "grad_norm": 0.0037838416174054146, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "rewards/perceptual_score_reward_func/mean": 0.45866450667381287, |
| "rewards/perceptual_score_reward_func/std": 0.15732455253601074, |
| "step": 101, |
| "time_profile/curr_logprobs_and_update": 0.2890072492300533, |
| "time_profile/image_rollout": 89.5529919564724, |
| "time_profile/old_logps": 35.6248143799603, |
| "time_profile/ref_logps": 35.6683790050447, |
| "time_profile/reward": 1.463647324591875, |
| "train/gen_step": 6404.0, |
| "unified/clip_ratio/high_mean": 0.00770890203784802, |
| "unified/clip_ratio/low_mean": 0.0010239453413305455, |
| "unified/clip_ratio/region_mean": 0.008732847421924816, |
| "unified/kl": 0.10659060714533553, |
| "unified/loss": 0.001588718660059385 |
| }, |
| { |
| "epoch": 1.2946357085668536, |
| "grad_norm": 0.0035527099389582872, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 102, |
| "time_profile/curr_logprobs_and_update": 0.29435791762080044, |
| "train/gen_step": 6468.0, |
| "unified/clip_ratio/high_mean": 0.00019829840948659694, |
| "unified/clip_ratio/low_mean": 1.2361550943751354e-05, |
| "unified/clip_ratio/region_mean": 0.0002106599595208536, |
| "unified/kl": 0.029273734951857477, |
| "unified/loss": 0.0010736352372759939 |
| }, |
| { |
| "epoch": 1.3074459567654122, |
| "gen/advantages_abs_mean": 0.05255986750125885, |
| "gen/advantages_max": 0.26030582189559937, |
| "gen/advantages_mean": 2.9103830456733704e-11, |
| "gen/advantages_min": -0.1432579606771469, |
| "gen/advantages_std": 0.06540312618017197, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 159.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.0877436026930809, |
| "gen/reward_std": 0.07358266413211823, |
| "grad_norm": 0.004366860259324312, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "rewards/perceptual_score_reward_func/mean": 0.07803992182016373, |
| "rewards/perceptual_score_reward_func/std": 0.07350844889879227, |
| "step": 103, |
| "time_profile/curr_logprobs_and_update": 0.2862840360903647, |
| "time_profile/image_rollout": 92.44728139974177, |
| "time_profile/old_logps": 35.6767134424299, |
| "time_profile/ref_logps": 35.706137884408236, |
| "time_profile/reward": 1.3225576486438513, |
| "train/gen_step": 6532.0, |
| "unified/clip_ratio/high_mean": 0.0006179989059091895, |
| "unified/clip_ratio/low_mean": 2.4732017664064188e-05, |
| "unified/clip_ratio/region_mean": 0.0006427309235732537, |
| "unified/kl": 0.030823647743090987, |
| "unified/loss": 0.0035945791969425045 |
| }, |
| { |
| "epoch": 1.3202562049639712, |
| "grad_norm": 0.0035135592333972454, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 104, |
| "time_profile/curr_logprobs_and_update": 0.28596270937123336, |
| "train/gen_step": 6596.0, |
| "unified/clip_ratio/high_mean": 4.980298490409041e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 4.980298490409041e-05, |
| "unified/kl": 0.032854080345714465, |
| "unified/loss": 0.001322075961070368 |
| }, |
| { |
| "epoch": 1.33306645316253, |
| "gen/advantages_abs_mean": 0.0665295422077179, |
| "gen/advantages_max": 0.19884777069091797, |
| "gen/advantages_mean": 2.3283064365386963e-09, |
| "gen/advantages_min": -0.34449654817581177, |
| "gen/advantages_std": 0.08466022461652756, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.7172415256500244, |
| "gen/reward_std": 0.09059640765190125, |
| "grad_norm": 0.0032354136928915977, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/perceptual_score_reward_func/mean": 0.707398533821106, |
| "rewards/perceptual_score_reward_func/std": 0.0924367904663086, |
| "step": 105, |
| "time_profile/curr_logprobs_and_update": 0.2953199516341556, |
| "time_profile/image_rollout": 88.1711419057101, |
| "time_profile/old_logps": 35.69154427945614, |
| "time_profile/ref_logps": 35.82033765874803, |
| "time_profile/reward": 0.9439821243286133, |
| "train/gen_step": 6660.0, |
| "unified/clip_ratio/high_mean": 0.0004632073678294546, |
| "unified/clip_ratio/low_mean": 0.0002651929771673167, |
| "unified/clip_ratio/region_mean": 0.0007284003477252554, |
| "unified/kl": 0.0403441485541407, |
| "unified/loss": 0.0012512003850133624 |
| }, |
| { |
| "epoch": 1.3458767013610888, |
| "grad_norm": 0.003011377528309822, |
| "learning_rate": 1e-05, |
| "loss": 0.0065, |
| "step": 106, |
| "time_profile/curr_logprobs_and_update": 0.2940919303218834, |
| "train/gen_step": 6724.0, |
| "unified/clip_ratio/high_mean": 0.00030953719215176534, |
| "unified/clip_ratio/low_mean": 8.872071430232609e-05, |
| "unified/clip_ratio/region_mean": 0.00039825790736358613, |
| "unified/kl": 0.1624628723366186, |
| "unified/loss": 0.006346872902213363 |
| }, |
| { |
| "epoch": 1.3586869495596476, |
| "gen/advantages_abs_mean": 0.028656376525759697, |
| "gen/advantages_max": 0.07740437984466553, |
| "gen/advantages_mean": 4.889443516731262e-09, |
| "gen/advantages_min": -0.1487799882888794, |
| "gen/advantages_std": 0.03779109567403793, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 39.0, |
| "gen/prompt/mean_length": 36.5, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.7954732179641724, |
| "gen/reward_std": 0.042653292417526245, |
| "grad_norm": 0.0025170508306473494, |
| "learning_rate": 1e-05, |
| "loss": 0.0067, |
| "rewards/perceptual_score_reward_func/mean": 0.7931220531463623, |
| "rewards/perceptual_score_reward_func/std": 0.038766633719205856, |
| "step": 107, |
| "time_profile/curr_logprobs_and_update": 0.28722655613091774, |
| "time_profile/image_rollout": 85.07742638885975, |
| "time_profile/old_logps": 35.61971877142787, |
| "time_profile/ref_logps": 35.68408760428429, |
| "time_profile/reward": 1.3477066438645124, |
| "train/gen_step": 6788.0, |
| "unified/clip_ratio/high_mean": 0.0007559659843536792, |
| "unified/clip_ratio/low_mean": 0.00031796366874914384, |
| "unified/clip_ratio/region_mean": 0.0010739296567408019, |
| "unified/kl": 0.14353025154559873, |
| "unified/loss": 0.004765539683660336 |
| }, |
| { |
| "epoch": 1.3714971977582064, |
| "grad_norm": 0.00173460494261235, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 108, |
| "time_profile/curr_logprobs_and_update": 0.2885909783653915, |
| "train/gen_step": 6852.0, |
| "unified/clip_ratio/high_mean": 0.00010551881587161915, |
| "unified/clip_ratio/low_mean": 1.3563368156610522e-05, |
| "unified/clip_ratio/region_mean": 0.00011908218402822968, |
| "unified/kl": 0.033308936399407685, |
| "unified/loss": 0.001296318120239448 |
| }, |
| { |
| "epoch": 1.3843074459567655, |
| "gen/advantages_abs_mean": 0.0636034607887268, |
| "gen/advantages_max": 0.3713875412940979, |
| "gen/advantages_mean": 1.6298145055770874e-09, |
| "gen/advantages_min": -0.274211585521698, |
| "gen/advantages_std": 0.09082217514514923, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 32.0, |
| "gen/prompt/mean_length": 29.5, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.5426168441772461, |
| "gen/reward_std": 0.2932545244693756, |
| "grad_norm": 0.001911607920192182, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.5345706343650818, |
| "rewards/perceptual_score_reward_func/std": 0.29058483242988586, |
| "step": 109, |
| "time_profile/curr_logprobs_and_update": 0.28650623420253396, |
| "time_profile/image_rollout": 91.26650758460164, |
| "time_profile/old_logps": 35.63152730278671, |
| "time_profile/ref_logps": 35.694553695619106, |
| "time_profile/reward": 1.3631694037467241, |
| "train/gen_step": 6916.0, |
| "unified/clip_ratio/high_mean": 0.0006030555850884411, |
| "unified/clip_ratio/low_mean": 0.0001932560098794056, |
| "unified/clip_ratio/region_mean": 0.0007963115995153203, |
| "unified/kl": 0.032178554829442874, |
| "unified/loss": 0.0007431395524690743 |
| }, |
| { |
| "epoch": 1.3971176941553243, |
| "grad_norm": 0.002165607176721096, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 110, |
| "time_profile/curr_logprobs_and_update": 0.28561883803922683, |
| "train/gen_step": 6980.0, |
| "unified/clip_ratio/high_mean": 0.00024020071941777132, |
| "unified/clip_ratio/low_mean": 5.193096149014309e-05, |
| "unified/clip_ratio/region_mean": 0.0002921316809079144, |
| "unified/kl": 0.031306360120652243, |
| "unified/loss": 0.0011849405273096636 |
| }, |
| { |
| "epoch": 1.409927942353883, |
| "gen/advantages_abs_mean": 0.07240858674049377, |
| "gen/advantages_max": 0.26393023133277893, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.3260378837585449, |
| "gen/advantages_std": 0.0921124517917633, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 159.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.422931969165802, |
| "gen/reward_std": 0.16759854555130005, |
| "grad_norm": 0.0034576484467834234, |
| "learning_rate": 1e-05, |
| "loss": 0.0, |
| "rewards/perceptual_score_reward_func/mean": 0.4374734163284302, |
| "rewards/perceptual_score_reward_func/std": 0.15218466520309448, |
| "step": 111, |
| "time_profile/curr_logprobs_and_update": 0.2904707919806242, |
| "time_profile/image_rollout": 94.59738933853805, |
| "time_profile/old_logps": 35.6568603720516, |
| "time_profile/ref_logps": 35.68659848533571, |
| "time_profile/reward": 1.427611980587244, |
| "train/gen_step": 7044.0, |
| "unified/clip_ratio/high_mean": 0.0007604033617099049, |
| "unified/clip_ratio/low_mean": 0.00046580425623687916, |
| "unified/clip_ratio/region_mean": 0.0012262076288607204, |
| "unified/kl": 0.03141445969231427, |
| "unified/loss": -0.0003618052833189722 |
| }, |
| { |
| "epoch": 1.4227381905524419, |
| "grad_norm": 0.004003474954515696, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 112, |
| "time_profile/curr_logprobs_and_update": 0.2907797862426378, |
| "train/gen_step": 7108.0, |
| "unified/clip_ratio/high_mean": 0.00041981838876381516, |
| "unified/clip_ratio/low_mean": 0.00023284091003006324, |
| "unified/clip_ratio/region_mean": 0.0006526592924274155, |
| "unified/kl": 0.04061797269969247, |
| "unified/loss": 0.0016009146888791292 |
| }, |
| { |
| "epoch": 1.4355484387510007, |
| "gen/advantages_abs_mean": 0.03790595009922981, |
| "gen/advantages_max": 0.1838318109512329, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.1129482239484787, |
| "gen/advantages_std": 0.04889063909649849, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 30.0, |
| "gen/prompt/mean_length": 27.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.03915974497795105, |
| "gen/reward_std": 0.05308791622519493, |
| "grad_norm": 0.003365787910297513, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "rewards/perceptual_score_reward_func/mean": 0.04437080770730972, |
| "rewards/perceptual_score_reward_func/std": 0.055337294936180115, |
| "step": 113, |
| "time_profile/curr_logprobs_and_update": 0.2875677521515172, |
| "time_profile/image_rollout": 93.94608847610652, |
| "time_profile/old_logps": 35.64288152009249, |
| "time_profile/ref_logps": 35.68590772897005, |
| "time_profile/reward": 1.4800133537501097, |
| "train/gen_step": 7172.0, |
| "unified/clip_ratio/high_mean": 0.0013470338944898685, |
| "unified/clip_ratio/low_mean": 0.00042899202344415244, |
| "unified/clip_ratio/region_mean": 0.0017760259106580634, |
| "unified/kl": 0.03961936090490781, |
| "unified/loss": 0.0038841106043037144 |
| }, |
| { |
| "epoch": 1.4483586869495597, |
| "grad_norm": 0.0020599612034857273, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 114, |
| "time_profile/curr_logprobs_and_update": 0.29330829688115045, |
| "train/gen_step": 7236.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 1.4256386748456862e-05, |
| "unified/clip_ratio/region_mean": 1.4256386748456862e-05, |
| "unified/kl": 0.033574721310287714, |
| "unified/loss": 0.001462343894672813 |
| }, |
| { |
| "epoch": 1.4611689351481185, |
| "gen/advantages_abs_mean": 0.044344015419483185, |
| "gen/advantages_max": 0.2023053765296936, |
| "gen/advantages_mean": 1.1641532182693481e-09, |
| "gen/advantages_min": -0.2494000792503357, |
| "gen/advantages_std": 0.06300931423902512, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 33.0, |
| "gen/prompt/mean_length": 29.5, |
| "gen/prompt/min_length": 26.0, |
| "gen/reward": 0.643923819065094, |
| "gen/reward_std": 0.17081858217716217, |
| "grad_norm": 0.004796881694346666, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "rewards/perceptual_score_reward_func/mean": 0.6462793350219727, |
| "rewards/perceptual_score_reward_func/std": 0.174828439950943, |
| "step": 115, |
| "time_profile/curr_logprobs_and_update": 0.2918398874462582, |
| "time_profile/image_rollout": 90.91947788372636, |
| "time_profile/old_logps": 35.63139848783612, |
| "time_profile/ref_logps": 35.679666850715876, |
| "time_profile/reward": 1.5054155122488737, |
| "train/gen_step": 7300.0, |
| "unified/clip_ratio/high_mean": 5.238996800471796e-05, |
| "unified/clip_ratio/low_mean": 5.107643664814532e-05, |
| "unified/clip_ratio/region_mean": 0.00010346640465286328, |
| "unified/kl": 0.03278353181667626, |
| "unified/loss": 0.003379457935807295 |
| }, |
| { |
| "epoch": 1.4739791833466773, |
| "grad_norm": 0.0059644728899002075, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 116, |
| "time_profile/curr_logprobs_and_update": 0.2902962447842583, |
| "train/gen_step": 7364.0, |
| "unified/clip_ratio/high_mean": 0.00014891066621203208, |
| "unified/clip_ratio/low_mean": 0.00020414480513863964, |
| "unified/clip_ratio/region_mean": 0.0003530554713506717, |
| "unified/kl": 0.03125500594615005, |
| "unified/loss": 0.0011424217318563024 |
| }, |
| { |
| "epoch": 1.4867894315452361, |
| "gen/advantages_abs_mean": 0.056864481419324875, |
| "gen/advantages_max": 0.19812063872814178, |
| "gen/advantages_mean": -2.3283064365386963e-09, |
| "gen/advantages_min": -0.19689124822616577, |
| "gen/advantages_std": 0.06950084120035172, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 126.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.36290961503982544, |
| "gen/reward_std": 0.2883332371711731, |
| "grad_norm": 0.00461900420486927, |
| "learning_rate": 1e-05, |
| "loss": 0.0006, |
| "rewards/perceptual_score_reward_func/mean": 0.3551962971687317, |
| "rewards/perceptual_score_reward_func/std": 0.2728025019168854, |
| "step": 117, |
| "time_profile/curr_logprobs_and_update": 0.28732451432733797, |
| "time_profile/image_rollout": 100.65153944864869, |
| "time_profile/old_logps": 35.655188797041774, |
| "time_profile/ref_logps": 35.70040735602379, |
| "time_profile/reward": 1.2644598968327045, |
| "train/gen_step": 7428.0, |
| "unified/clip_ratio/high_mean": 0.00042217884856654564, |
| "unified/clip_ratio/low_mean": 0.0009703336363600101, |
| "unified/clip_ratio/region_mean": 0.0013925124894740293, |
| "unified/kl": 0.031373919220641255, |
| "unified/loss": -0.001355293581582373 |
| }, |
| { |
| "epoch": 1.499599679743795, |
| "grad_norm": 0.0028887134976685047, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "step": 118, |
| "time_profile/curr_logprobs_and_update": 0.28666515793884173, |
| "train/gen_step": 7492.0, |
| "unified/clip_ratio/high_mean": 9.078754828806268e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 9.078754828806268e-05, |
| "unified/kl": 0.04551572597119957, |
| "unified/loss": 0.0018532902213337366 |
| }, |
| { |
| "epoch": 1.512409927942354, |
| "gen/advantages_abs_mean": 0.0431256927549839, |
| "gen/advantages_max": 0.1255958080291748, |
| "gen/advantages_mean": 2.3283064365386963e-09, |
| "gen/advantages_min": -0.21674871444702148, |
| "gen/advantages_std": 0.05766208469867706, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.7449733018875122, |
| "gen/reward_std": 0.06443611532449722, |
| "grad_norm": 0.007539812941104174, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "rewards/perceptual_score_reward_func/mean": 0.7542487978935242, |
| "rewards/perceptual_score_reward_func/std": 0.061425261199474335, |
| "step": 119, |
| "time_profile/curr_logprobs_and_update": 0.2824957420816645, |
| "time_profile/image_rollout": 95.6516018435359, |
| "time_profile/old_logps": 35.68137123994529, |
| "time_profile/ref_logps": 35.80113796517253, |
| "time_profile/reward": 0.9460577331483364, |
| "train/gen_step": 7556.0, |
| "unified/clip_ratio/high_mean": 0.0005107632741783164, |
| "unified/clip_ratio/low_mean": 0.00025477437520748936, |
| "unified/clip_ratio/region_mean": 0.000765537648476311, |
| "unified/kl": 0.054741554835345596, |
| "unified/loss": 0.002122770885762293 |
| }, |
| { |
| "epoch": 1.5252201761409128, |
| "grad_norm": 0.002637062221765518, |
| "learning_rate": 1e-05, |
| "loss": 0.0053, |
| "step": 120, |
| "time_profile/curr_logprobs_and_update": 0.2882592770329211, |
| "train/gen_step": 7620.0, |
| "unified/clip_ratio/high_mean": 0.0011127292445962667, |
| "unified/clip_ratio/low_mean": 0.0010107289936058805, |
| "unified/clip_ratio/region_mean": 0.0021234582272882108, |
| "unified/kl": 0.13152603071648628, |
| "unified/loss": 0.005372763577724982 |
| }, |
| { |
| "epoch": 1.5380304243394716, |
| "gen/advantages_abs_mean": 0.05986716225743294, |
| "gen/advantages_max": 0.13977259397506714, |
| "gen/advantages_mean": -4.190951585769653e-09, |
| "gen/advantages_min": -0.3370237946510315, |
| "gen/advantages_std": 0.07795548439025879, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.7155458927154541, |
| "gen/reward_std": 0.09138889610767365, |
| "grad_norm": 0.002922995714470744, |
| "learning_rate": 1e-05, |
| "loss": 0.0055, |
| "rewards/perceptual_score_reward_func/mean": 0.7051100730895996, |
| "rewards/perceptual_score_reward_func/std": 0.1026398092508316, |
| "step": 121, |
| "time_profile/curr_logprobs_and_update": 0.2949224690091796, |
| "time_profile/image_rollout": 93.05562633834779, |
| "time_profile/old_logps": 35.69093172252178, |
| "time_profile/ref_logps": 35.76676655560732, |
| "time_profile/reward": 0.9614674616605043, |
| "train/gen_step": 7684.0, |
| "unified/clip_ratio/high_mean": 0.0037278791087373975, |
| "unified/clip_ratio/low_mean": 0.00411738019920449, |
| "unified/clip_ratio/region_mean": 0.007845259402529337, |
| "unified/kl": 0.13341835048049688, |
| "unified/loss": 0.0043207566068304 |
| }, |
| { |
| "epoch": 1.5508406725380304, |
| "grad_norm": 0.0042500728741288185, |
| "learning_rate": 1e-05, |
| "loss": 0.004, |
| "step": 122, |
| "time_profile/curr_logprobs_and_update": 0.2829219346458558, |
| "train/gen_step": 7748.0, |
| "unified/clip_ratio/high_mean": 0.0016792991782494937, |
| "unified/clip_ratio/low_mean": 0.0006439993148887879, |
| "unified/clip_ratio/region_mean": 0.0023232984931382816, |
| "unified/kl": 0.09978884411975741, |
| "unified/loss": 0.0038766780953665148 |
| }, |
| { |
| "epoch": 1.5636509207365892, |
| "gen/advantages_abs_mean": 0.05314895510673523, |
| "gen/advantages_max": 0.28923487663269043, |
| "gen/advantages_mean": -1.5133991837501526e-09, |
| "gen/advantages_min": -0.27263307571411133, |
| "gen/advantages_std": 0.07695531100034714, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 27.0, |
| "gen/prompt/mean_length": 25.5, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.5273290276527405, |
| "gen/reward_std": 0.3176587224006653, |
| "grad_norm": 0.005965746007859707, |
| "learning_rate": 1e-05, |
| "loss": 0.0063, |
| "rewards/perceptual_score_reward_func/mean": 0.528980553150177, |
| "rewards/perceptual_score_reward_func/std": 0.31866586208343506, |
| "step": 123, |
| "time_profile/curr_logprobs_and_update": 0.28925655310740694, |
| "time_profile/image_rollout": 85.26398288831115, |
| "time_profile/old_logps": 35.60682420618832, |
| "time_profile/ref_logps": 35.67962844111025, |
| "time_profile/reward": 1.2899844255298376, |
| "train/gen_step": 7812.0, |
| "unified/clip_ratio/high_mean": 0.006747730705683352, |
| "unified/clip_ratio/low_mean": 0.0036483843277892447, |
| "unified/clip_ratio/region_mean": 0.010396115059847943, |
| "unified/kl": 0.09209522936725989, |
| "unified/loss": 0.003992128267327644 |
| }, |
| { |
| "epoch": 1.5764611689351482, |
| "grad_norm": 0.006018125917762518, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 124, |
| "time_profile/curr_logprobs_and_update": 0.2918457875493914, |
| "train/gen_step": 7876.0, |
| "unified/clip_ratio/high_mean": 0.00030464172141364543, |
| "unified/clip_ratio/low_mean": 0.0022733521536792978, |
| "unified/clip_ratio/region_mean": 0.0025779938796404167, |
| "unified/kl": 0.06117875926429406, |
| "unified/loss": 0.0025565675814966937 |
| }, |
| { |
| "epoch": 1.589271417133707, |
| "gen/advantages_abs_mean": 0.061115965247154236, |
| "gen/advantages_max": 0.1915382742881775, |
| "gen/advantages_mean": -9.313225746154785e-10, |
| "gen/advantages_min": -0.31723618507385254, |
| "gen/advantages_std": 0.07902835309505463, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 185.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.5624587535858154, |
| "gen/reward_std": 0.09737126529216766, |
| "grad_norm": 0.01172794308513403, |
| "learning_rate": 1e-05, |
| "loss": 0.0009, |
| "rewards/perceptual_score_reward_func/mean": 0.5736739635467529, |
| "rewards/perceptual_score_reward_func/std": 0.07946990430355072, |
| "step": 125, |
| "time_profile/curr_logprobs_and_update": 0.2807448592793662, |
| "time_profile/image_rollout": 84.8983690943569, |
| "time_profile/old_logps": 35.744687812402844, |
| "time_profile/ref_logps": 35.80844911374152, |
| "time_profile/reward": 1.2122203204780817, |
| "train/gen_step": 7940.0, |
| "unified/clip_ratio/high_mean": 0.0011306394399070996, |
| "unified/clip_ratio/low_mean": 0.011417139139666688, |
| "unified/clip_ratio/region_mean": 0.01254777849317179, |
| "unified/kl": 0.07883448613574728, |
| "unified/loss": 0.0028802082304082433 |
| }, |
| { |
| "epoch": 1.6020816653322658, |
| "grad_norm": 0.00505675608292222, |
| "learning_rate": 1e-05, |
| "loss": 0.0041, |
| "step": 126, |
| "time_profile/curr_logprobs_and_update": 0.28650646412279457, |
| "train/gen_step": 8004.0, |
| "unified/clip_ratio/high_mean": 0.07535966531486338, |
| "unified/clip_ratio/low_mean": 0.0053497172903007595, |
| "unified/clip_ratio/region_mean": 0.08070938309538178, |
| "unified/kl": 0.08585448877420276, |
| "unified/loss": 0.004471837332857831 |
| }, |
| { |
| "epoch": 1.6148919135308246, |
| "gen/advantages_abs_mean": 0.07567422091960907, |
| "gen/advantages_max": 0.2743659019470215, |
| "gen/advantages_mean": -1.6298145055770874e-09, |
| "gen/advantages_min": -0.23675748705863953, |
| "gen/advantages_std": 0.09291587769985199, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 83.5, |
| "gen/prompt/min_length": 30.0, |
| "gen/reward": 0.264026939868927, |
| "gen/reward_std": 0.15199002623558044, |
| "grad_norm": 0.00418127654120326, |
| "learning_rate": 1e-05, |
| "loss": 0.0059, |
| "rewards/perceptual_score_reward_func/mean": 0.24559339880943298, |
| "rewards/perceptual_score_reward_func/std": 0.15700195729732513, |
| "step": 127, |
| "time_profile/curr_logprobs_and_update": 0.287341259769164, |
| "time_profile/image_rollout": 85.78850851021707, |
| "time_profile/old_logps": 35.69921772927046, |
| "time_profile/ref_logps": 35.74098035879433, |
| "time_profile/reward": 1.3971008583903313, |
| "train/gen_step": 8068.0, |
| "unified/clip_ratio/high_mean": 0.10724178397231299, |
| "unified/clip_ratio/low_mean": 0.015299416099878727, |
| "unified/clip_ratio/region_mean": 0.12254120000216062, |
| "unified/kl": 0.06688741553807631, |
| "unified/loss": 0.008423287796631485 |
| }, |
| { |
| "epoch": 1.6277021617293834, |
| "grad_norm": 0.004291560500860214, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 128, |
| "time_profile/curr_logprobs_and_update": 0.2904924996837508, |
| "train/gen_step": 8132.0, |
| "unified/clip_ratio/high_mean": 3.791005929087987e-05, |
| "unified/clip_ratio/low_mean": 2.5746619030542206e-05, |
| "unified/clip_ratio/region_mean": 6.365667832142208e-05, |
| "unified/kl": 0.04011628415901214, |
| "unified/loss": 0.0016968616482699872 |
| }, |
| { |
| "epoch": 1.6405124099279424, |
| "gen/advantages_abs_mean": 0.07780550420284271, |
| "gen/advantages_max": 0.33822619915008545, |
| "gen/advantages_mean": 2.0372681319713593e-09, |
| "gen/advantages_min": -0.32445240020751953, |
| "gen/advantages_std": 0.10094477981328964, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 91.5, |
| "gen/prompt/min_length": 40.0, |
| "gen/reward": 0.38963890075683594, |
| "gen/reward_std": 0.27400505542755127, |
| "grad_norm": 0.0064746844582259655, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/perceptual_score_reward_func/mean": 0.39440298080444336, |
| "rewards/perceptual_score_reward_func/std": 0.2802719473838806, |
| "step": 129, |
| "time_profile/curr_logprobs_and_update": 0.29365084203891456, |
| "time_profile/image_rollout": 92.23527741245925, |
| "time_profile/old_logps": 35.69436052814126, |
| "time_profile/ref_logps": 35.73389621451497, |
| "time_profile/reward": 1.062905428931117, |
| "train/gen_step": 8196.0, |
| "unified/clip_ratio/high_mean": 0.0001702761464912328, |
| "unified/clip_ratio/low_mean": 0.0010935042027995223, |
| "unified/clip_ratio/region_mean": 0.0012637803538382286, |
| "unified/kl": 0.0411043684289325, |
| "unified/loss": 0.004106405973288929 |
| }, |
| { |
| "epoch": 1.6533226581265013, |
| "grad_norm": 0.006903781555593014, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "step": 130, |
| "time_profile/curr_logprobs_and_update": 0.28863042485318147, |
| "train/gen_step": 8260.0, |
| "unified/clip_ratio/high_mean": 0.01118387773840368, |
| "unified/clip_ratio/low_mean": 0.00042925817706418457, |
| "unified/clip_ratio/region_mean": 0.011613136058258533, |
| "unified/kl": 0.08873327681794763, |
| "unified/loss": 0.0033595796521694865 |
| }, |
| { |
| "epoch": 1.66613290632506, |
| "gen/advantages_abs_mean": 0.02932814322412014, |
| "gen/advantages_max": 0.15440410375595093, |
| "gen/advantages_mean": 1.1641532182693481e-09, |
| "gen/advantages_min": -0.2082865834236145, |
| "gen/advantages_std": 0.05175040289759636, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.5, |
| "gen/prompt/max_length": 36.0, |
| "gen/prompt/mean_length": 35.0, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.35747069120407104, |
| "gen/reward_std": 0.36191922426223755, |
| "grad_norm": 0.00403651362285018, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "rewards/perceptual_score_reward_func/mean": 0.35938212275505066, |
| "rewards/perceptual_score_reward_func/std": 0.36509737372398376, |
| "step": 131, |
| "time_profile/curr_logprobs_and_update": 0.2961919621739071, |
| "time_profile/image_rollout": 96.8908116389066, |
| "time_profile/old_logps": 35.64496449753642, |
| "time_profile/ref_logps": 35.696318335831165, |
| "time_profile/reward": 1.4984830934554338, |
| "train/gen_step": 8324.0, |
| "unified/clip_ratio/high_mean": 0.03772879852567712, |
| "unified/clip_ratio/low_mean": 0.003505356321511499, |
| "unified/clip_ratio/region_mean": 0.041234154683479574, |
| "unified/kl": 0.09730945219052956, |
| "unified/loss": 0.0015052031849336345 |
| }, |
| { |
| "epoch": 1.6789431545236189, |
| "grad_norm": 0.0016463575884699821, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "step": 132, |
| "time_profile/curr_logprobs_and_update": 0.2821661222842522, |
| "train/gen_step": 8388.0, |
| "unified/clip_ratio/high_mean": 0.0008666838857607218, |
| "unified/clip_ratio/low_mean": 1.3754401152255014e-05, |
| "unified/clip_ratio/region_mean": 0.0008804382905509556, |
| "unified/kl": 0.042414410738274455, |
| "unified/loss": 0.0016847542519826675 |
| }, |
| { |
| "epoch": 1.6917534027221777, |
| "gen/advantages_abs_mean": 0.05056470260024071, |
| "gen/advantages_max": 0.18678975105285645, |
| "gen/advantages_mean": 2.0954757928848267e-09, |
| "gen/advantages_min": -0.365351140499115, |
| "gen/advantages_std": 0.06941158324480057, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.7006023526191711, |
| "gen/reward_std": 0.07870075106620789, |
| "grad_norm": 0.00302127399481833, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "rewards/perceptual_score_reward_func/mean": 0.715668261051178, |
| "rewards/perceptual_score_reward_func/std": 0.05759961158037186, |
| "step": 133, |
| "time_profile/curr_logprobs_and_update": 0.29358167183818296, |
| "time_profile/image_rollout": 89.95654336549342, |
| "time_profile/old_logps": 35.73023318499327, |
| "time_profile/ref_logps": 35.7922641094774, |
| "time_profile/reward": 0.9506809506565332, |
| "train/gen_step": 8452.0, |
| "unified/clip_ratio/high_mean": 0.003891350916092051, |
| "unified/clip_ratio/low_mean": 7.476141399820335e-05, |
| "unified/clip_ratio/region_mean": 0.003966112333728233, |
| "unified/kl": 0.053177921363385394, |
| "unified/loss": 0.000371901216567494 |
| }, |
| { |
| "epoch": 1.7045636509207367, |
| "grad_norm": 0.0037558316253125668, |
| "learning_rate": 1e-05, |
| "loss": 0.0091, |
| "step": 134, |
| "time_profile/curr_logprobs_and_update": 0.29027548429439776, |
| "train/gen_step": 8516.0, |
| "unified/clip_ratio/high_mean": 0.0017724815052133636, |
| "unified/clip_ratio/low_mean": 3.741087857633829e-05, |
| "unified/clip_ratio/region_mean": 0.001809892386518186, |
| "unified/kl": 0.22769839805550873, |
| "unified/loss": 0.009044620092936384 |
| }, |
| { |
| "epoch": 1.7173738991192953, |
| "gen/advantages_abs_mean": 0.06249502673745155, |
| "gen/advantages_max": 0.18479061126708984, |
| "gen/advantages_mean": -1.862645149230957e-09, |
| "gen/advantages_min": -0.28166067600250244, |
| "gen/advantages_std": 0.0770028680562973, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 215.5, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.4174192249774933, |
| "gen/reward_std": 0.23804545402526855, |
| "grad_norm": 0.003767220536246896, |
| "learning_rate": 1e-05, |
| "loss": 0.0071, |
| "rewards/perceptual_score_reward_func/mean": 0.41816747188568115, |
| "rewards/perceptual_score_reward_func/std": 0.2535645663738251, |
| "step": 135, |
| "time_profile/curr_logprobs_and_update": 0.28429412076366134, |
| "time_profile/image_rollout": 102.21069337241352, |
| "time_profile/old_logps": 35.70307997614145, |
| "time_profile/ref_logps": 35.79909221455455, |
| "time_profile/reward": 1.4969095662236214, |
| "train/gen_step": 8580.0, |
| "unified/clip_ratio/high_mean": 0.0030967974716986646, |
| "unified/clip_ratio/low_mean": 0.0007302079957298702, |
| "unified/clip_ratio/region_mean": 0.003827005463790556, |
| "unified/kl": 0.21062234381679446, |
| "unified/loss": 0.010004522626331891 |
| }, |
| { |
| "epoch": 1.7301841473178543, |
| "grad_norm": 0.004301386885344982, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "step": 136, |
| "time_profile/curr_logprobs_and_update": 0.29434079653583467, |
| "train/gen_step": 8644.0, |
| "unified/clip_ratio/high_mean": 0.0004625109258995508, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0004625109258995508, |
| "unified/kl": 0.05430893413722515, |
| "unified/loss": 0.0022077317435105215 |
| }, |
| { |
| "epoch": 1.742994395516413, |
| "gen/advantages_abs_mean": 0.06403308361768723, |
| "gen/advantages_max": 0.33869343996047974, |
| "gen/advantages_mean": 2.0954757928848267e-09, |
| "gen/advantages_min": -0.20772579312324524, |
| "gen/advantages_std": 0.08191482722759247, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.40738505125045776, |
| "gen/reward_std": 0.30709385871887207, |
| "grad_norm": 0.004496556706726551, |
| "learning_rate": 1e-05, |
| "loss": 0.0056, |
| "rewards/perceptual_score_reward_func/mean": 0.4147990345954895, |
| "rewards/perceptual_score_reward_func/std": 0.3243771195411682, |
| "step": 137, |
| "time_profile/curr_logprobs_and_update": 0.2963087991520297, |
| "time_profile/image_rollout": 91.59672148153186, |
| "time_profile/old_logps": 35.74044441990554, |
| "time_profile/ref_logps": 35.80338528752327, |
| "time_profile/reward": 2.680176628753543, |
| "train/gen_step": 8708.0, |
| "unified/clip_ratio/high_mean": 0.014924995422916254, |
| "unified/clip_ratio/low_mean": 0.0006889987589602242, |
| "unified/clip_ratio/region_mean": 0.01561399412821629, |
| "unified/kl": 0.05471922585275024, |
| "unified/loss": 0.001671851314313244 |
| }, |
| { |
| "epoch": 1.755804643714972, |
| "grad_norm": 0.006244179792702198, |
| "learning_rate": 1e-05, |
| "loss": 0.004, |
| "step": 138, |
| "time_profile/curr_logprobs_and_update": 0.29202507858281024, |
| "train/gen_step": 8772.0, |
| "unified/clip_ratio/high_mean": 0.000596910118474625, |
| "unified/clip_ratio/low_mean": 0.0009496516950093792, |
| "unified/clip_ratio/region_mean": 0.0015465618116650148, |
| "unified/kl": 0.09889116248814389, |
| "unified/loss": 0.003862968309476855 |
| }, |
| { |
| "epoch": 1.768614891913531, |
| "gen/advantages_abs_mean": 0.06389250606298447, |
| "gen/advantages_max": 0.23165170848369598, |
| "gen/advantages_mean": 4.656612873077393e-10, |
| "gen/advantages_min": -0.26901906728744507, |
| "gen/advantages_std": 0.07976555079221725, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 137.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.18621030449867249, |
| "gen/reward_std": 0.11706754565238953, |
| "grad_norm": 0.006589157041162252, |
| "learning_rate": 1e-05, |
| "loss": 0.001, |
| "rewards/perceptual_score_reward_func/mean": 0.18768814206123352, |
| "rewards/perceptual_score_reward_func/std": 0.1021382138133049, |
| "step": 139, |
| "time_profile/curr_logprobs_and_update": 0.2866535442008171, |
| "time_profile/image_rollout": 93.9379496704787, |
| "time_profile/old_logps": 35.6809767074883, |
| "time_profile/ref_logps": 35.78240888006985, |
| "time_profile/reward": 5.514092801138759, |
| "train/gen_step": 8836.0, |
| "unified/clip_ratio/high_mean": 0.001998860437197436, |
| "unified/clip_ratio/low_mean": 0.0026950046894853585, |
| "unified/clip_ratio/region_mean": 0.0046938651212258264, |
| "unified/kl": 0.09669042826863006, |
| "unified/loss": 0.0023558811117254663 |
| }, |
| { |
| "epoch": 1.7814251401120895, |
| "grad_norm": 0.0057668304070830345, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "step": 140, |
| "time_profile/curr_logprobs_and_update": 0.2863264895277098, |
| "train/gen_step": 8900.0, |
| "unified/clip_ratio/high_mean": 0.00048513619458390167, |
| "unified/clip_ratio/low_mean": 2.7048740776081104e-05, |
| "unified/clip_ratio/region_mean": 0.0005121849326314987, |
| "unified/kl": 0.03493243327829987, |
| "unified/loss": 0.001315825039455376 |
| }, |
| { |
| "epoch": 1.7942353883106485, |
| "gen/advantages_abs_mean": 0.048334699124097824, |
| "gen/advantages_max": 0.19973474740982056, |
| "gen/advantages_mean": -1.6298145055770874e-09, |
| "gen/advantages_min": -0.2896121144294739, |
| "gen/advantages_std": 0.0633920356631279, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.35955485701560974, |
| "gen/reward_std": 0.25457412004470825, |
| "grad_norm": 0.00502097187563777, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "rewards/perceptual_score_reward_func/mean": 0.3785892724990845, |
| "rewards/perceptual_score_reward_func/std": 0.2508572041988373, |
| "step": 141, |
| "time_profile/curr_logprobs_and_update": 0.29334245243808255, |
| "time_profile/image_rollout": 94.66671554744244, |
| "time_profile/old_logps": 35.725636603310704, |
| "time_profile/ref_logps": 35.787481896579266, |
| "time_profile/reward": 1.2374453376978636, |
| "train/gen_step": 8964.0, |
| "unified/clip_ratio/high_mean": 0.008519951419657446, |
| "unified/clip_ratio/low_mean": 0.00013759005014435388, |
| "unified/clip_ratio/region_mean": 0.008657541471620789, |
| "unified/kl": 0.034399580414174125, |
| "unified/loss": 0.00496306180002648 |
| }, |
| { |
| "epoch": 1.8070456365092074, |
| "grad_norm": 0.0034979088231921196, |
| "learning_rate": 1e-05, |
| "loss": 0.0062, |
| "step": 142, |
| "time_profile/curr_logprobs_and_update": 0.28757468741969205, |
| "train/gen_step": 9028.0, |
| "unified/clip_ratio/high_mean": 0.0002629072369018104, |
| "unified/clip_ratio/low_mean": 2.528555523895193e-05, |
| "unified/clip_ratio/region_mean": 0.000288192793050257, |
| "unified/kl": 0.1550389884505421, |
| "unified/loss": 0.006169562449940713 |
| }, |
| { |
| "epoch": 1.8198558847077662, |
| "gen/advantages_abs_mean": 0.0765346884727478, |
| "gen/advantages_max": 0.26568108797073364, |
| "gen/advantages_mean": -1.3969838619232178e-09, |
| "gen/advantages_min": -0.24493470788002014, |
| "gen/advantages_std": 0.09568614512681961, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 126.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.2732863426208496, |
| "gen/reward_std": 0.1075901985168457, |
| "grad_norm": 0.003704902483150363, |
| "learning_rate": 1e-05, |
| "loss": 0.006, |
| "rewards/perceptual_score_reward_func/mean": 0.2819960117340088, |
| "rewards/perceptual_score_reward_func/std": 0.10969275236129761, |
| "step": 143, |
| "time_profile/curr_logprobs_and_update": 0.29061931784963235, |
| "time_profile/image_rollout": 102.85954966209829, |
| "time_profile/old_logps": 35.6852895244956, |
| "time_profile/ref_logps": 35.699950790032744, |
| "time_profile/reward": 5.994517290964723, |
| "train/gen_step": 9092.0, |
| "unified/clip_ratio/high_mean": 0.0024094630516628968, |
| "unified/clip_ratio/low_mean": 0.00028702112922474043, |
| "unified/clip_ratio/region_mean": 0.0026964841772496584, |
| "unified/kl": 0.1615976605389733, |
| "unified/loss": 0.0051123992066095525 |
| }, |
| { |
| "epoch": 1.8326661329063252, |
| "grad_norm": 0.00441337563097477, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 144, |
| "time_profile/curr_logprobs_and_update": 0.29150564002338797, |
| "train/gen_step": 9156.0, |
| "unified/clip_ratio/high_mean": 5.041367967351107e-05, |
| "unified/clip_ratio/low_mean": 1.2934602636960335e-05, |
| "unified/clip_ratio/region_mean": 6.33482823104714e-05, |
| "unified/kl": 0.03936847756267525, |
| "unified/loss": 0.0016968364725471474 |
| }, |
| { |
| "epoch": 1.8454763811048838, |
| "gen/advantages_abs_mean": 0.05427496135234833, |
| "gen/advantages_max": 0.16081571578979492, |
| "gen/advantages_mean": -1.3969838619232178e-09, |
| "gen/advantages_min": -0.27252915501594543, |
| "gen/advantages_std": 0.07097128033638, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 140.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.6301897764205933, |
| "gen/reward_std": 0.10973169654607773, |
| "grad_norm": 0.005718636326491833, |
| "learning_rate": 1e-05, |
| "loss": 0.0008, |
| "rewards/perceptual_score_reward_func/mean": 0.6228070259094238, |
| "rewards/perceptual_score_reward_func/std": 0.11832652240991592, |
| "step": 145, |
| "time_profile/curr_logprobs_and_update": 0.29172540063154884, |
| "time_profile/image_rollout": 95.14329688437283, |
| "time_profile/old_logps": 35.679359233006835, |
| "time_profile/ref_logps": 35.740789292380214, |
| "time_profile/reward": 1.1167409159243107, |
| "train/gen_step": 9220.0, |
| "unified/clip_ratio/high_mean": 0.000587068720960815, |
| "unified/clip_ratio/low_mean": 0.000173021161572251, |
| "unified/clip_ratio/region_mean": 0.0007600898861710448, |
| "unified/kl": 0.04188113062991761, |
| "unified/loss": -0.0002558948690420948 |
| }, |
| { |
| "epoch": 1.8582866293034428, |
| "grad_norm": 0.005676935892552137, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "step": 146, |
| "time_profile/curr_logprobs_and_update": 0.28573712857905775, |
| "train/gen_step": 9284.0, |
| "unified/clip_ratio/high_mean": 0.0002970744308186113, |
| "unified/clip_ratio/low_mean": 8.519378025084734e-05, |
| "unified/clip_ratio/region_mean": 0.00038226821015996393, |
| "unified/kl": 0.09014151419978589, |
| "unified/loss": 0.003581891072826693 |
| }, |
| { |
| "epoch": 1.8710968775020016, |
| "gen/advantages_abs_mean": 0.051070600748062134, |
| "gen/advantages_max": 0.13527166843414307, |
| "gen/advantages_mean": -5.122274160385132e-09, |
| "gen/advantages_min": -0.3856954574584961, |
| "gen/advantages_std": 0.06852003186941147, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.7004803419113159, |
| "gen/reward_std": 0.07513634115457535, |
| "grad_norm": 0.00409558555111289, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "rewards/perceptual_score_reward_func/mean": 0.6796281337738037, |
| "rewards/perceptual_score_reward_func/std": 0.08104143291711807, |
| "step": 147, |
| "time_profile/curr_logprobs_and_update": 0.285855452704709, |
| "time_profile/image_rollout": 85.7736628819257, |
| "time_profile/old_logps": 35.67286063730717, |
| "time_profile/ref_logps": 35.779720950871706, |
| "time_profile/reward": 1.0234551653265953, |
| "train/gen_step": 9348.0, |
| "unified/clip_ratio/high_mean": 0.0008755821590966661, |
| "unified/clip_ratio/low_mean": 0.0004622845581252477, |
| "unified/clip_ratio/region_mean": 0.0013378667035794933, |
| "unified/kl": 0.09117380547104403, |
| "unified/loss": 0.002188693568314193 |
| }, |
| { |
| "epoch": 1.8839071257005604, |
| "grad_norm": 0.0033700531348586082, |
| "learning_rate": 1e-05, |
| "loss": 0.0044, |
| "step": 148, |
| "time_profile/curr_logprobs_and_update": 0.2985293152742088, |
| "train/gen_step": 9412.0, |
| "unified/clip_ratio/high_mean": 0.002091699916491052, |
| "unified/clip_ratio/low_mean": 6.537473291245988e-05, |
| "unified/clip_ratio/region_mean": 0.002157074643037049, |
| "unified/kl": 0.11110673216171563, |
| "unified/loss": 0.004461315405933419 |
| }, |
| { |
| "epoch": 1.8967173738991194, |
| "gen/advantages_abs_mean": 0.06669476628303528, |
| "gen/advantages_max": 0.29541581869125366, |
| "gen/advantages_mean": -1.1932570487260818e-09, |
| "gen/advantages_min": -0.31305474042892456, |
| "gen/advantages_std": 0.08556267619132996, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 86.0, |
| "gen/prompt/min_length": 35.0, |
| "gen/reward": 0.442577600479126, |
| "gen/reward_std": 0.3167572021484375, |
| "grad_norm": 0.0039604133926332, |
| "learning_rate": 1e-05, |
| "loss": 0.0039, |
| "rewards/perceptual_score_reward_func/mean": 0.41718146204948425, |
| "rewards/perceptual_score_reward_func/std": 0.31259584426879883, |
| "step": 149, |
| "time_profile/curr_logprobs_and_update": 0.291258782352088, |
| "time_profile/image_rollout": 85.42934694699943, |
| "time_profile/old_logps": 35.6700346339494, |
| "time_profile/ref_logps": 35.718164034187794, |
| "time_profile/reward": 3.843767935410142, |
| "train/gen_step": 9476.0, |
| "unified/clip_ratio/high_mean": 0.005506029001480783, |
| "unified/clip_ratio/low_mean": 0.0008064525482041063, |
| "unified/clip_ratio/region_mean": 0.0063124815624178154, |
| "unified/kl": 0.10582851071376354, |
| "unified/loss": 0.005097084878798341 |
| }, |
| { |
| "epoch": 1.909527622097678, |
| "grad_norm": 0.0031276314985007048, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "step": 150, |
| "time_profile/curr_logprobs_and_update": 0.2896409893874079, |
| "train/gen_step": 9540.0, |
| "unified/clip_ratio/high_mean": 7.758394167467486e-05, |
| "unified/clip_ratio/low_mean": 4.173634351900546e-05, |
| "unified/clip_ratio/region_mean": 0.00011932028519368032, |
| "unified/kl": 0.04871381289558485, |
| "unified/loss": 0.002004596136885084 |
| }, |
| { |
| "epoch": 1.922337870296237, |
| "gen/advantages_abs_mean": 0.06253403425216675, |
| "gen/advantages_max": 0.22776862978935242, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.24407082796096802, |
| "gen/advantages_std": 0.07795893400907516, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 80.5, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.2212839424610138, |
| "gen/reward_std": 0.10819225758314133, |
| "grad_norm": 0.0028782610315829515, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "rewards/perceptual_score_reward_func/mean": 0.20518934726715088, |
| "rewards/perceptual_score_reward_func/std": 0.11293523758649826, |
| "step": 151, |
| "time_profile/curr_logprobs_and_update": 0.2882380739611108, |
| "time_profile/image_rollout": 94.66589397005737, |
| "time_profile/old_logps": 35.84556386433542, |
| "time_profile/ref_logps": 35.70588610507548, |
| "time_profile/reward": 1.3458904176950455, |
| "train/gen_step": 9604.0, |
| "unified/clip_ratio/high_mean": 0.0004462642273210804, |
| "unified/clip_ratio/low_mean": 0.0003407068516025902, |
| "unified/clip_ratio/region_mean": 0.0007869710789236706, |
| "unified/kl": 0.04785197728779167, |
| "unified/loss": 0.005950479607236048 |
| }, |
| { |
| "epoch": 1.9351481184947958, |
| "grad_norm": 0.003339177230373025, |
| "learning_rate": 1e-05, |
| "loss": 0.0014, |
| "step": 152, |
| "time_profile/curr_logprobs_and_update": 0.28296438709367067, |
| "train/gen_step": 9668.0, |
| "unified/clip_ratio/high_mean": 2.6393581720185466e-05, |
| "unified/clip_ratio/low_mean": 5.193366814637557e-05, |
| "unified/clip_ratio/region_mean": 7.832724986656103e-05, |
| "unified/kl": 0.03657471065525897, |
| "unified/loss": 0.0014128756847640034 |
| }, |
| { |
| "epoch": 1.9479583666933546, |
| "gen/advantages_abs_mean": 0.027933739125728607, |
| "gen/advantages_max": 0.09872519969940186, |
| "gen/advantages_mean": -3.4924596548080444e-09, |
| "gen/advantages_min": -0.21751081943511963, |
| "gen/advantages_std": 0.039107732474803925, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 92.5, |
| "gen/prompt/min_length": 42.0, |
| "gen/reward": 0.780127227306366, |
| "gen/reward_std": 0.0778065100312233, |
| "grad_norm": 0.0027934254612773657, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "rewards/perceptual_score_reward_func/mean": 0.7786700129508972, |
| "rewards/perceptual_score_reward_func/std": 0.07780160754919052, |
| "step": 153, |
| "time_profile/curr_logprobs_and_update": 0.28734079509740695, |
| "time_profile/image_rollout": 94.81195741146803, |
| "time_profile/old_logps": 35.69596145488322, |
| "time_profile/ref_logps": 35.74558510072529, |
| "time_profile/reward": 1.2448325883597136, |
| "train/gen_step": 9732.0, |
| "unified/clip_ratio/high_mean": 0.00020744220455526374, |
| "unified/clip_ratio/low_mean": 0.0002956635626105708, |
| "unified/clip_ratio/region_mean": 0.0005031057635278557, |
| "unified/kl": 0.040840220666723326, |
| "unified/loss": 0.000912024426725111 |
| }, |
| { |
| "epoch": 1.9607686148919137, |
| "grad_norm": 0.0028558243066072464, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "step": 154, |
| "time_profile/curr_logprobs_and_update": 0.29257669125217944, |
| "train/gen_step": 9796.0, |
| "unified/clip_ratio/high_mean": 0.0008039099420784623, |
| "unified/clip_ratio/low_mean": 0.0016663962333041127, |
| "unified/clip_ratio/region_mean": 0.002470306179020554, |
| "unified/kl": 0.0655196302686818, |
| "unified/loss": 0.0026364892204355783 |
| }, |
| { |
| "epoch": 1.9735788630904723, |
| "gen/advantages_abs_mean": 0.06897905468940735, |
| "gen/advantages_max": 0.25565266609191895, |
| "gen/advantages_mean": -3.6670826375484467e-09, |
| "gen/advantages_min": -0.2908056974411011, |
| "gen/advantages_std": 0.08907780051231384, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 84.0, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.5265546441078186, |
| "gen/reward_std": 0.23570306599140167, |
| "grad_norm": 0.002455601003021002, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/perceptual_score_reward_func/mean": 0.5150833129882812, |
| "rewards/perceptual_score_reward_func/std": 0.25435975193977356, |
| "step": 155, |
| "time_profile/curr_logprobs_and_update": 0.28119892042013817, |
| "time_profile/image_rollout": 83.67556969821453, |
| "time_profile/old_logps": 35.684941904619336, |
| "time_profile/ref_logps": 35.68470383249223, |
| "time_profile/reward": 1.1146302074193954, |
| "train/gen_step": 9860.0, |
| "unified/clip_ratio/high_mean": 0.0033073803624574794, |
| "unified/clip_ratio/low_mean": 0.0065782893370851525, |
| "unified/clip_ratio/region_mean": 0.009885669787763618, |
| "unified/kl": 0.06662523333216086, |
| "unified/loss": -0.000262459849636798 |
| }, |
| { |
| "epoch": 1.9863891112890313, |
| "grad_norm": 0.0022808618377894163, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "step": 156, |
| "time_profile/curr_logprobs_and_update": 0.2870417326630559, |
| "train/gen_step": 9924.0, |
| "unified/clip_ratio/high_mean": 0.00015197796437860234, |
| "unified/clip_ratio/low_mean": 0.00020154173307673773, |
| "unified/clip_ratio/region_mean": 0.00035351969836483477, |
| "unified/kl": 0.07948597773793153, |
| "unified/loss": 0.00320796342975882 |
| }, |
| { |
| "epoch": 1.99919935948759, |
| "gen/advantages_abs_mean": 0.04813893511891365, |
| "gen/advantages_max": 0.19551599025726318, |
| "gen/advantages_mean": 4.656612873077393e-10, |
| "gen/advantages_min": -0.24950778484344482, |
| "gen/advantages_std": 0.06611930578947067, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.6326351761817932, |
| "gen/reward_std": 0.12095728516578674, |
| "grad_norm": 0.0022853959817439318, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "rewards/perceptual_score_reward_func/mean": 0.6287431716918945, |
| "rewards/perceptual_score_reward_func/std": 0.1246950700879097, |
| "step": 157, |
| "time_profile/curr_logprobs_and_update": 0.2884384596545715, |
| "time_profile/image_rollout": 99.05054134689271, |
| "time_profile/old_logps": 35.70570847764611, |
| "time_profile/ref_logps": 35.81650428660214, |
| "time_profile/reward": 1.1606343444436789, |
| "train/gen_step": 9988.0, |
| "unified/clip_ratio/high_mean": 0.0007348101480602054, |
| "unified/clip_ratio/low_mean": 0.0016179103122340166, |
| "unified/clip_ratio/region_mean": 0.002352720430280897, |
| "unified/kl": 0.07506306233699434, |
| "unified/loss": 0.0051671818937393255 |
| }, |
| { |
| "epoch": 2.0, |
| "grad_norm": 0.0020002492237836123, |
| "learning_rate": 1e-05, |
| "loss": 0.0003, |
| "step": 158, |
| "time_profile/curr_logprobs_and_update": 0.2812675591558218, |
| "train/gen_step": 9992.0, |
| "unified/clip_ratio/high_mean": 0.001988460251595825, |
| "unified/clip_ratio/low_mean": 0.003396499145310372, |
| "unified/clip_ratio/region_mean": 0.005384959629736841, |
| "unified/kl": 0.07596240937709808, |
| "unified/loss": -0.0008202246390283108 |
| }, |
| { |
| "epoch": 2.012810248198559, |
| "grad_norm": 0.0026500935200601816, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "step": 159, |
| "time_profile/curr_logprobs_and_update": 0.28652132936986163, |
| "train/gen_step": 10056.0, |
| "unified/clip_ratio/high_mean": 0.005975998764370161, |
| "unified/clip_ratio/low_mean": 0.008422538764534693, |
| "unified/clip_ratio/region_mean": 0.014398537565284641, |
| "unified/kl": 0.06567471823655069, |
| "unified/loss": 0.002683598070007065 |
| }, |
| { |
| "epoch": 2.0256204963971176, |
| "gen/advantages_abs_mean": 0.056865792721509933, |
| "gen/advantages_max": 0.25363510847091675, |
| "gen/advantages_mean": -4.656612873077393e-10, |
| "gen/advantages_min": -0.25625455379486084, |
| "gen/advantages_std": 0.07274879515171051, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 84.5, |
| "gen/prompt/min_length": 26.0, |
| "gen/reward": 0.41545069217681885, |
| "gen/reward_std": 0.32199838757514954, |
| "grad_norm": 0.00292989076115191, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "rewards/perceptual_score_reward_func/mean": 0.4275937080383301, |
| "rewards/perceptual_score_reward_func/std": 0.3249584436416626, |
| "step": 160, |
| "time_profile/curr_logprobs_and_update": 0.29021278861910105, |
| "time_profile/image_rollout": 90.17475041560829, |
| "time_profile/old_logps": 35.65966880507767, |
| "time_profile/ref_logps": 35.711894784122705, |
| "time_profile/reward": 1.1305186413228512, |
| "train/gen_step": 10120.0, |
| "unified/clip_ratio/high_mean": 0.008071905474025698, |
| "unified/clip_ratio/low_mean": 0.010683667308512668, |
| "unified/clip_ratio/region_mean": 0.01875557277980988, |
| "unified/kl": 0.08813725365325809, |
| "unified/loss": 0.0036637967293700058 |
| }, |
| { |
| "epoch": 2.0384307445956766, |
| "grad_norm": 0.005531059578061104, |
| "learning_rate": 1e-05, |
| "loss": 0.0061, |
| "step": 161, |
| "time_profile/curr_logprobs_and_update": 0.29515198740409687, |
| "train/gen_step": 10184.0, |
| "unified/clip_ratio/high_mean": 0.00033335702755721286, |
| "unified/clip_ratio/low_mean": 0.00017928345914697275, |
| "unified/clip_ratio/region_mean": 0.0005126404903421644, |
| "unified/kl": 0.1533038376364857, |
| "unified/loss": 0.005756833081250079 |
| }, |
| { |
| "epoch": 2.051240992794235, |
| "gen/advantages_abs_mean": 0.03855127468705177, |
| "gen/advantages_max": 0.1778925061225891, |
| "gen/advantages_mean": -3.026798367500305e-09, |
| "gen/advantages_min": -0.22928708791732788, |
| "gen/advantages_std": 0.05448167398571968, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 86.5, |
| "gen/prompt/min_length": 36.0, |
| "gen/reward": 0.7568866014480591, |
| "gen/reward_std": 0.10604741424322128, |
| "grad_norm": 0.0052727325819432735, |
| "learning_rate": 1e-05, |
| "loss": 0.0048, |
| "rewards/perceptual_score_reward_func/mean": 0.7629588842391968, |
| "rewards/perceptual_score_reward_func/std": 0.11004883050918579, |
| "step": 162, |
| "time_profile/curr_logprobs_and_update": 0.28484814503462985, |
| "time_profile/image_rollout": 88.95044229365885, |
| "time_profile/old_logps": 35.69716415554285, |
| "time_profile/ref_logps": 35.729007912799716, |
| "time_profile/reward": 1.3262334875762463, |
| "train/gen_step": 10248.0, |
| "unified/clip_ratio/high_mean": 0.004101160412574245, |
| "unified/clip_ratio/low_mean": 0.003542088237736607, |
| "unified/clip_ratio/region_mean": 0.007643248653948831, |
| "unified/kl": 0.11847320757806301, |
| "unified/loss": 0.004571665815092274 |
| }, |
| { |
| "epoch": 2.0640512409927942, |
| "grad_norm": 0.003853349946439266, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "step": 163, |
| "time_profile/curr_logprobs_and_update": 0.29358074415358715, |
| "train/gen_step": 10312.0, |
| "unified/clip_ratio/high_mean": 0.0007447073858202202, |
| "unified/clip_ratio/low_mean": 2.7229518309468403e-05, |
| "unified/clip_ratio/region_mean": 0.0007719369050391833, |
| "unified/kl": 0.0694197312113829, |
| "unified/loss": 0.0026930213625746546 |
| }, |
| { |
| "epoch": 2.0768614891913533, |
| "gen/advantages_abs_mean": 0.05062304437160492, |
| "gen/advantages_max": 0.23533660173416138, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.3021351099014282, |
| "gen/advantages_std": 0.06885077804327011, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 83.5, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.5297470092773438, |
| "gen/reward_std": 0.20814448595046997, |
| "grad_norm": 0.00271556805819273, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "rewards/perceptual_score_reward_func/mean": 0.5205060243606567, |
| "rewards/perceptual_score_reward_func/std": 0.22350168228149414, |
| "step": 164, |
| "time_profile/curr_logprobs_and_update": 0.29248257720610127, |
| "time_profile/image_rollout": 88.6062811203301, |
| "time_profile/old_logps": 35.64170744456351, |
| "time_profile/ref_logps": 35.71029465831816, |
| "time_profile/reward": 1.0758190751075745, |
| "train/gen_step": 10376.0, |
| "unified/clip_ratio/high_mean": 0.011243771637964528, |
| "unified/clip_ratio/low_mean": 0.00031519579079031246, |
| "unified/clip_ratio/region_mean": 0.011558967485143512, |
| "unified/kl": 0.06918471836252138, |
| "unified/loss": 0.002829152402227919 |
| }, |
| { |
| "epoch": 2.089671737389912, |
| "grad_norm": 0.004280099645256996, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "step": 165, |
| "time_profile/curr_logprobs_and_update": 0.28327618300681934, |
| "train/gen_step": 10440.0, |
| "unified/clip_ratio/high_mean": 0.004382391351100523, |
| "unified/clip_ratio/low_mean": 0.00019320666888233973, |
| "unified/clip_ratio/region_mean": 0.0045755980208923575, |
| "unified/kl": 0.04727714671753347, |
| "unified/loss": 0.0019169329607393593 |
| }, |
| { |
| "epoch": 2.102481985588471, |
| "gen/advantages_abs_mean": 0.035958182066679, |
| "gen/advantages_max": 0.17724670469760895, |
| "gen/advantages_mean": 1.6298145055770874e-09, |
| "gen/advantages_min": -0.12050622701644897, |
| "gen/advantages_std": 0.04613741859793663, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 31.0, |
| "gen/prompt/mean_length": 31.0, |
| "gen/prompt/min_length": 31.0, |
| "gen/reward": 0.44015035033226013, |
| "gen/reward_std": 0.3897111415863037, |
| "grad_norm": 0.0035524514969438314, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "rewards/perceptual_score_reward_func/mean": 0.4246736764907837, |
| "rewards/perceptual_score_reward_func/std": 0.3995981812477112, |
| "step": 166, |
| "time_profile/curr_logprobs_and_update": 0.29225988683174364, |
| "time_profile/image_rollout": 89.83664705976844, |
| "time_profile/old_logps": 35.64525066129863, |
| "time_profile/ref_logps": 35.67439313046634, |
| "time_profile/reward": 1.3415015526115894, |
| "train/gen_step": 10504.0, |
| "unified/clip_ratio/high_mean": 0.010256347398353682, |
| "unified/clip_ratio/low_mean": 0.0009354562380394782, |
| "unified/clip_ratio/region_mean": 0.01119180366731598, |
| "unified/kl": 0.043152328697033226, |
| "unified/loss": 0.0016656069524287886 |
| }, |
| { |
| "epoch": 2.1152922337870295, |
| "grad_norm": 0.003104103496298194, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "step": 167, |
| "time_profile/curr_logprobs_and_update": 0.28839108292595483, |
| "train/gen_step": 10568.0, |
| "unified/clip_ratio/high_mean": 0.0014482807009699172, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0014482807009699172, |
| "unified/kl": 0.042221105250064284, |
| "unified/loss": 0.0016389742395404028 |
| }, |
| { |
| "epoch": 2.1281024819855885, |
| "gen/advantages_abs_mean": 0.04718589410185814, |
| "gen/advantages_max": 0.25266826152801514, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.3306066393852234, |
| "gen/advantages_std": 0.07132115215063095, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.09375, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 130.0, |
| "gen/prompt/min_length": 32.0, |
| "gen/reward": 0.20786593854427338, |
| "gen/reward_std": 0.20858700573444366, |
| "grad_norm": 0.0031762246508151293, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "rewards/perceptual_score_reward_func/mean": 0.21164503693580627, |
| "rewards/perceptual_score_reward_func/std": 0.22171460092067719, |
| "step": 168, |
| "time_profile/curr_logprobs_and_update": 0.28383773076348007, |
| "time_profile/image_rollout": 86.03059590421617, |
| "time_profile/old_logps": 35.66852024383843, |
| "time_profile/ref_logps": 35.73095652461052, |
| "time_profile/reward": 1.5308976825326681, |
| "train/gen_step": 10632.0, |
| "unified/clip_ratio/high_mean": 0.007134736979423906, |
| "unified/clip_ratio/low_mean": 0.0001799682322598528, |
| "unified/clip_ratio/region_mean": 0.007314705258977483, |
| "unified/kl": 0.03931887086946517, |
| "unified/loss": 0.0015712360254838131 |
| }, |
| { |
| "epoch": 2.1409127301841475, |
| "grad_norm": 0.0041104997508227825, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 169, |
| "time_profile/curr_logprobs_and_update": 0.29093144927173853, |
| "train/gen_step": 10696.0, |
| "unified/clip_ratio/high_mean": 2.531525296944892e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 2.531525296944892e-05, |
| "unified/kl": 0.041141612542560324, |
| "unified/loss": 0.0015832010067242663 |
| }, |
| { |
| "epoch": 2.153722978382706, |
| "gen/advantages_abs_mean": 0.07721328735351562, |
| "gen/advantages_max": 0.3477572798728943, |
| "gen/advantages_mean": 2.7939677238464355e-09, |
| "gen/advantages_min": -0.2912459969520569, |
| "gen/advantages_std": 0.10059475153684616, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 137.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.4667072296142578, |
| "gen/reward_std": 0.25318169593811035, |
| "grad_norm": 0.004474426619708538, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "rewards/perceptual_score_reward_func/mean": 0.4682084918022156, |
| "rewards/perceptual_score_reward_func/std": 0.24427475035190582, |
| "step": 170, |
| "time_profile/curr_logprobs_and_update": 0.28858580024098046, |
| "time_profile/image_rollout": 89.55203330516815, |
| "time_profile/old_logps": 35.70584703609347, |
| "time_profile/ref_logps": 35.79972547106445, |
| "time_profile/reward": 1.1563408393412828, |
| "train/gen_step": 10760.0, |
| "unified/clip_ratio/high_mean": 0.0006694951052850229, |
| "unified/clip_ratio/low_mean": 6.273121380218072e-05, |
| "unified/clip_ratio/region_mean": 0.0007322263181777089, |
| "unified/kl": 0.046405625878833234, |
| "unified/loss": 0.0018544251106504817 |
| }, |
| { |
| "epoch": 2.166533226581265, |
| "grad_norm": 0.0052252947352826595, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 171, |
| "time_profile/curr_logprobs_and_update": 0.28510314726736397, |
| "train/gen_step": 10824.0, |
| "unified/clip_ratio/high_mean": 3.648532583611086e-05, |
| "unified/clip_ratio/low_mean": 1.2934602636960335e-05, |
| "unified/clip_ratio/region_mean": 4.94199293825659e-05, |
| "unified/kl": 0.06382056430447847, |
| "unified/loss": 0.0025927637943823356 |
| }, |
| { |
| "epoch": 2.1793434747798237, |
| "gen/advantages_abs_mean": 0.0301031656563282, |
| "gen/advantages_max": 0.11702626943588257, |
| "gen/advantages_mean": -4.889443516731262e-09, |
| "gen/advantages_min": -0.1823127269744873, |
| "gen/advantages_std": 0.04142240807414055, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 88.5, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.7539215087890625, |
| "gen/reward_std": 0.08662469685077667, |
| "grad_norm": 0.0047385720536112785, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "rewards/perceptual_score_reward_func/mean": 0.7586275339126587, |
| "rewards/perceptual_score_reward_func/std": 0.08558635413646698, |
| "step": 172, |
| "time_profile/curr_logprobs_and_update": 0.29184379530488513, |
| "time_profile/image_rollout": 92.46348539367318, |
| "time_profile/old_logps": 35.65855384618044, |
| "time_profile/ref_logps": 35.705090867355466, |
| "time_profile/reward": 1.2204638700932264, |
| "train/gen_step": 10888.0, |
| "unified/clip_ratio/high_mean": 0.0006851561292933184, |
| "unified/clip_ratio/low_mean": 0.000610612833952473, |
| "unified/clip_ratio/region_mean": 0.0012957689750692225, |
| "unified/kl": 0.058761145861353725, |
| "unified/loss": 0.0022805339922342682 |
| }, |
| { |
| "epoch": 2.1921537229783827, |
| "grad_norm": 0.0027263937518000603, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "step": 173, |
| "time_profile/curr_logprobs_and_update": 0.2959218217874877, |
| "train/gen_step": 10952.0, |
| "unified/clip_ratio/high_mean": 0.0005940043447481003, |
| "unified/clip_ratio/low_mean": 1.3377568393480033e-05, |
| "unified/clip_ratio/region_mean": 0.0006073819131415803, |
| "unified/kl": 0.07492666487814859, |
| "unified/loss": 0.0030623077946074773 |
| }, |
| { |
| "epoch": 2.2049639711769418, |
| "gen/advantages_abs_mean": 0.01619063690304756, |
| "gen/advantages_max": 0.048489928245544434, |
| "gen/advantages_mean": -1.862645149230957e-09, |
| "gen/advantages_min": -0.06617963314056396, |
| "gen/advantages_std": 0.020318645983934402, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 39.0, |
| "gen/prompt/mean_length": 36.5, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.8497151732444763, |
| "gen/reward_std": 0.023617252707481384, |
| "grad_norm": 0.002491321414709091, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "rewards/perceptual_score_reward_func/mean": 0.8468820452690125, |
| "rewards/perceptual_score_reward_func/std": 0.025489442050457, |
| "step": 174, |
| "time_profile/curr_logprobs_and_update": 0.2864244414377026, |
| "time_profile/image_rollout": 95.55869417823851, |
| "time_profile/old_logps": 35.637094331905246, |
| "time_profile/ref_logps": 35.696372866630554, |
| "time_profile/reward": 1.4515288285911083, |
| "train/gen_step": 11016.0, |
| "unified/clip_ratio/high_mean": 0.0013642299363709753, |
| "unified/clip_ratio/low_mean": 0.00038164848774613347, |
| "unified/clip_ratio/region_mean": 0.0017458784241171088, |
| "unified/kl": 0.07242366363061592, |
| "unified/loss": 0.002966343375192082 |
| }, |
| { |
| "epoch": 2.2177742193755003, |
| "grad_norm": 0.0011181713780388236, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "step": 175, |
| "time_profile/curr_logprobs_and_update": 0.2914645765267778, |
| "train/gen_step": 11080.0, |
| "unified/clip_ratio/high_mean": 0.00037641884682670934, |
| "unified/clip_ratio/low_mean": 3.8014602978364564e-05, |
| "unified/clip_ratio/region_mean": 0.0004144334498050739, |
| "unified/kl": 0.03686668295995332, |
| "unified/loss": 0.0014178659844787944 |
| }, |
| { |
| "epoch": 2.2305844675740594, |
| "gen/advantages_abs_mean": 0.06131240725517273, |
| "gen/advantages_max": 0.21112197637557983, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.2674327492713928, |
| "gen/advantages_std": 0.07824128866195679, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 215.5, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.5548129081726074, |
| "gen/reward_std": 0.13008329272270203, |
| "grad_norm": 0.0041183605790138245, |
| "learning_rate": 1e-05, |
| "loss": 0.0015, |
| "rewards/perceptual_score_reward_func/mean": 0.5545451641082764, |
| "rewards/perceptual_score_reward_func/std": 0.11709090322256088, |
| "step": 176, |
| "time_profile/curr_logprobs_and_update": 0.29137875014566816, |
| "time_profile/image_rollout": 91.59421370364726, |
| "time_profile/old_logps": 35.71455822326243, |
| "time_profile/ref_logps": 35.81172662228346, |
| "time_profile/reward": 1.4883500207215548, |
| "train/gen_step": 11144.0, |
| "unified/clip_ratio/high_mean": 0.002236859237200406, |
| "unified/clip_ratio/low_mean": 0.0003157618521072436, |
| "unified/clip_ratio/region_mean": 0.0025526210829411866, |
| "unified/kl": 0.03860792936757207, |
| "unified/loss": 0.0014944951885809132 |
| }, |
| { |
| "epoch": 2.243394715772618, |
| "grad_norm": 0.006271969992667437, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "step": 177, |
| "time_profile/curr_logprobs_and_update": 0.2987609440460801, |
| "train/gen_step": 11208.0, |
| "unified/clip_ratio/high_mean": 9.015938485390507e-05, |
| "unified/clip_ratio/low_mean": 9.958242844732013e-05, |
| "unified/clip_ratio/region_mean": 0.0001897418133012252, |
| "unified/kl": 0.056187750946264714, |
| "unified/loss": 0.002266170584334759 |
| }, |
| { |
| "epoch": 2.256204963971177, |
| "gen/advantages_abs_mean": 0.05737482011318207, |
| "gen/advantages_max": 0.26223427057266235, |
| "gen/advantages_mean": -1.1641532182693481e-09, |
| "gen/advantages_min": -0.2832987308502197, |
| "gen/advantages_std": 0.07611891627311707, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.6172780990600586, |
| "gen/reward_std": 0.1598697006702423, |
| "grad_norm": 0.006493598222732544, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "rewards/perceptual_score_reward_func/mean": 0.6214098930358887, |
| "rewards/perceptual_score_reward_func/std": 0.1510515958070755, |
| "step": 178, |
| "time_profile/curr_logprobs_and_update": 0.288628595037153, |
| "time_profile/image_rollout": 93.41255523450673, |
| "time_profile/old_logps": 35.712708877399564, |
| "time_profile/ref_logps": 35.819618970155716, |
| "time_profile/reward": 1.2369329910725355, |
| "train/gen_step": 11272.0, |
| "unified/clip_ratio/high_mean": 0.0009131154902206617, |
| "unified/clip_ratio/low_mean": 0.0013802611756545957, |
| "unified/clip_ratio/region_mean": 0.002293376676789194, |
| "unified/kl": 0.05572476948145777, |
| "unified/loss": 0.002130313092493452 |
| }, |
| { |
| "epoch": 2.2690152121697356, |
| "grad_norm": 0.00450574466958642, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "step": 179, |
| "time_profile/curr_logprobs_and_update": 0.2841369793168269, |
| "train/gen_step": 11336.0, |
| "unified/clip_ratio/high_mean": 0.0005733701354984078, |
| "unified/clip_ratio/low_mean": 0.0003713260139193153, |
| "unified/clip_ratio/region_mean": 0.0009446961430512602, |
| "unified/kl": 0.07562511297874153, |
| "unified/loss": 0.002866666169211385 |
| }, |
| { |
| "epoch": 2.2818254603682946, |
| "gen/advantages_abs_mean": 0.03693922609090805, |
| "gen/advantages_max": 0.1345970630645752, |
| "gen/advantages_mean": 9.313225746154785e-10, |
| "gen/advantages_min": -0.3643248677253723, |
| "gen/advantages_std": 0.05376308783888817, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 89.0, |
| "gen/prompt/min_length": 35.0, |
| "gen/reward": 0.7307989597320557, |
| "gen/reward_std": 0.12450557202100754, |
| "grad_norm": 0.0036411313340067863, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "rewards/perceptual_score_reward_func/mean": 0.7366804480552673, |
| "rewards/perceptual_score_reward_func/std": 0.12658695876598358, |
| "step": 180, |
| "time_profile/curr_logprobs_and_update": 0.29716587750590406, |
| "time_profile/image_rollout": 88.12802308052778, |
| "time_profile/old_logps": 35.651363495737314, |
| "time_profile/ref_logps": 35.73251064121723, |
| "time_profile/reward": 1.130325747653842, |
| "train/gen_step": 11400.0, |
| "unified/clip_ratio/high_mean": 0.006377894053912314, |
| "unified/clip_ratio/low_mean": 0.0008435255413132836, |
| "unified/clip_ratio/region_mean": 0.007221419669804163, |
| "unified/kl": 0.08534415764734149, |
| "unified/loss": 0.003086884494223341 |
| }, |
| { |
| "epoch": 2.2946357085668536, |
| "grad_norm": 0.0029174236115068197, |
| "learning_rate": 1e-05, |
| "loss": 0.0039, |
| "step": 181, |
| "time_profile/curr_logprobs_and_update": 0.29214068927103654, |
| "train/gen_step": 11464.0, |
| "unified/clip_ratio/high_mean": 0.0026058121047753957, |
| "unified/clip_ratio/low_mean": 0.00016552963006688515, |
| "unified/clip_ratio/region_mean": 0.002771341710285924, |
| "unified/kl": 0.09713484643725678, |
| "unified/loss": 0.003910853625711752 |
| }, |
| { |
| "epoch": 2.307445956765412, |
| "gen/advantages_abs_mean": 0.08743979036808014, |
| "gen/advantages_max": 0.3487345278263092, |
| "gen/advantages_mean": 6.693881005048752e-10, |
| "gen/advantages_min": -0.3564333915710449, |
| "gen/advantages_std": 0.11450637131929398, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 85.0, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.5280884504318237, |
| "gen/reward_std": 0.21100233495235443, |
| "grad_norm": 0.0019279182888567448, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "rewards/perceptual_score_reward_func/mean": 0.4943096935749054, |
| "rewards/perceptual_score_reward_func/std": 0.23395422101020813, |
| "step": 182, |
| "time_profile/curr_logprobs_and_update": 0.2942618557426613, |
| "time_profile/image_rollout": 91.16302826814353, |
| "time_profile/old_logps": 35.6648780554533, |
| "time_profile/ref_logps": 35.72973041422665, |
| "time_profile/reward": 1.0635294429957867, |
| "train/gen_step": 11528.0, |
| "unified/clip_ratio/high_mean": 0.01493707289773738, |
| "unified/clip_ratio/low_mean": 0.0015484474943150417, |
| "unified/clip_ratio/region_mean": 0.01648552050573926, |
| "unified/kl": 0.08991632401011884, |
| "unified/loss": 0.003767051275644917 |
| }, |
| { |
| "epoch": 2.3202562049639712, |
| "grad_norm": 0.004897973965853453, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 183, |
| "time_profile/curr_logprobs_and_update": 0.2830031361372676, |
| "train/gen_step": 11592.0, |
| "unified/clip_ratio/high_mean": 0.013359589624087675, |
| "unified/clip_ratio/low_mean": 0.0002670826233952539, |
| "unified/clip_ratio/region_mean": 0.01362667221292213, |
| "unified/kl": 0.061459658725652844, |
| "unified/loss": 0.002509741234462126 |
| }, |
| { |
| "epoch": 2.3330664531625303, |
| "gen/advantages_abs_mean": 0.04273279756307602, |
| "gen/advantages_max": 0.18777477741241455, |
| "gen/advantages_mean": 6.984919309616089e-10, |
| "gen/advantages_min": -0.27026069164276123, |
| "gen/advantages_std": 0.06035928428173065, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 90.0, |
| "gen/prompt/min_length": 37.0, |
| "gen/reward": 0.716698408126831, |
| "gen/reward_std": 0.09943482279777527, |
| "grad_norm": 0.005198562052100897, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "rewards/perceptual_score_reward_func/mean": 0.719241201877594, |
| "rewards/perceptual_score_reward_func/std": 0.09961231797933578, |
| "step": 184, |
| "time_profile/curr_logprobs_and_update": 0.28848187151015736, |
| "time_profile/image_rollout": 97.41653303615749, |
| "time_profile/old_logps": 35.6586435046047, |
| "time_profile/ref_logps": 35.720301274210215, |
| "time_profile/reward": 1.2692248299717903, |
| "train/gen_step": 11656.0, |
| "unified/clip_ratio/high_mean": 0.027763162452174583, |
| "unified/clip_ratio/low_mean": 0.0007210646463136072, |
| "unified/clip_ratio/region_mean": 0.028484226852015126, |
| "unified/kl": 0.06450571143068373, |
| "unified/loss": 0.0026937655838992214 |
| }, |
| { |
| "epoch": 2.345876701361089, |
| "grad_norm": 0.0029400514904409647, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "step": 185, |
| "time_profile/curr_logprobs_and_update": 0.29348416050197557, |
| "train/gen_step": 11720.0, |
| "unified/clip_ratio/high_mean": 0.001847269874815538, |
| "unified/clip_ratio/low_mean": 8.904416790755931e-05, |
| "unified/clip_ratio/region_mean": 0.0019363140418136027, |
| "unified/kl": 0.06811343174194917, |
| "unified/loss": 0.00265883334084549 |
| }, |
| { |
| "epoch": 2.358686949559648, |
| "gen/advantages_abs_mean": 0.060479603707790375, |
| "gen/advantages_max": 0.2555360794067383, |
| "gen/advantages_mean": -2.3283064365386963e-10, |
| "gen/advantages_min": -0.20094865560531616, |
| "gen/advantages_std": 0.07668271660804749, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 80.0, |
| "gen/prompt/min_length": 23.0, |
| "gen/reward": 0.2623538672924042, |
| "gen/reward_std": 0.11184389144182205, |
| "grad_norm": 0.0025075250305235386, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "rewards/perceptual_score_reward_func/mean": 0.2549823820590973, |
| "rewards/perceptual_score_reward_func/std": 0.11117707192897797, |
| "step": 186, |
| "time_profile/curr_logprobs_and_update": 0.2875999791431241, |
| "time_profile/image_rollout": 95.74755357392132, |
| "time_profile/old_logps": 35.969283543527126, |
| "time_profile/ref_logps": 35.6872977539897, |
| "time_profile/reward": 1.2131715137511492, |
| "train/gen_step": 11784.0, |
| "unified/clip_ratio/high_mean": 0.0022697771437378833, |
| "unified/clip_ratio/low_mean": 0.0003853140624414664, |
| "unified/clip_ratio/region_mean": 0.002655091193446424, |
| "unified/kl": 0.059487957099918276, |
| "unified/loss": 0.002435175606251505 |
| }, |
| { |
| "epoch": 2.3714971977582064, |
| "grad_norm": 0.003168602241203189, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 187, |
| "time_profile/curr_logprobs_and_update": 0.28446206089574844, |
| "train/gen_step": 11848.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.039400996727636084, |
| "unified/loss": 0.0016360291774617508 |
| }, |
| { |
| "epoch": 2.3843074459567655, |
| "gen/advantages_abs_mean": 0.05380699038505554, |
| "gen/advantages_max": 0.18637949228286743, |
| "gen/advantages_mean": -3.958120942115784e-09, |
| "gen/advantages_min": -0.247217059135437, |
| "gen/advantages_std": 0.06866227835416794, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 90.5, |
| "gen/prompt/min_length": 38.0, |
| "gen/reward": 0.6251561045646667, |
| "gen/reward_std": 0.0872267559170723, |
| "grad_norm": 0.0032522021792829037, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "rewards/perceptual_score_reward_func/mean": 0.6136674880981445, |
| "rewards/perceptual_score_reward_func/std": 0.08085624128580093, |
| "step": 188, |
| "time_profile/curr_logprobs_and_update": 0.29212979626026936, |
| "time_profile/image_rollout": 90.07975675538182, |
| "time_profile/old_logps": 35.646439569070935, |
| "time_profile/ref_logps": 35.706678591668606, |
| "time_profile/reward": 1.2349300608038902, |
| "train/gen_step": 11912.0, |
| "unified/clip_ratio/high_mean": 0.00013954428322904278, |
| "unified/clip_ratio/low_mean": 1.3196790860092733e-05, |
| "unified/clip_ratio/region_mean": 0.0001527410740891355, |
| "unified/kl": 0.048110857111169025, |
| "unified/loss": 0.0018662362217582995 |
| }, |
| { |
| "epoch": 2.397117694155324, |
| "grad_norm": 0.002924871863797307, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "step": 189, |
| "time_profile/curr_logprobs_and_update": 0.29138450731988996, |
| "train/gen_step": 11976.0, |
| "unified/clip_ratio/high_mean": 0.00043166354043933097, |
| "unified/clip_ratio/low_mean": 0.00023087494355422677, |
| "unified/clip_ratio/region_mean": 0.000662538483084063, |
| "unified/kl": 0.0693313748924993, |
| "unified/loss": 0.0026826070568404248 |
| }, |
| { |
| "epoch": 2.409927942353883, |
| "gen/advantages_abs_mean": 0.036966800689697266, |
| "gen/advantages_max": 0.19963926076889038, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.17272275686264038, |
| "gen/advantages_std": 0.0572461299598217, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.125, |
| "gen/prompt/max_length": 30.0, |
| "gen/prompt/mean_length": 27.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.11605678498744965, |
| "gen/reward_std": 0.12552930414676666, |
| "grad_norm": 0.003882054705172777, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "rewards/perceptual_score_reward_func/mean": 0.10083503276109695, |
| "rewards/perceptual_score_reward_func/std": 0.11439882963895798, |
| "step": 190, |
| "time_profile/curr_logprobs_and_update": 0.2918980286340229, |
| "time_profile/image_rollout": 91.15622867643833, |
| "time_profile/old_logps": 35.61673030629754, |
| "time_profile/ref_logps": 35.67608836479485, |
| "time_profile/reward": 1.4936586115509272, |
| "train/gen_step": 12040.0, |
| "unified/clip_ratio/high_mean": 0.0009299532603108673, |
| "unified/clip_ratio/low_mean": 0.0018241363704873947, |
| "unified/clip_ratio/region_mean": 0.0027540896044229157, |
| "unified/kl": 0.06421229938860051, |
| "unified/loss": 0.0030946720908104908 |
| }, |
| { |
| "epoch": 2.422738190552442, |
| "grad_norm": 0.003422857029363513, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 191, |
| "time_profile/curr_logprobs_and_update": 0.2808142671419773, |
| "train/gen_step": 12104.0, |
| "unified/clip_ratio/high_mean": 1.4575559362128843e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 1.4575559362128843e-05, |
| "unified/kl": 0.04138320931815542, |
| "unified/loss": 0.0015813396021258086 |
| }, |
| { |
| "epoch": 2.4355484387510007, |
| "gen/advantages_abs_mean": 0.045888688415288925, |
| "gen/advantages_max": 0.15211229026317596, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.3356465697288513, |
| "gen/advantages_std": 0.060004960745573044, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 84.0, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.39801621437072754, |
| "gen/reward_std": 0.1737685650587082, |
| "grad_norm": 0.002050932962447405, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/perceptual_score_reward_func/mean": 0.40392911434173584, |
| "rewards/perceptual_score_reward_func/std": 0.17653802037239075, |
| "step": 192, |
| "time_profile/curr_logprobs_and_update": 0.2883164933009539, |
| "time_profile/image_rollout": 92.29078873991966, |
| "time_profile/old_logps": 35.65081191062927, |
| "time_profile/ref_logps": 35.689905505627394, |
| "time_profile/reward": 1.0758352540433407, |
| "train/gen_step": 12168.0, |
| "unified/clip_ratio/high_mean": 8.769621945248218e-05, |
| "unified/clip_ratio/low_mean": 9.131907700066222e-05, |
| "unified/clip_ratio/region_mean": 0.0001790152964531444, |
| "unified/kl": 0.04167587656411342, |
| "unified/loss": 0.0017008707181958016 |
| }, |
| { |
| "epoch": 2.4483586869495597, |
| "grad_norm": 0.00368260545656085, |
| "learning_rate": 1e-05, |
| "loss": 0.0041, |
| "step": 193, |
| "time_profile/curr_logprobs_and_update": 0.2881558191438671, |
| "train/gen_step": 12232.0, |
| "unified/clip_ratio/high_mean": 0.00012643504942388972, |
| "unified/clip_ratio/low_mean": 2.5052377168321982e-05, |
| "unified/clip_ratio/region_mean": 0.0001514874265922117, |
| "unified/kl": 0.1030606998829171, |
| "unified/loss": 0.004149410695390543 |
| }, |
| { |
| "epoch": 2.4611689351481187, |
| "gen/advantages_abs_mean": 0.04738696664571762, |
| "gen/advantages_max": 0.15458166599273682, |
| "gen/advantages_mean": -3.725290298461914e-09, |
| "gen/advantages_min": -0.3881222605705261, |
| "gen/advantages_std": 0.06603256613016129, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 88.0, |
| "gen/prompt/min_length": 39.0, |
| "gen/reward": 0.7343631982803345, |
| "gen/reward_std": 0.10265038162469864, |
| "grad_norm": 0.003888089442625642, |
| "learning_rate": 1e-05, |
| "loss": 0.0045, |
| "rewards/perceptual_score_reward_func/mean": 0.7463352680206299, |
| "rewards/perceptual_score_reward_func/std": 0.09803234785795212, |
| "step": 194, |
| "time_profile/curr_logprobs_and_update": 0.287883290235186, |
| "time_profile/image_rollout": 90.74564089998603, |
| "time_profile/old_logps": 35.664300525560975, |
| "time_profile/ref_logps": 35.708490991964936, |
| "time_profile/reward": 1.5299178566783667, |
| "train/gen_step": 12296.0, |
| "unified/clip_ratio/high_mean": 0.0008944852061176789, |
| "unified/clip_ratio/low_mean": 0.0011902720243597287, |
| "unified/clip_ratio/region_mean": 0.002084757244119828, |
| "unified/kl": 0.11233595450175926, |
| "unified/loss": 0.004423787195264595 |
| }, |
| { |
| "epoch": 2.4739791833466773, |
| "grad_norm": 0.0031231113243848085, |
| "learning_rate": 1e-05, |
| "loss": 0.004, |
| "step": 195, |
| "time_profile/curr_logprobs_and_update": 0.29107131843920797, |
| "train/gen_step": 12360.0, |
| "unified/clip_ratio/high_mean": 0.00021144409492990235, |
| "unified/clip_ratio/low_mean": 7.923106568341609e-05, |
| "unified/clip_ratio/region_mean": 0.00029067516061331844, |
| "unified/kl": 0.098656052199658, |
| "unified/loss": 0.004180431839358789 |
| }, |
| { |
| "epoch": 2.4867894315452364, |
| "gen/advantages_abs_mean": 0.0698113888502121, |
| "gen/advantages_max": 0.27165061235427856, |
| "gen/advantages_mean": -3.958120942115784e-09, |
| "gen/advantages_min": -0.3109276294708252, |
| "gen/advantages_std": 0.0891900584101677, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 140.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.6346923112869263, |
| "gen/reward_std": 0.11421144008636475, |
| "grad_norm": 0.004553945269435644, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "rewards/perceptual_score_reward_func/mean": 0.6285253763198853, |
| "rewards/perceptual_score_reward_func/std": 0.11653388291597366, |
| "step": 196, |
| "time_profile/curr_logprobs_and_update": 0.2902447068481706, |
| "time_profile/image_rollout": 96.71165009960532, |
| "time_profile/old_logps": 35.683688862249255, |
| "time_profile/ref_logps": 35.757541788741946, |
| "time_profile/reward": 1.2395988013595343, |
| "train/gen_step": 12424.0, |
| "unified/clip_ratio/high_mean": 0.003897015763868694, |
| "unified/clip_ratio/low_mean": 0.004954910057676898, |
| "unified/clip_ratio/region_mean": 0.008851925810631656, |
| "unified/kl": 0.08233964705141261, |
| "unified/loss": 0.003243297023004743 |
| }, |
| { |
| "epoch": 2.499599679743795, |
| "grad_norm": 0.004913492128252983, |
| "learning_rate": 1e-05, |
| "loss": 0.0041, |
| "step": 197, |
| "time_profile/curr_logprobs_and_update": 0.2921755504794419, |
| "train/gen_step": 12488.0, |
| "unified/clip_ratio/high_mean": 0.0013023042602071655, |
| "unified/clip_ratio/low_mean": 0.0032974790565276635, |
| "unified/clip_ratio/region_mean": 0.0045997833303772495, |
| "unified/kl": 0.1036716791568324, |
| "unified/loss": 0.004193980672425823 |
| }, |
| { |
| "epoch": 2.512409927942354, |
| "gen/advantages_abs_mean": 0.07193036377429962, |
| "gen/advantages_max": 0.28605392575263977, |
| "gen/advantages_mean": 6.984919309616089e-10, |
| "gen/advantages_min": -0.32319527864456177, |
| "gen/advantages_std": 0.09065697342157364, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 43.0, |
| "gen/prompt/mean_length": 35.5, |
| "gen/prompt/min_length": 28.0, |
| "gen/reward": 0.5142920017242432, |
| "gen/reward_std": 0.1796254962682724, |
| "grad_norm": 0.004819054156541824, |
| "learning_rate": 1e-05, |
| "loss": 0.0038, |
| "rewards/perceptual_score_reward_func/mean": 0.515720009803772, |
| "rewards/perceptual_score_reward_func/std": 0.1884678304195404, |
| "step": 198, |
| "time_profile/curr_logprobs_and_update": 0.2872099138912745, |
| "time_profile/image_rollout": 90.45758785679936, |
| "time_profile/old_logps": 35.63970177434385, |
| "time_profile/ref_logps": 35.68582937121391, |
| "time_profile/reward": 1.3650418017059565, |
| "train/gen_step": 12552.0, |
| "unified/clip_ratio/high_mean": 0.00796576717220887, |
| "unified/clip_ratio/low_mean": 0.03178466257486434, |
| "unified/clip_ratio/region_mean": 0.039750429803461884, |
| "unified/kl": 0.08939005044521764, |
| "unified/loss": 0.004010563134215772 |
| }, |
| { |
| "epoch": 2.5252201761409125, |
| "grad_norm": 0.00535630714148283, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "step": 199, |
| "time_profile/curr_logprobs_and_update": 0.2841974784096237, |
| "train/gen_step": 12616.0, |
| "unified/clip_ratio/high_mean": 0.0009237372660209076, |
| "unified/clip_ratio/low_mean": 0.00028113492589909583, |
| "unified/clip_ratio/region_mean": 0.0012048721910105087, |
| "unified/kl": 0.0417316788516473, |
| "unified/loss": 0.0017045280983438715 |
| }, |
| { |
| "epoch": 2.5380304243394716, |
| "gen/advantages_abs_mean": 0.037127502262592316, |
| "gen/advantages_max": 0.1399545967578888, |
| "gen/advantages_mean": 1.3969838619232178e-09, |
| "gen/advantages_min": -0.2307451367378235, |
| "gen/advantages_std": 0.050681211054325104, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.015625, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 228.0, |
| "gen/prompt/min_length": 228.0, |
| "gen/reward": 0.2671893835067749, |
| "gen/reward_std": 0.24783937633037567, |
| "grad_norm": 0.004151855129748583, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "rewards/perceptual_score_reward_func/mean": 0.2690585255622864, |
| "rewards/perceptual_score_reward_func/std": 0.24623475968837738, |
| "step": 200, |
| "time_profile/curr_logprobs_and_update": 0.2871844826149754, |
| "time_profile/image_rollout": 95.29160342365503, |
| "time_profile/old_logps": 35.766085494309664, |
| "time_profile/ref_logps": 35.8024538513273, |
| "time_profile/reward": 1.1446097139269114, |
| "train/gen_step": 12680.0, |
| "unified/clip_ratio/high_mean": 0.0077927707197886775, |
| "unified/clip_ratio/low_mean": 0.0029419622351269936, |
| "unified/clip_ratio/region_mean": 0.010734732993114449, |
| "unified/kl": 0.0463491125847213, |
| "unified/loss": 0.0019507477118168026 |
| }, |
| { |
| "epoch": 2.5508406725380306, |
| "grad_norm": 0.002670120680704713, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 201, |
| "time_profile/curr_logprobs_and_update": 0.28650486533297226, |
| "train/gen_step": 12744.0, |
| "unified/clip_ratio/high_mean": 0.00013077845142106526, |
| "unified/clip_ratio/low_mean": 8.67168810145813e-05, |
| "unified/clip_ratio/region_mean": 0.00021749533243564656, |
| "unified/kl": 0.06202875357121229, |
| "unified/loss": 0.002423393003482488 |
| }, |
| { |
| "epoch": 2.563650920736589, |
| "gen/advantages_abs_mean": 0.09604242444038391, |
| "gen/advantages_max": 0.28835737705230713, |
| "gen/advantages_mean": 2.6193447411060333e-10, |
| "gen/advantages_min": -0.29365015029907227, |
| "gen/advantages_std": 0.11690658330917358, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 32.0, |
| "gen/prompt/mean_length": 29.0, |
| "gen/prompt/min_length": 26.0, |
| "gen/reward": 0.25929340720176697, |
| "gen/reward_std": 0.12897181510925293, |
| "grad_norm": 0.003636396722868085, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/perceptual_score_reward_func/mean": 0.26449474692344666, |
| "rewards/perceptual_score_reward_func/std": 0.1299288123846054, |
| "step": 202, |
| "time_profile/curr_logprobs_and_update": 0.29194199573248625, |
| "time_profile/image_rollout": 94.50923268683255, |
| "time_profile/old_logps": 35.63929962925613, |
| "time_profile/ref_logps": 35.69228541851044, |
| "time_profile/reward": 1.3253274355083704, |
| "train/gen_step": 12808.0, |
| "unified/clip_ratio/high_mean": 0.001574381493810506, |
| "unified/clip_ratio/low_mean": 0.003362546233802277, |
| "unified/clip_ratio/region_mean": 0.004936927789458423, |
| "unified/kl": 0.062456131854560226, |
| "unified/loss": 0.002444851220388955 |
| }, |
| { |
| "epoch": 2.576461168935148, |
| "grad_norm": 0.004620482213795185, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "step": 203, |
| "time_profile/curr_logprobs_and_update": 0.29166353173786774, |
| "train/gen_step": 12872.0, |
| "unified/clip_ratio/high_mean": 1.220703143189894e-05, |
| "unified/clip_ratio/low_mean": 3.780533188546542e-05, |
| "unified/clip_ratio/region_mean": 5.001236331736436e-05, |
| "unified/kl": 0.05314113886561245, |
| "unified/loss": 0.001977884821826592 |
| }, |
| { |
| "epoch": 2.5892714171337072, |
| "gen/advantages_abs_mean": 0.04947504773736, |
| "gen/advantages_max": 0.18993930518627167, |
| "gen/advantages_mean": 9.604264050722122e-10, |
| "gen/advantages_min": -0.25990980863571167, |
| "gen/advantages_std": 0.06558328121900558, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 84.5, |
| "gen/prompt/min_length": 26.0, |
| "gen/reward": 0.44555872678756714, |
| "gen/reward_std": 0.27785491943359375, |
| "grad_norm": 0.004160444252192974, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "rewards/perceptual_score_reward_func/mean": 0.453355610370636, |
| "rewards/perceptual_score_reward_func/std": 0.28221702575683594, |
| "step": 204, |
| "time_profile/curr_logprobs_and_update": 0.2906250981905032, |
| "time_profile/image_rollout": 87.65974761359394, |
| "time_profile/old_logps": 35.657574282959104, |
| "time_profile/ref_logps": 35.68245464004576, |
| "time_profile/reward": 1.1088725440204144, |
| "train/gen_step": 12936.0, |
| "unified/clip_ratio/high_mean": 0.00038497244440804934, |
| "unified/clip_ratio/low_mean": 0.0012731670121866046, |
| "unified/clip_ratio/region_mean": 0.0016581394465902122, |
| "unified/kl": 0.06184308050433174, |
| "unified/loss": 0.0024940579969552346 |
| }, |
| { |
| "epoch": 2.602081665332266, |
| "grad_norm": 0.0026204350870102644, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "step": 205, |
| "time_profile/curr_logprobs_and_update": 0.29234114653081633, |
| "train/gen_step": 13000.0, |
| "unified/clip_ratio/high_mean": 0.0005670754262609989, |
| "unified/clip_ratio/low_mean": 0.0005148732343513984, |
| "unified/clip_ratio/region_mean": 0.001081948661521892, |
| "unified/kl": 0.07684545189840719, |
| "unified/loss": 0.0031584093303536065 |
| }, |
| { |
| "epoch": 2.6148919135308244, |
| "gen/advantages_abs_mean": 0.056859977543354034, |
| "gen/advantages_max": 0.2253839373588562, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.2270420789718628, |
| "gen/advantages_std": 0.07238562405109406, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 46.0, |
| "gen/prompt/mean_length": 35.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.6065893769264221, |
| "gen/reward_std": 0.19950497150421143, |
| "grad_norm": 0.003692910773679614, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "rewards/perceptual_score_reward_func/mean": 0.6051943302154541, |
| "rewards/perceptual_score_reward_func/std": 0.20580682158470154, |
| "step": 206, |
| "time_profile/curr_logprobs_and_update": 0.29456248341011815, |
| "time_profile/image_rollout": 89.29932282492518, |
| "time_profile/old_logps": 35.62615482322872, |
| "time_profile/ref_logps": 35.67942949384451, |
| "time_profile/reward": 1.7956994157284498, |
| "train/gen_step": 13064.0, |
| "unified/clip_ratio/high_mean": 0.0021543388202189817, |
| "unified/clip_ratio/low_mean": 0.002322014155652141, |
| "unified/clip_ratio/region_mean": 0.0044763529740521335, |
| "unified/kl": 0.07198869198327884, |
| "unified/loss": 0.00294267701337958 |
| }, |
| { |
| "epoch": 2.6277021617293834, |
| "grad_norm": 0.0037548041436821222, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "step": 207, |
| "time_profile/curr_logprobs_and_update": 0.28642980358563364, |
| "train/gen_step": 13128.0, |
| "unified/clip_ratio/high_mean": 7.485922742489493e-05, |
| "unified/clip_ratio/low_mean": 1.198236168420408e-05, |
| "unified/clip_ratio/region_mean": 8.6841589109099e-05, |
| "unified/kl": 0.04835753160296008, |
| "unified/loss": 0.0018652965663932264 |
| }, |
| { |
| "epoch": 2.6405124099279424, |
| "gen/advantages_abs_mean": 0.07412971556186676, |
| "gen/advantages_max": 0.34676653146743774, |
| "gen/advantages_mean": -9.604264050722122e-10, |
| "gen/advantages_min": -0.23221302032470703, |
| "gen/advantages_std": 0.09449668973684311, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 140.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.46234989166259766, |
| "gen/reward_std": 0.30892524123191833, |
| "grad_norm": 0.0039537930861115456, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "rewards/perceptual_score_reward_func/mean": 0.4601737856864929, |
| "rewards/perceptual_score_reward_func/std": 0.31586548686027527, |
| "step": 208, |
| "time_profile/curr_logprobs_and_update": 0.28737615986028686, |
| "time_profile/image_rollout": 97.1835206206888, |
| "time_profile/old_logps": 35.69620050303638, |
| "time_profile/ref_logps": 35.75604062154889, |
| "time_profile/reward": 1.0410658437758684, |
| "train/gen_step": 13192.0, |
| "unified/clip_ratio/high_mean": 0.0007957720790727763, |
| "unified/clip_ratio/low_mean": 2.688549557205988e-05, |
| "unified/clip_ratio/region_mean": 0.0008226575746448361, |
| "unified/kl": 0.049520838365424424, |
| "unified/loss": 0.0019604504323069705 |
| }, |
| { |
| "epoch": 2.653322658126501, |
| "grad_norm": 0.005656134337186813, |
| "learning_rate": 1e-05, |
| "loss": 0.0047, |
| "step": 209, |
| "time_profile/curr_logprobs_and_update": 0.287895544199273, |
| "train/gen_step": 13256.0, |
| "unified/clip_ratio/high_mean": 0.00036484682641457766, |
| "unified/clip_ratio/low_mean": 2.5205809834005777e-05, |
| "unified/clip_ratio/region_mean": 0.00039005263442959404, |
| "unified/kl": 0.1169839627109468, |
| "unified/loss": 0.00449724481424596 |
| }, |
| { |
| "epoch": 2.66613290632506, |
| "gen/advantages_abs_mean": 0.05343227833509445, |
| "gen/advantages_max": 0.2956569492816925, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.29374969005584717, |
| "gen/advantages_std": 0.07137347757816315, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 88.5, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.3955264389514923, |
| "gen/reward_std": 0.3297499120235443, |
| "grad_norm": 0.0059889634139835835, |
| "learning_rate": 1e-05, |
| "loss": 0.0043, |
| "rewards/perceptual_score_reward_func/mean": 0.3783816695213318, |
| "rewards/perceptual_score_reward_func/std": 0.3314151465892792, |
| "step": 210, |
| "time_profile/curr_logprobs_and_update": 0.2907968986837659, |
| "time_profile/image_rollout": 92.30888234451413, |
| "time_profile/old_logps": 35.662212174385786, |
| "time_profile/ref_logps": 35.70151925459504, |
| "time_profile/reward": 1.0841166283935308, |
| "train/gen_step": 13320.0, |
| "unified/clip_ratio/high_mean": 0.00975874760933948, |
| "unified/clip_ratio/low_mean": 0.00022591120068682358, |
| "unified/clip_ratio/region_mean": 0.009984658883695374, |
| "unified/kl": 0.10625315242214128, |
| "unified/loss": 0.003979327900481167 |
| }, |
| { |
| "epoch": 2.678943154523619, |
| "grad_norm": 0.006428052205592394, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "step": 211, |
| "time_profile/curr_logprobs_and_update": 0.2929955120489467, |
| "train/gen_step": 13384.0, |
| "unified/clip_ratio/high_mean": 0.005091632628136722, |
| "unified/clip_ratio/low_mean": 0.0001600822142791003, |
| "unified/clip_ratio/region_mean": 0.005251714813311992, |
| "unified/kl": 0.07312529330374673, |
| "unified/loss": 0.002942810344393365 |
| }, |
| { |
| "epoch": 2.6917534027221777, |
| "gen/advantages_abs_mean": 0.05805864930152893, |
| "gen/advantages_max": 0.19847756624221802, |
| "gen/advantages_mean": -2.7939677238464355e-09, |
| "gen/advantages_min": -0.28302890062332153, |
| "gen/advantages_std": 0.07244789600372314, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 261.0, |
| "gen/prompt/min_length": 228.0, |
| "gen/reward": 0.4525802731513977, |
| "gen/reward_std": 0.13093777000904083, |
| "grad_norm": 0.0053682648576796055, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "rewards/perceptual_score_reward_func/mean": 0.4573948085308075, |
| "rewards/perceptual_score_reward_func/std": 0.13452716171741486, |
| "step": 212, |
| "time_profile/curr_logprobs_and_update": 0.2902073429140728, |
| "time_profile/image_rollout": 95.18189113773406, |
| "time_profile/old_logps": 35.68532197549939, |
| "time_profile/ref_logps": 35.79711914435029, |
| "time_profile/reward": 1.238876884803176, |
| "train/gen_step": 13448.0, |
| "unified/clip_ratio/high_mean": 0.01146003782923799, |
| "unified/clip_ratio/low_mean": 0.005478247154314886, |
| "unified/clip_ratio/region_mean": 0.01693828505813144, |
| "unified/kl": 0.06899716431507841, |
| "unified/loss": 0.0027768009167630225 |
| }, |
| { |
| "epoch": 2.7045636509207367, |
| "grad_norm": 0.004297909326851368, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "step": 213, |
| "time_profile/curr_logprobs_and_update": 0.28509608868625946, |
| "train/gen_step": 13512.0, |
| "unified/clip_ratio/high_mean": 0.0002028881699516205, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0002028881699516205, |
| "unified/kl": 0.05968526436481625, |
| "unified/loss": 0.0023397608256345848 |
| }, |
| { |
| "epoch": 2.7173738991192953, |
| "gen/advantages_abs_mean": 0.04815659299492836, |
| "gen/advantages_max": 0.1666778326034546, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.26088178157806396, |
| "gen/advantages_std": 0.06279948353767395, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.6226699948310852, |
| "gen/reward_std": 0.10022295266389847, |
| "grad_norm": 0.008419697172939777, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/perceptual_score_reward_func/mean": 0.6231578588485718, |
| "rewards/perceptual_score_reward_func/std": 0.09478815644979477, |
| "step": 214, |
| "time_profile/curr_logprobs_and_update": 0.28961416336824186, |
| "time_profile/image_rollout": 86.57739455625415, |
| "time_profile/old_logps": 35.68044064939022, |
| "time_profile/ref_logps": 35.7602315954864, |
| "time_profile/reward": 0.8626856170594692, |
| "train/gen_step": 13576.0, |
| "unified/clip_ratio/high_mean": 0.0016434026474598795, |
| "unified/clip_ratio/low_mean": 0.0001981396180781303, |
| "unified/clip_ratio/region_mean": 0.001841542259171547, |
| "unified/kl": 0.06322463700780645, |
| "unified/loss": 0.002458966589983902 |
| }, |
| { |
| "epoch": 2.7301841473178543, |
| "grad_norm": 0.004779498558491468, |
| "learning_rate": 1e-05, |
| "loss": 0.0044, |
| "step": 215, |
| "time_profile/curr_logprobs_and_update": 0.2893037685425952, |
| "train/gen_step": 13640.0, |
| "unified/clip_ratio/high_mean": 0.004174642702309939, |
| "unified/clip_ratio/low_mean": 0.006409092700778274, |
| "unified/clip_ratio/region_mean": 0.01058373525575007, |
| "unified/kl": 0.10860151320230216, |
| "unified/loss": 0.004160132786637405 |
| }, |
| { |
| "epoch": 2.742994395516413, |
| "gen/advantages_abs_mean": 0.04880675673484802, |
| "gen/advantages_max": 0.1898176670074463, |
| "gen/advantages_mean": 1.6298145055770874e-09, |
| "gen/advantages_min": -0.25191187858581543, |
| "gen/advantages_std": 0.06369016319513321, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 140.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.5992552042007446, |
| "gen/reward_std": 0.06945131719112396, |
| "grad_norm": 0.00413332786411047, |
| "learning_rate": 1e-05, |
| "loss": 0.0041, |
| "rewards/perceptual_score_reward_func/mean": 0.6106417179107666, |
| "rewards/perceptual_score_reward_func/std": 0.06783042103052139, |
| "step": 216, |
| "time_profile/curr_logprobs_and_update": 0.2896010137919802, |
| "time_profile/image_rollout": 83.2615770790726, |
| "time_profile/old_logps": 35.682660933583975, |
| "time_profile/ref_logps": 35.74056311137974, |
| "time_profile/reward": 1.1472970116883516, |
| "train/gen_step": 13704.0, |
| "unified/clip_ratio/high_mean": 0.03075463885033969, |
| "unified/clip_ratio/low_mean": 0.01234049254344427, |
| "unified/clip_ratio/region_mean": 0.0430951316957362, |
| "unified/kl": 0.09634337952593341, |
| "unified/loss": 0.004022476720820123 |
| }, |
| { |
| "epoch": 2.755804643714972, |
| "grad_norm": 0.0032422547228634357, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "step": 217, |
| "time_profile/curr_logprobs_and_update": 0.2900120613339823, |
| "train/gen_step": 13768.0, |
| "unified/clip_ratio/high_mean": 0.022472380485851318, |
| "unified/clip_ratio/low_mean": 0.0018062998951791087, |
| "unified/clip_ratio/region_mean": 0.02427868053837301, |
| "unified/kl": 0.0843336817342788, |
| "unified/loss": 0.0032579440839981544 |
| }, |
| { |
| "epoch": 2.768614891913531, |
| "gen/advantages_abs_mean": 0.027143746614456177, |
| "gen/advantages_max": 0.18130779266357422, |
| "gen/advantages_mean": -2.7939677238464355e-09, |
| "gen/advantages_min": -0.14696919918060303, |
| "gen/advantages_std": 0.04213613271713257, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.171875, |
| "gen/prompt/max_length": 44.0, |
| "gen/prompt/mean_length": 34.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.3068550229072571, |
| "gen/reward_std": 0.301911324262619, |
| "grad_norm": 0.002202271483838558, |
| "learning_rate": 1e-05, |
| "loss": 0.0041, |
| "rewards/perceptual_score_reward_func/mean": 0.30286717414855957, |
| "rewards/perceptual_score_reward_func/std": 0.29476356506347656, |
| "step": 218, |
| "time_profile/curr_logprobs_and_update": 0.2916179225721862, |
| "time_profile/image_rollout": 77.22199551947415, |
| "time_profile/old_logps": 35.64645759947598, |
| "time_profile/ref_logps": 35.69035775959492, |
| "time_profile/reward": 1.2508190497756004, |
| "train/gen_step": 13832.0, |
| "unified/clip_ratio/high_mean": 0.06337775956217229, |
| "unified/clip_ratio/low_mean": 0.008298227623527055, |
| "unified/clip_ratio/region_mean": 0.07167598631258443, |
| "unified/kl": 0.09170893608825281, |
| "unified/loss": 0.003902100499090011 |
| }, |
| { |
| "epoch": 2.7814251401120895, |
| "grad_norm": 0.0019570006988942623, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "step": 219, |
| "time_profile/curr_logprobs_and_update": 0.29028787108836696, |
| "train/gen_step": 13896.0, |
| "unified/clip_ratio/high_mean": 0.00017125081467384007, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.00017125081467384007, |
| "unified/kl": 0.06491467054001987, |
| "unified/loss": 0.0025595305351089337 |
| }, |
| { |
| "epoch": 2.7942353883106485, |
| "gen/advantages_abs_mean": 0.01623566634953022, |
| "gen/advantages_max": 0.047403693199157715, |
| "gen/advantages_mean": -2.7939677238464355e-09, |
| "gen/advantages_min": -0.09490931034088135, |
| "gen/advantages_std": 0.021146854385733604, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 36.0, |
| "gen/prompt/mean_length": 35.0, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.8461313247680664, |
| "gen/reward_std": 0.03506263345479965, |
| "grad_norm": 0.002078034682199359, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/perceptual_score_reward_func/mean": 0.8457783460617065, |
| "rewards/perceptual_score_reward_func/std": 0.03492708504199982, |
| "step": 220, |
| "time_profile/curr_logprobs_and_update": 0.29303739650640637, |
| "time_profile/image_rollout": 85.6809338927269, |
| "time_profile/old_logps": 35.65229227952659, |
| "time_profile/ref_logps": 35.679596012458205, |
| "time_profile/reward": 1.6915065199136734, |
| "train/gen_step": 13960.0, |
| "unified/clip_ratio/high_mean": 0.0017831098166425363, |
| "unified/clip_ratio/low_mean": 0.00016667497675371123, |
| "unified/clip_ratio/region_mean": 0.0019497847970342264, |
| "unified/kl": 0.06186116026947275, |
| "unified/loss": 0.002318331704088905 |
| }, |
| { |
| "epoch": 2.8070456365092076, |
| "grad_norm": 0.0016549167921766639, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "step": 221, |
| "time_profile/curr_logprobs_and_update": 0.28271615999983624, |
| "train/gen_step": 14024.0, |
| "unified/clip_ratio/high_mean": 0.0004973574132236536, |
| "unified/clip_ratio/low_mean": 3.728380397660658e-05, |
| "unified/clip_ratio/region_mean": 0.0005346412172002601, |
| "unified/kl": 0.055540885601658374, |
| "unified/loss": 0.0021841253794718796 |
| }, |
| { |
| "epoch": 2.819855884707766, |
| "gen/advantages_abs_mean": 0.015649918466806412, |
| "gen/advantages_max": 0.10429143905639648, |
| "gen/advantages_mean": 1.3969838619232178e-09, |
| "gen/advantages_min": -0.1413099765777588, |
| "gen/advantages_std": 0.024658359587192535, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.15625, |
| "gen/prompt/max_length": 33.0, |
| "gen/prompt/mean_length": 32.0, |
| "gen/prompt/min_length": 31.0, |
| "gen/reward": 0.4220384359359741, |
| "gen/reward_std": 0.416412353515625, |
| "grad_norm": 0.0014546598540619016, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "rewards/perceptual_score_reward_func/mean": 0.4332512319087982, |
| "rewards/perceptual_score_reward_func/std": 0.4173754155635834, |
| "step": 222, |
| "time_profile/curr_logprobs_and_update": 0.2969915381690953, |
| "time_profile/image_rollout": 90.88989748992026, |
| "time_profile/old_logps": 35.64420824497938, |
| "time_profile/ref_logps": 35.67107901349664, |
| "time_profile/reward": 2.101542688906193, |
| "train/gen_step": 14088.0, |
| "unified/clip_ratio/high_mean": 0.0011120515982838697, |
| "unified/clip_ratio/low_mean": 0.0002531600803195033, |
| "unified/clip_ratio/region_mean": 0.0013652116831508465, |
| "unified/kl": 0.055687586311250925, |
| "unified/loss": 0.0022361490337061696 |
| }, |
| { |
| "epoch": 2.832666132906325, |
| "grad_norm": 0.0014627320924773812, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "step": 223, |
| "time_profile/curr_logprobs_and_update": 0.2890927131229546, |
| "train/gen_step": 14152.0, |
| "unified/clip_ratio/high_mean": 6.403710813174257e-05, |
| "unified/clip_ratio/low_mean": 2.539202614570968e-05, |
| "unified/clip_ratio/region_mean": 8.942913427745225e-05, |
| "unified/kl": 0.060441506502684206, |
| "unified/loss": 0.0024136415540851885 |
| }, |
| { |
| "epoch": 2.8454763811048838, |
| "gen/advantages_abs_mean": 0.06136321276426315, |
| "gen/advantages_max": 0.2737995982170105, |
| "gen/advantages_mean": -2.3283064365386963e-10, |
| "gen/advantages_min": -0.27382445335388184, |
| "gen/advantages_std": 0.079731285572052, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 30.0, |
| "gen/prompt/mean_length": 27.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.2018337845802307, |
| "gen/reward_std": 0.14443330466747284, |
| "grad_norm": 0.0031836305279284716, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "rewards/perceptual_score_reward_func/mean": 0.19993999600410461, |
| "rewards/perceptual_score_reward_func/std": 0.15581290423870087, |
| "step": 224, |
| "time_profile/curr_logprobs_and_update": 0.2822554874292109, |
| "time_profile/image_rollout": 86.69328206963837, |
| "time_profile/old_logps": 35.651397259905934, |
| "time_profile/ref_logps": 35.67883262783289, |
| "time_profile/reward": 1.7083067148923874, |
| "train/gen_step": 14216.0, |
| "unified/clip_ratio/high_mean": 0.00011875155269081006, |
| "unified/clip_ratio/low_mean": 0.0006717665119140293, |
| "unified/clip_ratio/region_mean": 0.0007905180646048393, |
| "unified/kl": 0.05397293172427453, |
| "unified/loss": 0.002211625182098942 |
| }, |
| { |
| "epoch": 2.858286629303443, |
| "grad_norm": 0.0036669722758233547, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 225, |
| "time_profile/curr_logprobs_and_update": 0.2889335306826979, |
| "train/gen_step": 14280.0, |
| "unified/clip_ratio/high_mean": 3.7436996535689104e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 3.7436996535689104e-05, |
| "unified/kl": 0.03027197896153666, |
| "unified/loss": 0.0011347782128723338 |
| }, |
| { |
| "epoch": 2.8710968775020014, |
| "gen/advantages_abs_mean": 0.060056887567043304, |
| "gen/advantages_max": 0.2160344123840332, |
| "gen/advantages_mean": 1.6298145055770874e-09, |
| "gen/advantages_min": -0.29413020610809326, |
| "gen/advantages_std": 0.07890523970127106, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 185.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.6171910762786865, |
| "gen/reward_std": 0.10012081265449524, |
| "grad_norm": 0.002574487356469035, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "rewards/perceptual_score_reward_func/mean": 0.6302154064178467, |
| "rewards/perceptual_score_reward_func/std": 0.09106422960758209, |
| "step": 226, |
| "time_profile/curr_logprobs_and_update": 0.28898188957828097, |
| "time_profile/image_rollout": 94.76224956661463, |
| "time_profile/old_logps": 35.704933416098356, |
| "time_profile/ref_logps": 35.78546800278127, |
| "time_profile/reward": 1.1764553282409906, |
| "train/gen_step": 14344.0, |
| "unified/clip_ratio/high_mean": 0.00030724572025064845, |
| "unified/clip_ratio/low_mean": 4.166991584497737e-05, |
| "unified/clip_ratio/region_mean": 0.0003489156360956258, |
| "unified/kl": 0.04056441711145453, |
| "unified/loss": 0.0017030270500981715 |
| }, |
| { |
| "epoch": 2.8839071257005604, |
| "grad_norm": 0.005676161963492632, |
| "learning_rate": 1e-05, |
| "loss": 0.0046, |
| "step": 227, |
| "time_profile/curr_logprobs_and_update": 0.2964037763304077, |
| "train/gen_step": 14408.0, |
| "unified/clip_ratio/high_mean": 0.0003664525029307697, |
| "unified/clip_ratio/low_mean": 8.569373949285364e-05, |
| "unified/clip_ratio/region_mean": 0.000452146234238171, |
| "unified/kl": 0.11509677162393928, |
| "unified/loss": 0.004453327323972189 |
| }, |
| { |
| "epoch": 2.8967173738991194, |
| "gen/advantages_abs_mean": 0.033830348402261734, |
| "gen/advantages_max": 0.14224708080291748, |
| "gen/advantages_mean": -1.3969838619232178e-09, |
| "gen/advantages_min": -0.2486705780029297, |
| "gen/advantages_std": 0.04694213718175888, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 87.0, |
| "gen/prompt/min_length": 31.0, |
| "gen/reward": 0.7471082210540771, |
| "gen/reward_std": 0.09803838282823563, |
| "grad_norm": 0.005515103228390217, |
| "learning_rate": 1e-05, |
| "loss": 0.0044, |
| "rewards/perceptual_score_reward_func/mean": 0.7399195432662964, |
| "rewards/perceptual_score_reward_func/std": 0.10945288091897964, |
| "step": 228, |
| "time_profile/curr_logprobs_and_update": 0.29186375689459965, |
| "time_profile/image_rollout": 89.08404904603958, |
| "time_profile/old_logps": 35.664834290742874, |
| "time_profile/ref_logps": 35.68366582505405, |
| "time_profile/reward": 1.2496778517961502, |
| "train/gen_step": 14472.0, |
| "unified/clip_ratio/high_mean": 0.013442613232655276, |
| "unified/clip_ratio/low_mean": 0.0009425432981515769, |
| "unified/clip_ratio/region_mean": 0.014385156513526454, |
| "unified/kl": 0.1082348006311804, |
| "unified/loss": 0.00420809791364718 |
| }, |
| { |
| "epoch": 2.909527622097678, |
| "grad_norm": 0.0044300127774477005, |
| "learning_rate": 1e-05, |
| "loss": 0.0042, |
| "step": 229, |
| "time_profile/curr_logprobs_and_update": 0.29350452186190523, |
| "train/gen_step": 14536.0, |
| "unified/clip_ratio/high_mean": 0.000650564178613422, |
| "unified/clip_ratio/low_mean": 0.005874285083336872, |
| "unified/clip_ratio/region_mean": 0.0065248492301179795, |
| "unified/kl": 0.10480844264384359, |
| "unified/loss": 0.004268346909498177 |
| }, |
| { |
| "epoch": 2.922337870296237, |
| "gen/advantages_abs_mean": 0.08245858550071716, |
| "gen/advantages_max": 0.2647748589515686, |
| "gen/advantages_mean": -3.7834979593753815e-10, |
| "gen/advantages_min": -0.3321816921234131, |
| "gen/advantages_std": 0.10184665769338608, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 42.0, |
| "gen/prompt/mean_length": 33.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.36596447229385376, |
| "gen/reward_std": 0.22325140237808228, |
| "grad_norm": 0.003050280502066016, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "rewards/perceptual_score_reward_func/mean": 0.35242119431495667, |
| "rewards/perceptual_score_reward_func/std": 0.22620998322963715, |
| "step": 230, |
| "time_profile/curr_logprobs_and_update": 0.29015435941983014, |
| "time_profile/image_rollout": 98.16377575136721, |
| "time_profile/old_logps": 35.6339815557003, |
| "time_profile/ref_logps": 35.689124369993806, |
| "time_profile/reward": 1.4899360593408346, |
| "train/gen_step": 14600.0, |
| "unified/clip_ratio/high_mean": 0.003210875910554023, |
| "unified/clip_ratio/low_mean": 0.017312906777988246, |
| "unified/clip_ratio/region_mean": 0.02052378270400368, |
| "unified/kl": 0.08625791693339124, |
| "unified/loss": 0.0036507347490442044 |
| }, |
| { |
| "epoch": 2.935148118494796, |
| "grad_norm": 0.0037899394519627094, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "step": 231, |
| "time_profile/curr_logprobs_and_update": 0.2809451754146721, |
| "train/gen_step": 14664.0, |
| "unified/clip_ratio/high_mean": 0.0003451885913818842, |
| "unified/clip_ratio/low_mean": 0.00013730012778978562, |
| "unified/clip_ratio/region_mean": 0.0004824887219001539, |
| "unified/kl": 0.0409627566114068, |
| "unified/loss": 0.0017391420788044343 |
| }, |
| { |
| "epoch": 2.9479583666933546, |
| "gen/advantages_abs_mean": 0.05695874243974686, |
| "gen/advantages_max": 0.2941986918449402, |
| "gen/advantages_mean": 2.0954757928848267e-09, |
| "gen/advantages_min": -0.2176368236541748, |
| "gen/advantages_std": 0.07234527170658112, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 80.5, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.4000024199485779, |
| "gen/reward_std": 0.31324514746665955, |
| "grad_norm": 0.0029120896942913532, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/perceptual_score_reward_func/mean": 0.39365696907043457, |
| "rewards/perceptual_score_reward_func/std": 0.31685662269592285, |
| "step": 232, |
| "time_profile/curr_logprobs_and_update": 0.2896963453677017, |
| "time_profile/image_rollout": 90.54285294935107, |
| "time_profile/old_logps": 35.659948740154505, |
| "time_profile/ref_logps": 35.70540129579604, |
| "time_profile/reward": 1.3401405308395624, |
| "train/gen_step": 14728.0, |
| "unified/clip_ratio/high_mean": 0.004589359151395911, |
| "unified/clip_ratio/low_mean": 0.0004914687769996817, |
| "unified/clip_ratio/region_mean": 0.0050808279165721615, |
| "unified/kl": 0.041754993551876396, |
| "unified/loss": 0.0016714632110961247 |
| }, |
| { |
| "epoch": 2.9607686148919137, |
| "grad_norm": 0.003255711402744055, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "step": 233, |
| "time_profile/curr_logprobs_and_update": 0.28853817415074445, |
| "train/gen_step": 14792.0, |
| "unified/clip_ratio/high_mean": 0.00022434214497479843, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.00022434214497479843, |
| "unified/kl": 0.04886005027219653, |
| "unified/loss": 0.0021349032186321892 |
| }, |
| { |
| "epoch": 2.9735788630904723, |
| "gen/advantages_abs_mean": 0.06837429106235504, |
| "gen/advantages_max": 0.20085787773132324, |
| "gen/advantages_mean": -1.3969838619232178e-09, |
| "gen/advantages_min": -0.24849843978881836, |
| "gen/advantages_std": 0.08356224000453949, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 215.5, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.5199116468429565, |
| "gen/reward_std": 0.08993534743785858, |
| "grad_norm": 0.005596943199634552, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "rewards/perceptual_score_reward_func/mean": 0.5330313444137573, |
| "rewards/perceptual_score_reward_func/std": 0.08884021639823914, |
| "step": 234, |
| "time_profile/curr_logprobs_and_update": 0.28999189706519246, |
| "time_profile/image_rollout": 90.35698483511806, |
| "time_profile/old_logps": 35.76694936491549, |
| "time_profile/ref_logps": 35.79822871275246, |
| "time_profile/reward": 1.2922454122453928, |
| "train/gen_step": 14856.0, |
| "unified/clip_ratio/high_mean": 0.002169189732740051, |
| "unified/clip_ratio/low_mean": 7.525764704041649e-05, |
| "unified/clip_ratio/region_mean": 0.0022444473797804676, |
| "unified/kl": 0.05060361948562786, |
| "unified/loss": 0.0021131058651917556 |
| }, |
| { |
| "epoch": 2.9863891112890313, |
| "grad_norm": 0.00613767746835947, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "step": 235, |
| "time_profile/curr_logprobs_and_update": 0.2892706178536173, |
| "train/gen_step": 14920.0, |
| "unified/clip_ratio/high_mean": 0.00022474777415482095, |
| "unified/clip_ratio/low_mean": 1.6009220416890457e-05, |
| "unified/clip_ratio/region_mean": 0.0002407569945717114, |
| "unified/kl": 0.0750474848318845, |
| "unified/loss": 0.0028697888183160103 |
| }, |
| { |
| "epoch": 2.99919935948759, |
| "gen/advantages_abs_mean": 0.05524168536067009, |
| "gen/advantages_max": 0.19999942183494568, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.35143446922302246, |
| "gen/advantages_std": 0.07161935418844223, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.576038658618927, |
| "gen/reward_std": 0.09756524860858917, |
| "grad_norm": 0.004759375471621752, |
| "learning_rate": 1e-05, |
| "loss": 0.0028, |
| "rewards/perceptual_score_reward_func/mean": 0.5787273645401001, |
| "rewards/perceptual_score_reward_func/std": 0.08526910096406937, |
| "step": 236, |
| "time_profile/curr_logprobs_and_update": 0.2881124487903435, |
| "time_profile/image_rollout": 91.76546191610396, |
| "time_profile/old_logps": 35.70581114850938, |
| "time_profile/ref_logps": 35.819912476465106, |
| "time_profile/reward": 1.0791106317192316, |
| "train/gen_step": 14984.0, |
| "unified/clip_ratio/high_mean": 0.0028773223521056934, |
| "unified/clip_ratio/low_mean": 0.0034019033937511267, |
| "unified/clip_ratio/region_mean": 0.006279225777689135, |
| "unified/kl": 0.07088860735530034, |
| "unified/loss": 0.0028101922980567906 |
| }, |
| { |
| "epoch": 3.0, |
| "grad_norm": 0.005377649795264006, |
| "learning_rate": 1e-05, |
| "loss": 0.0006, |
| "step": 237, |
| "time_profile/curr_logprobs_and_update": 0.3263985142111778, |
| "train/gen_step": 14988.0, |
| "unified/clip_ratio/high_mean": 0.00037564977537840605, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.00037564977537840605, |
| "unified/kl": 0.23306765407323837, |
| "unified/loss": -0.02903024887200445 |
| }, |
| { |
| "epoch": 3.012810248198559, |
| "grad_norm": 0.0040442608296871185, |
| "learning_rate": 1e-05, |
| "loss": 0.0054, |
| "step": 238, |
| "time_profile/curr_logprobs_and_update": 0.2922834643977694, |
| "train/gen_step": 15052.0, |
| "unified/clip_ratio/high_mean": 0.007114615003956715, |
| "unified/clip_ratio/low_mean": 0.005583924483289593, |
| "unified/clip_ratio/region_mean": 0.01269853939083987, |
| "unified/kl": 0.13325618638191372, |
| "unified/loss": 0.005477978451381205 |
| }, |
| { |
| "epoch": 3.0256204963971176, |
| "gen/advantages_abs_mean": 0.06815211474895477, |
| "gen/advantages_max": 0.28162434697151184, |
| "gen/advantages_mean": -4.656612873077393e-10, |
| "gen/advantages_min": -0.36226972937583923, |
| "gen/advantages_std": 0.0867423266172409, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 126.0, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.48455798625946045, |
| "gen/reward_std": 0.14753340184688568, |
| "grad_norm": 0.004220765084028244, |
| "learning_rate": 1e-05, |
| "loss": 0.0047, |
| "rewards/perceptual_score_reward_func/mean": 0.4674367308616638, |
| "rewards/perceptual_score_reward_func/std": 0.15108731389045715, |
| "step": 239, |
| "time_profile/curr_logprobs_and_update": 0.2890873631986324, |
| "time_profile/image_rollout": 85.50802209228277, |
| "time_profile/old_logps": 35.64921068586409, |
| "time_profile/ref_logps": 35.71976723894477, |
| "time_profile/reward": 1.38679701089859, |
| "train/gen_step": 15116.0, |
| "unified/clip_ratio/high_mean": 0.016038957779528573, |
| "unified/clip_ratio/low_mean": 0.021020568940002704, |
| "unified/clip_ratio/region_mean": 0.03705952680320479, |
| "unified/kl": 0.11398179235402495, |
| "unified/loss": 0.006121751257524011 |
| }, |
| { |
| "epoch": 3.0384307445956766, |
| "grad_norm": 0.004147673025727272, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 240, |
| "time_profile/curr_logprobs_and_update": 0.29534867490292527, |
| "train/gen_step": 15180.0, |
| "unified/clip_ratio/high_mean": 0.00034446546123945154, |
| "unified/clip_ratio/low_mean": 0.0001585743866598932, |
| "unified/clip_ratio/region_mean": 0.0005030398488088395, |
| "unified/kl": 0.062158423592336476, |
| "unified/loss": 0.0025324274738522945 |
| }, |
| { |
| "epoch": 3.051240992794235, |
| "gen/advantages_abs_mean": 0.05185795575380325, |
| "gen/advantages_max": 0.19245928525924683, |
| "gen/advantages_mean": 4.656612873077393e-10, |
| "gen/advantages_min": -0.19449368119239807, |
| "gen/advantages_std": 0.06433796137571335, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 80.5, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.18533486127853394, |
| "gen/reward_std": 0.12151897698640823, |
| "grad_norm": 0.0035060045775026083, |
| "learning_rate": 1e-05, |
| "loss": 0.0037, |
| "rewards/perceptual_score_reward_func/mean": 0.18132202327251434, |
| "rewards/perceptual_score_reward_func/std": 0.12059197574853897, |
| "step": 241, |
| "time_profile/curr_logprobs_and_update": 0.2936995446216315, |
| "time_profile/image_rollout": 92.46782938763499, |
| "time_profile/old_logps": 35.647363713011146, |
| "time_profile/ref_logps": 35.7009810525924, |
| "time_profile/reward": 1.3794072847813368, |
| "train/gen_step": 15244.0, |
| "unified/clip_ratio/high_mean": 0.0025567706343281316, |
| "unified/clip_ratio/low_mean": 0.0014044004792594933, |
| "unified/clip_ratio/region_mean": 0.0039611710963072255, |
| "unified/kl": 0.05547615705290809, |
| "unified/loss": 0.004834193030546885 |
| }, |
| { |
| "epoch": 3.0640512409927942, |
| "grad_norm": 0.002918737241998315, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 242, |
| "time_profile/curr_logprobs_and_update": 0.28326690808171406, |
| "train/gen_step": 15308.0, |
| "unified/clip_ratio/high_mean": 0.0, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0, |
| "unified/kl": 0.03205126043758355, |
| "unified/loss": 0.0012558446269395063 |
| }, |
| { |
| "epoch": 3.0768614891913533, |
| "gen/advantages_abs_mean": 0.04403460770845413, |
| "gen/advantages_max": 0.2058252990245819, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.245433509349823, |
| "gen/advantages_std": 0.06192699819803238, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 35.0, |
| "gen/prompt/mean_length": 29.5, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.6110314726829529, |
| "gen/reward_std": 0.24258162081241608, |
| "grad_norm": 0.0026578244287520647, |
| "learning_rate": 1e-05, |
| "loss": 0.0001, |
| "rewards/perceptual_score_reward_func/mean": 0.6051622629165649, |
| "rewards/perceptual_score_reward_func/std": 0.24469920992851257, |
| "step": 243, |
| "time_profile/curr_logprobs_and_update": 0.2965333420142997, |
| "time_profile/image_rollout": 88.86197500675917, |
| "time_profile/old_logps": 35.613538667559624, |
| "time_profile/ref_logps": 35.66835015080869, |
| "time_profile/reward": 1.5575009007006884, |
| "train/gen_step": 15372.0, |
| "unified/clip_ratio/high_mean": 5.28329064763966e-05, |
| "unified/clip_ratio/low_mean": 3.773496882786276e-05, |
| "unified/clip_ratio/region_mean": 9.056787530425936e-05, |
| "unified/kl": 0.03381443230318837, |
| "unified/loss": 7.984058447618736e-05 |
| }, |
| { |
| "epoch": 3.089671737389912, |
| "grad_norm": 0.0016529714921489358, |
| "learning_rate": 1e-05, |
| "loss": 0.0018, |
| "step": 244, |
| "time_profile/curr_logprobs_and_update": 0.28808122701593675, |
| "train/gen_step": 15436.0, |
| "unified/clip_ratio/high_mean": 0.0001275367403650307, |
| "unified/clip_ratio/low_mean": 1.1909298336831853e-05, |
| "unified/clip_ratio/region_mean": 0.00013944603870186256, |
| "unified/kl": 0.04395691870013252, |
| "unified/loss": 0.001838372672409605 |
| }, |
| { |
| "epoch": 3.102481985588471, |
| "gen/advantages_abs_mean": 0.05394706502556801, |
| "gen/advantages_max": 0.24188965559005737, |
| "gen/advantages_mean": 6.984919309616089e-10, |
| "gen/advantages_min": -0.29084014892578125, |
| "gen/advantages_std": 0.0771428570151329, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 133.0, |
| "gen/prompt/min_length": 38.0, |
| "gen/reward": 0.5607863068580627, |
| "gen/reward_std": 0.30254271626472473, |
| "grad_norm": 0.0015357909724116325, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "rewards/perceptual_score_reward_func/mean": 0.5605348348617554, |
| "rewards/perceptual_score_reward_func/std": 0.30506521463394165, |
| "step": 245, |
| "time_profile/curr_logprobs_and_update": 0.2840186757093761, |
| "time_profile/image_rollout": 86.64872058853507, |
| "time_profile/old_logps": 35.670260759070516, |
| "time_profile/ref_logps": 35.72483950294554, |
| "time_profile/reward": 2.2138646077364683, |
| "train/gen_step": 15500.0, |
| "unified/clip_ratio/high_mean": 0.000323626689350931, |
| "unified/clip_ratio/low_mean": 0.0001830254377637175, |
| "unified/clip_ratio/region_mean": 0.0005066521271146485, |
| "unified/kl": 0.0435222196392715, |
| "unified/loss": 0.0005058303959231125 |
| }, |
| { |
| "epoch": 3.1152922337870295, |
| "grad_norm": 0.0013075098395347595, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "step": 246, |
| "time_profile/curr_logprobs_and_update": 0.29552398252417333, |
| "train/gen_step": 15564.0, |
| "unified/clip_ratio/high_mean": 0.0008637319715489866, |
| "unified/clip_ratio/low_mean": 5.0364660637569614e-05, |
| "unified/clip_ratio/region_mean": 0.0009140966330960509, |
| "unified/kl": 0.04288260970497504, |
| "unified/loss": 0.0017171224171761423 |
| }, |
| { |
| "epoch": 3.1281024819855885, |
| "gen/advantages_abs_mean": 0.037114981561899185, |
| "gen/advantages_max": 0.16461652517318726, |
| "gen/advantages_mean": 4.6566128730773926e-09, |
| "gen/advantages_min": -0.19582903385162354, |
| "gen/advantages_std": 0.05011643096804619, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 143.0, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.6422978043556213, |
| "gen/reward_std": 0.09519995748996735, |
| "grad_norm": 0.003914408851414919, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "rewards/perceptual_score_reward_func/mean": 0.6484772562980652, |
| "rewards/perceptual_score_reward_func/std": 0.09529213607311249, |
| "step": 247, |
| "time_profile/curr_logprobs_and_update": 0.2900813828164246, |
| "time_profile/image_rollout": 93.27366822212934, |
| "time_profile/old_logps": 35.687854655086994, |
| "time_profile/ref_logps": 35.74452555552125, |
| "time_profile/reward": 0.8686411567032337, |
| "train/gen_step": 15628.0, |
| "unified/clip_ratio/high_mean": 0.004472575677027635, |
| "unified/clip_ratio/low_mean": 0.0002732068396653631, |
| "unified/clip_ratio/region_mean": 0.004745782523059461, |
| "unified/kl": 0.09377857428626157, |
| "unified/loss": 0.00661650193069363 |
| }, |
| { |
| "epoch": 3.1409127301841475, |
| "grad_norm": 0.004214221611618996, |
| "learning_rate": 1e-05, |
| "loss": 0.0095, |
| "step": 248, |
| "time_profile/curr_logprobs_and_update": 0.2855500411824323, |
| "train/gen_step": 15692.0, |
| "unified/clip_ratio/high_mean": 0.002388650888860866, |
| "unified/clip_ratio/low_mean": 0.00043833946892846143, |
| "unified/clip_ratio/region_mean": 0.002826990362336801, |
| "unified/kl": 0.23597962281201035, |
| "unified/loss": 0.008077786382727936 |
| }, |
| { |
| "epoch": 3.153722978382706, |
| "gen/advantages_abs_mean": 0.057654447853565216, |
| "gen/advantages_max": 0.2809915542602539, |
| "gen/advantages_mean": -1.1641532182693481e-09, |
| "gen/advantages_min": -0.29573673009872437, |
| "gen/advantages_std": 0.0747082307934761, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 185.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.5992565155029297, |
| "gen/reward_std": 0.17862142622470856, |
| "grad_norm": 0.004478012677282095, |
| "learning_rate": 1e-05, |
| "loss": 0.0098, |
| "rewards/perceptual_score_reward_func/mean": 0.6018717288970947, |
| "rewards/perceptual_score_reward_func/std": 0.18333376944065094, |
| "step": 249, |
| "time_profile/curr_logprobs_and_update": 0.29048692525248043, |
| "time_profile/image_rollout": 98.79939872026443, |
| "time_profile/old_logps": 35.707147816196084, |
| "time_profile/ref_logps": 35.76749726943672, |
| "time_profile/reward": 1.223227709531784, |
| "train/gen_step": 15756.0, |
| "unified/clip_ratio/high_mean": 0.004625103229045635, |
| "unified/clip_ratio/low_mean": 0.004679095056417282, |
| "unified/clip_ratio/region_mean": 0.00930419833457563, |
| "unified/kl": 0.2325795415090397, |
| "unified/loss": 0.006798942915338557 |
| }, |
| { |
| "epoch": 3.166533226581265, |
| "grad_norm": 0.005828510504215956, |
| "learning_rate": 1e-05, |
| "loss": 0.0053, |
| "step": 250, |
| "time_profile/curr_logprobs_and_update": 0.295324581413297, |
| "train/gen_step": 15820.0, |
| "unified/clip_ratio/high_mean": 0.0033738677548171836, |
| "unified/clip_ratio/low_mean": 0.00015024944696051534, |
| "unified/clip_ratio/region_mean": 0.003524117216329614, |
| "unified/kl": 0.1311271166196093, |
| "unified/loss": 0.00534750197766698 |
| }, |
| { |
| "epoch": 3.1793434747798237, |
| "gen/advantages_abs_mean": 0.051578670740127563, |
| "gen/advantages_max": 0.2560831308364868, |
| "gen/advantages_mean": 4.889443516731262e-09, |
| "gen/advantages_min": -0.24882763624191284, |
| "gen/advantages_std": 0.07479658722877502, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 130.0, |
| "gen/prompt/min_length": 32.0, |
| "gen/reward": 0.6277204751968384, |
| "gen/reward_std": 0.1594085395336151, |
| "grad_norm": 0.005873980466276407, |
| "learning_rate": 1e-05, |
| "loss": 0.0032, |
| "rewards/perceptual_score_reward_func/mean": 0.6243882775306702, |
| "rewards/perceptual_score_reward_func/std": 0.15960080921649933, |
| "step": 251, |
| "time_profile/curr_logprobs_and_update": 0.28632479067891836, |
| "time_profile/image_rollout": 93.90890072844923, |
| "time_profile/old_logps": 35.66965501010418, |
| "time_profile/ref_logps": 35.7002028003335, |
| "time_profile/reward": 1.3597954772412777, |
| "train/gen_step": 15884.0, |
| "unified/clip_ratio/high_mean": 0.024379982000027667, |
| "unified/clip_ratio/low_mean": 0.001214746066580119, |
| "unified/clip_ratio/region_mean": 0.02559472790289874, |
| "unified/kl": 0.10064856286044233, |
| "unified/loss": 0.002321239304365008 |
| }, |
| { |
| "epoch": 3.1921537229783827, |
| "grad_norm": 0.0062650516629219055, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "step": 252, |
| "time_profile/curr_logprobs_and_update": 0.28965292443172075, |
| "train/gen_step": 15948.0, |
| "unified/clip_ratio/high_mean": 0.0006192159962665755, |
| "unified/clip_ratio/low_mean": 3.767355974559905e-05, |
| "unified/clip_ratio/region_mean": 0.000656889557831164, |
| "unified/kl": 0.051964796730317175, |
| "unified/loss": 0.0021718055440942408 |
| }, |
| { |
| "epoch": 3.2049639711769418, |
| "gen/advantages_abs_mean": 0.05800483375787735, |
| "gen/advantages_max": 0.22309106588363647, |
| "gen/advantages_mean": 1.3969838619232178e-09, |
| "gen/advantages_min": -0.30087709426879883, |
| "gen/advantages_std": 0.07652728259563446, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 134.5, |
| "gen/prompt/min_length": 41.0, |
| "gen/reward": 0.703700065612793, |
| "gen/reward_std": 0.1433587670326233, |
| "grad_norm": 0.003542720340192318, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "rewards/perceptual_score_reward_func/mean": 0.699893593788147, |
| "rewards/perceptual_score_reward_func/std": 0.14788110554218292, |
| "step": 253, |
| "time_profile/curr_logprobs_and_update": 0.2873108710628003, |
| "time_profile/image_rollout": 78.67467383109033, |
| "time_profile/old_logps": 35.68670476414263, |
| "time_profile/ref_logps": 35.72687746025622, |
| "time_profile/reward": 1.352814106270671, |
| "train/gen_step": 16012.0, |
| "unified/clip_ratio/high_mean": 0.00668658099584718, |
| "unified/clip_ratio/low_mean": 0.0005819909283673042, |
| "unified/clip_ratio/region_mean": 0.007268571912391053, |
| "unified/kl": 0.054736606223741546, |
| "unified/loss": 0.0021627833293678123 |
| }, |
| { |
| "epoch": 3.2177742193755003, |
| "grad_norm": 0.0020388448610901833, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "step": 254, |
| "time_profile/curr_logprobs_and_update": 0.2826185535523109, |
| "train/gen_step": 16076.0, |
| "unified/clip_ratio/high_mean": 0.0021658961159118917, |
| "unified/clip_ratio/low_mean": 4.8730007620179094e-05, |
| "unified/clip_ratio/region_mean": 0.0022146261235320708, |
| "unified/kl": 0.06000680977012962, |
| "unified/loss": 0.0024088529658570224 |
| }, |
| { |
| "epoch": 3.2305844675740594, |
| "gen/advantages_abs_mean": 0.03484244644641876, |
| "gen/advantages_max": 0.17698565125465393, |
| "gen/advantages_mean": 1.1641532182693481e-09, |
| "gen/advantages_min": -0.13506032526493073, |
| "gen/advantages_std": 0.0459853820502758, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 38.0, |
| "gen/prompt/mean_length": 31.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.5287748575210571, |
| "gen/reward_std": 0.29880011081695557, |
| "grad_norm": 0.0023810556158423424, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "rewards/perceptual_score_reward_func/mean": 0.5372272729873657, |
| "rewards/perceptual_score_reward_func/std": 0.2943369746208191, |
| "step": 255, |
| "time_profile/curr_logprobs_and_update": 0.2931378345238045, |
| "time_profile/image_rollout": 93.7112015336752, |
| "time_profile/old_logps": 35.630799155682325, |
| "time_profile/ref_logps": 35.65894949436188, |
| "time_profile/reward": 1.3760113697499037, |
| "train/gen_step": 16140.0, |
| "unified/clip_ratio/high_mean": 0.009280130810111586, |
| "unified/clip_ratio/low_mean": 0.000359229604327993, |
| "unified/clip_ratio/region_mean": 0.009639360386245244, |
| "unified/kl": 0.05741891771322116, |
| "unified/loss": 0.005403364793892251 |
| }, |
| { |
| "epoch": 3.243394715772618, |
| "grad_norm": 0.002519667847082019, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "step": 256, |
| "time_profile/curr_logprobs_and_update": 0.28939395473571494, |
| "train/gen_step": 16204.0, |
| "unified/clip_ratio/high_mean": 0.00035258051229902776, |
| "unified/clip_ratio/low_mean": 5.343413431546651e-05, |
| "unified/clip_ratio/region_mean": 0.00040601464388601016, |
| "unified/kl": 0.04141872317995876, |
| "unified/loss": 0.0016114869595185155 |
| }, |
| { |
| "epoch": 3.256204963971177, |
| "gen/advantages_abs_mean": 0.02622368559241295, |
| "gen/advantages_max": 0.08395981788635254, |
| "gen/advantages_mean": -2.0954757928848267e-09, |
| "gen/advantages_min": -0.14057499170303345, |
| "gen/advantages_std": 0.03329704329371452, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 42.0, |
| "gen/prompt/mean_length": 39.5, |
| "gen/prompt/min_length": 37.0, |
| "gen/reward": 0.7857663631439209, |
| "gen/reward_std": 0.04608968645334244, |
| "grad_norm": 0.0017559886910021305, |
| "learning_rate": 1e-05, |
| "loss": 0.0011, |
| "rewards/perceptual_score_reward_func/mean": 0.7845939993858337, |
| "rewards/perceptual_score_reward_func/std": 0.049433447420597076, |
| "step": 257, |
| "time_profile/curr_logprobs_and_update": 0.285079111228697, |
| "time_profile/image_rollout": 89.50349676422775, |
| "time_profile/old_logps": 35.64108816161752, |
| "time_profile/ref_logps": 35.680126287043095, |
| "time_profile/reward": 1.5028599482029676, |
| "train/gen_step": 16268.0, |
| "unified/clip_ratio/high_mean": 0.0007780354662827449, |
| "unified/clip_ratio/low_mean": 0.0001060587646861677, |
| "unified/clip_ratio/region_mean": 0.0008840942300594179, |
| "unified/kl": 0.042232642124872655, |
| "unified/loss": -0.0017025310626195278 |
| }, |
| { |
| "epoch": 3.2690152121697356, |
| "grad_norm": 0.0020468831062316895, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "step": 258, |
| "time_profile/curr_logprobs_and_update": 0.29102804412832484, |
| "train/gen_step": 16332.0, |
| "unified/clip_ratio/high_mean": 0.00019496802542562364, |
| "unified/clip_ratio/low_mean": 3.838280917989323e-05, |
| "unified/clip_ratio/region_mean": 0.00023335083460551687, |
| "unified/kl": 0.049175292369909585, |
| "unified/loss": 0.0019437705554992135 |
| }, |
| { |
| "epoch": 3.2818254603682946, |
| "gen/advantages_abs_mean": 0.05668795853853226, |
| "gen/advantages_max": 0.22122755646705627, |
| "gen/advantages_mean": 4.656612873077393e-10, |
| "gen/advantages_min": -0.30950745940208435, |
| "gen/advantages_std": 0.07221390306949615, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 126.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.3420964181423187, |
| "gen/reward_std": 0.18341200053691864, |
| "grad_norm": 0.0035164812579751015, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "rewards/perceptual_score_reward_func/mean": 0.332736998796463, |
| "rewards/perceptual_score_reward_func/std": 0.1799333691596985, |
| "step": 259, |
| "time_profile/curr_logprobs_and_update": 0.28926581400446594, |
| "time_profile/image_rollout": 84.9461916051805, |
| "time_profile/old_logps": 35.632869094610214, |
| "time_profile/ref_logps": 35.70344665646553, |
| "time_profile/reward": 1.4849092904478312, |
| "train/gen_step": 16396.0, |
| "unified/clip_ratio/high_mean": 0.000548151053408219, |
| "unified/clip_ratio/low_mean": 0.0005735490922234021, |
| "unified/clip_ratio/region_mean": 0.001121700142903137, |
| "unified/kl": 0.0501270120148547, |
| "unified/loss": 0.002472478828394742 |
| }, |
| { |
| "epoch": 3.2946357085668536, |
| "grad_norm": 0.00313062802888453, |
| "learning_rate": 1e-05, |
| "loss": 0.0022, |
| "step": 260, |
| "time_profile/curr_logprobs_and_update": 0.2928492010105401, |
| "train/gen_step": 16460.0, |
| "unified/clip_ratio/high_mean": 1.1160714166180696e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 1.1160714166180696e-05, |
| "unified/kl": 0.0554106883937493, |
| "unified/loss": 0.0023214585526147857 |
| }, |
| { |
| "epoch": 3.307445956765412, |
| "gen/advantages_abs_mean": 0.050924405455589294, |
| "gen/advantages_max": 0.20732958614826202, |
| "gen/advantages_mean": -1.3969838619232178e-09, |
| "gen/advantages_min": -0.24469858407974243, |
| "gen/advantages_std": 0.06935644894838333, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 132.5, |
| "gen/prompt/min_length": 37.0, |
| "gen/reward": 0.5703046321868896, |
| "gen/reward_std": 0.27719688415527344, |
| "grad_norm": 0.0019420040771365166, |
| "learning_rate": 1e-05, |
| "loss": 0.0031, |
| "rewards/perceptual_score_reward_func/mean": 0.5662599802017212, |
| "rewards/perceptual_score_reward_func/std": 0.27992069721221924, |
| "step": 261, |
| "time_profile/curr_logprobs_and_update": 0.29200352300540544, |
| "time_profile/image_rollout": 88.96158868446946, |
| "time_profile/old_logps": 35.90520252101123, |
| "time_profile/ref_logps": 35.727664368227124, |
| "time_profile/reward": 1.2210984416306019, |
| "train/gen_step": 16524.0, |
| "unified/clip_ratio/high_mean": 0.00047545267625537235, |
| "unified/clip_ratio/low_mean": 1.2765523024427239e-05, |
| "unified/clip_ratio/region_mean": 0.0004882181992797996, |
| "unified/kl": 0.058303156634792686, |
| "unified/loss": 0.00208740836387733 |
| }, |
| { |
| "epoch": 3.3202562049639712, |
| "grad_norm": 0.002041127998381853, |
| "learning_rate": 1e-05, |
| "loss": 0.0025, |
| "step": 262, |
| "time_profile/curr_logprobs_and_update": 0.28587006652378477, |
| "train/gen_step": 16588.0, |
| "unified/clip_ratio/high_mean": 8.956663259596098e-05, |
| "unified/clip_ratio/low_mean": 5.191157106310129e-05, |
| "unified/clip_ratio/region_mean": 0.00014147820365906227, |
| "unified/kl": 0.06144321715692058, |
| "unified/loss": 0.0024497655213053804 |
| }, |
| { |
| "epoch": 3.3330664531625303, |
| "gen/advantages_abs_mean": 0.04744440317153931, |
| "gen/advantages_max": 0.1995270550251007, |
| "gen/advantages_mean": 9.313225746154785e-10, |
| "gen/advantages_min": -0.21491345763206482, |
| "gen/advantages_std": 0.06199962645769119, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 140.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.5491054654121399, |
| "gen/reward_std": 0.08672457933425903, |
| "grad_norm": 0.002755101304501295, |
| "learning_rate": 1e-05, |
| "loss": 0.003, |
| "rewards/perceptual_score_reward_func/mean": 0.5549399852752686, |
| "rewards/perceptual_score_reward_func/std": 0.08706492930650711, |
| "step": 263, |
| "time_profile/curr_logprobs_and_update": 0.2881781238829717, |
| "time_profile/image_rollout": 91.32398653961718, |
| "time_profile/old_logps": 35.87527330592275, |
| "time_profile/ref_logps": 35.74318656511605, |
| "time_profile/reward": 1.2014333847910166, |
| "train/gen_step": 16652.0, |
| "unified/clip_ratio/high_mean": 0.0002988719397762907, |
| "unified/clip_ratio/low_mean": 0.0005312867833708879, |
| "unified/clip_ratio/region_mean": 0.0008301587204186944, |
| "unified/kl": 0.07159855647478253, |
| "unified/loss": 0.002919944567111088 |
| }, |
| { |
| "epoch": 3.345876701361089, |
| "grad_norm": 0.004094029776751995, |
| "learning_rate": 1e-05, |
| "loss": 0.0047, |
| "step": 264, |
| "time_profile/curr_logprobs_and_update": 0.28873762642615475, |
| "train/gen_step": 16716.0, |
| "unified/clip_ratio/high_mean": 0.00018977822855958948, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.00018977822855958948, |
| "unified/kl": 0.11650378745980561, |
| "unified/loss": 0.00521194833345362 |
| }, |
| { |
| "epoch": 3.358686949559648, |
| "gen/advantages_abs_mean": 0.05893790349364281, |
| "gen/advantages_max": 0.2842107117176056, |
| "gen/advantages_mean": -2.3283064365386963e-09, |
| "gen/advantages_min": -0.2532750964164734, |
| "gen/advantages_std": 0.08093169331550598, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 185.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.6093672513961792, |
| "gen/reward_std": 0.1476283073425293, |
| "grad_norm": 0.010794364847242832, |
| "learning_rate": 1e-05, |
| "loss": 0.0044, |
| "rewards/perceptual_score_reward_func/mean": 0.62360680103302, |
| "rewards/perceptual_score_reward_func/std": 0.13393056392669678, |
| "step": 265, |
| "time_profile/curr_logprobs_and_update": 0.2906798219191842, |
| "time_profile/image_rollout": 91.62773103825748, |
| "time_profile/old_logps": 35.756982708349824, |
| "time_profile/ref_logps": 35.8212514705956, |
| "time_profile/reward": 1.3091953620314598, |
| "train/gen_step": 16780.0, |
| "unified/clip_ratio/high_mean": 0.0007829357255104696, |
| "unified/clip_ratio/low_mean": 0.00030623077418567846, |
| "unified/clip_ratio/region_mean": 0.0010891665015151375, |
| "unified/kl": 0.11171365738846362, |
| "unified/loss": 0.005526022511730844 |
| }, |
| { |
| "epoch": 3.3714971977582064, |
| "grad_norm": 0.0054925777949392796, |
| "learning_rate": 1e-05, |
| "loss": 0.0058, |
| "step": 266, |
| "time_profile/curr_logprobs_and_update": 0.28407792330835946, |
| "train/gen_step": 16844.0, |
| "unified/clip_ratio/high_mean": 0.006384057131981535, |
| "unified/clip_ratio/low_mean": 0.00062740799603489, |
| "unified/clip_ratio/region_mean": 0.007011465103460068, |
| "unified/kl": 0.1416146681876853, |
| "unified/loss": 0.005470271369631519 |
| }, |
| { |
| "epoch": 3.3843074459567655, |
| "gen/advantages_abs_mean": 0.05981067568063736, |
| "gen/advantages_max": 0.239865243434906, |
| "gen/advantages_mean": -1.1641532182693481e-09, |
| "gen/advantages_min": -0.2284674048423767, |
| "gen/advantages_std": 0.07678545266389847, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.5440489649772644, |
| "gen/reward_std": 0.11048609018325806, |
| "grad_norm": 0.003351012710481882, |
| "learning_rate": 1e-05, |
| "loss": 0.0043, |
| "rewards/perceptual_score_reward_func/mean": 0.5470424294471741, |
| "rewards/perceptual_score_reward_func/std": 0.11494408547878265, |
| "step": 267, |
| "time_profile/curr_logprobs_and_update": 0.29731816417188384, |
| "time_profile/image_rollout": 90.62626018188894, |
| "time_profile/old_logps": 35.7683735601604, |
| "time_profile/ref_logps": 35.837917340919375, |
| "time_profile/reward": 1.1168938986957073, |
| "train/gen_step": 16908.0, |
| "unified/clip_ratio/high_mean": 0.0213104724916775, |
| "unified/clip_ratio/low_mean": 0.0021109240315126954, |
| "unified/clip_ratio/region_mean": 0.023421396685080254, |
| "unified/kl": 0.1302080180030316, |
| "unified/loss": 0.0008558859917684458 |
| }, |
| { |
| "epoch": 3.397117694155324, |
| "grad_norm": 0.004734479822218418, |
| "learning_rate": 1e-05, |
| "loss": 0.0051, |
| "step": 268, |
| "time_profile/curr_logprobs_and_update": 0.2901776253420394, |
| "train/gen_step": 16972.0, |
| "unified/clip_ratio/high_mean": 0.0021853026237295126, |
| "unified/clip_ratio/low_mean": 8.935081768868258e-05, |
| "unified/clip_ratio/region_mean": 0.0022746534550606157, |
| "unified/kl": 0.12629019923042506, |
| "unified/loss": 0.008493884124163742 |
| }, |
| { |
| "epoch": 3.409927942353883, |
| "gen/advantages_abs_mean": 0.07885642349720001, |
| "gen/advantages_max": 0.27771949768066406, |
| "gen/advantages_mean": 0.0, |
| "gen/advantages_min": -0.27976107597351074, |
| "gen/advantages_std": 0.0983208417892456, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 28.0, |
| "gen/prompt/mean_length": 26.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.31731441617012024, |
| "gen/reward_std": 0.1525343656539917, |
| "grad_norm": 0.005082281306385994, |
| "learning_rate": 1e-05, |
| "loss": 0.0065, |
| "rewards/perceptual_score_reward_func/mean": 0.3259761929512024, |
| "rewards/perceptual_score_reward_func/std": 0.14577125012874603, |
| "step": 269, |
| "time_profile/curr_logprobs_and_update": 0.28629041809472255, |
| "time_profile/image_rollout": 95.50635491497815, |
| "time_profile/old_logps": 35.63145359046757, |
| "time_profile/ref_logps": 35.680125527083874, |
| "time_profile/reward": 1.40938363596797, |
| "train/gen_step": 17036.0, |
| "unified/clip_ratio/high_mean": 0.006612370569200721, |
| "unified/clip_ratio/low_mean": 0.0005135899291417445, |
| "unified/clip_ratio/region_mean": 0.0071259605610975996, |
| "unified/kl": 0.10105247830506414, |
| "unified/loss": 0.009375499214911542 |
| }, |
| { |
| "epoch": 3.422738190552442, |
| "grad_norm": 0.004122909624129534, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 270, |
| "time_profile/curr_logprobs_and_update": 0.2959313581814058, |
| "train/gen_step": 17100.0, |
| "unified/clip_ratio/high_mean": 1.1556952813407406e-05, |
| "unified/clip_ratio/low_mean": 1.2056327250320464e-05, |
| "unified/clip_ratio/region_mean": 2.361328006372787e-05, |
| "unified/kl": 0.02998922864207998, |
| "unified/loss": 0.0011360755161149427 |
| }, |
| { |
| "epoch": 3.4355484387510007, |
| "gen/advantages_abs_mean": 0.05167783796787262, |
| "gen/advantages_max": 0.2315201759338379, |
| "gen/advantages_mean": 4.656612873077393e-10, |
| "gen/advantages_min": -0.3078737258911133, |
| "gen/advantages_std": 0.06637614965438843, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 140.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.6377387046813965, |
| "gen/reward_std": 0.07187502831220627, |
| "grad_norm": 0.0038060385268181562, |
| "learning_rate": 1e-05, |
| "loss": 0.0005, |
| "rewards/perceptual_score_reward_func/mean": 0.6316916346549988, |
| "rewards/perceptual_score_reward_func/std": 0.06951053440570831, |
| "step": 271, |
| "time_profile/curr_logprobs_and_update": 0.2952291261171922, |
| "time_profile/image_rollout": 92.62281246297061, |
| "time_profile/old_logps": 35.6961055342108, |
| "time_profile/ref_logps": 35.77339258044958, |
| "time_profile/reward": 1.0998948644846678, |
| "train/gen_step": 17164.0, |
| "unified/clip_ratio/high_mean": 7.730922334303614e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 7.730922334303614e-05, |
| "unified/kl": 0.04771494321175851, |
| "unified/loss": 0.0025305816452600993 |
| }, |
| { |
| "epoch": 3.4483586869495597, |
| "grad_norm": 0.0035483732353895903, |
| "learning_rate": 1e-05, |
| "loss": 0.0044, |
| "step": 272, |
| "time_profile/curr_logprobs_and_update": 0.2881234941887669, |
| "train/gen_step": 17228.0, |
| "unified/clip_ratio/high_mean": 0.0009083378363357042, |
| "unified/clip_ratio/low_mean": 2.7453177608549595e-05, |
| "unified/clip_ratio/region_mean": 0.0009357910148537485, |
| "unified/kl": 0.11065650603268296, |
| "unified/loss": 0.004713995716883801 |
| }, |
| { |
| "epoch": 3.4611689351481187, |
| "gen/advantages_abs_mean": 0.04610411450266838, |
| "gen/advantages_max": 0.15632766485214233, |
| "gen/advantages_mean": 2.3283064365386963e-10, |
| "gen/advantages_min": -0.20781588554382324, |
| "gen/advantages_std": 0.058578308671712875, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 32.0, |
| "gen/prompt/mean_length": 31.0, |
| "gen/prompt/min_length": 30.0, |
| "gen/reward": 0.54497230052948, |
| "gen/reward_std": 0.23054379224777222, |
| "grad_norm": 0.004041146952658892, |
| "learning_rate": 1e-05, |
| "loss": 0.0039, |
| "rewards/perceptual_score_reward_func/mean": 0.5383381843566895, |
| "rewards/perceptual_score_reward_func/std": 0.24360765516757965, |
| "step": 273, |
| "time_profile/curr_logprobs_and_update": 0.289339731563814, |
| "time_profile/image_rollout": 90.77092687599361, |
| "time_profile/old_logps": 35.637864058837295, |
| "time_profile/ref_logps": 35.69791112653911, |
| "time_profile/reward": 1.9225932322442532, |
| "train/gen_step": 17292.0, |
| "unified/clip_ratio/high_mean": 0.003151820457787835, |
| "unified/clip_ratio/low_mean": 0.00025966245357267326, |
| "unified/clip_ratio/region_mean": 0.0034114828986275825, |
| "unified/kl": 0.10243399511091411, |
| "unified/loss": 0.004216766497847857 |
| }, |
| { |
| "epoch": 3.4739791833466773, |
| "grad_norm": 0.0035277053248137236, |
| "learning_rate": 1e-05, |
| "loss": 0.0019, |
| "step": 274, |
| "time_profile/curr_logprobs_and_update": 0.2899961236398667, |
| "train/gen_step": 17356.0, |
| "unified/clip_ratio/high_mean": 0.00041052952474274207, |
| "unified/clip_ratio/low_mean": 0.0019760358327403083, |
| "unified/clip_ratio/region_mean": 0.002386565354754566, |
| "unified/kl": 0.04827476281207055, |
| "unified/loss": 0.0019140739777867566 |
| }, |
| { |
| "epoch": 3.4867894315452364, |
| "gen/advantages_abs_mean": 0.04911169782280922, |
| "gen/advantages_max": 0.26427870988845825, |
| "gen/advantages_mean": -4.656612873077393e-10, |
| "gen/advantages_min": -0.2291024774312973, |
| "gen/advantages_std": 0.0661192312836647, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 36.0, |
| "gen/prompt/mean_length": 31.5, |
| "gen/prompt/min_length": 27.0, |
| "gen/reward": 0.5072143077850342, |
| "gen/reward_std": 0.29288727045059204, |
| "grad_norm": 0.0041518728248775005, |
| "learning_rate": 1e-05, |
| "loss": -0.0001, |
| "rewards/perceptual_score_reward_func/mean": 0.5160332918167114, |
| "rewards/perceptual_score_reward_func/std": 0.27702122926712036, |
| "step": 275, |
| "time_profile/curr_logprobs_and_update": 0.2842450351163279, |
| "time_profile/image_rollout": 88.49953482858837, |
| "time_profile/old_logps": 35.63493550941348, |
| "time_profile/ref_logps": 35.67222382687032, |
| "time_profile/reward": 1.4600203577429056, |
| "train/gen_step": 17420.0, |
| "unified/clip_ratio/high_mean": 0.0034367842599749565, |
| "unified/clip_ratio/low_mean": 0.018341819089073397, |
| "unified/clip_ratio/region_mean": 0.021778603123493667, |
| "unified/kl": 0.04001146586961113, |
| "unified/loss": -3.612683156006824e-05 |
| }, |
| { |
| "epoch": 3.499599679743795, |
| "grad_norm": 0.004384258762001991, |
| "learning_rate": 1e-05, |
| "loss": 0.0013, |
| "step": 276, |
| "time_profile/curr_logprobs_and_update": 0.2950630830600858, |
| "train/gen_step": 17484.0, |
| "unified/clip_ratio/high_mean": 0.00029130261464160867, |
| "unified/clip_ratio/low_mean": 5.080641403765185e-05, |
| "unified/clip_ratio/region_mean": 0.0003421090295887552, |
| "unified/kl": 0.0315111142990645, |
| "unified/loss": 0.001208443153814187 |
| }, |
| { |
| "epoch": 3.512409927942354, |
| "gen/advantages_abs_mean": 0.05698845535516739, |
| "gen/advantages_max": 0.16889894008636475, |
| "gen/advantages_mean": 2.561137080192566e-09, |
| "gen/advantages_min": -0.3212686777114868, |
| "gen/advantages_std": 0.07203952968120575, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 185.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.5802289247512817, |
| "gen/reward_std": 0.09428770840167999, |
| "grad_norm": 0.0037689167074859142, |
| "learning_rate": 1e-05, |
| "loss": 0.0034, |
| "rewards/perceptual_score_reward_func/mean": 0.5970678925514221, |
| "rewards/perceptual_score_reward_func/std": 0.08215753734111786, |
| "step": 277, |
| "time_profile/curr_logprobs_and_update": 0.2907980792224407, |
| "time_profile/image_rollout": 95.3401379995048, |
| "time_profile/old_logps": 35.72558932006359, |
| "time_profile/ref_logps": 35.828392228111625, |
| "time_profile/reward": 3.191845502704382, |
| "train/gen_step": 17548.0, |
| "unified/clip_ratio/high_mean": 0.0012386310318106553, |
| "unified/clip_ratio/low_mean": 0.0001734837287585833, |
| "unified/clip_ratio/region_mean": 0.0014121147605692386, |
| "unified/kl": 0.05011636400013231, |
| "unified/loss": 0.0025757047515071463 |
| }, |
| { |
| "epoch": 3.5252201761409125, |
| "grad_norm": 0.0033455120865255594, |
| "learning_rate": 1e-05, |
| "loss": 0.0046, |
| "step": 278, |
| "time_profile/curr_logprobs_and_update": 0.28396442910889164, |
| "train/gen_step": 17612.0, |
| "unified/clip_ratio/high_mean": 0.00014397111044672783, |
| "unified/clip_ratio/low_mean": 3.5590467632573564e-05, |
| "unified/clip_ratio/region_mean": 0.0001795615780793014, |
| "unified/kl": 0.11658623901894316, |
| "unified/loss": 0.004760882653272347 |
| }, |
| { |
| "epoch": 3.5380304243394716, |
| "gen/advantages_abs_mean": 0.06220916286110878, |
| "gen/advantages_max": 0.19965755939483643, |
| "gen/advantages_mean": 3.958120942115784e-09, |
| "gen/advantages_min": -0.2763780355453491, |
| "gen/advantages_std": 0.07711361348628998, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 215.5, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.5751262903213501, |
| "gen/reward_std": 0.08173548430204391, |
| "grad_norm": 0.0033656784798949957, |
| "learning_rate": 1e-05, |
| "loss": 0.0045, |
| "rewards/perceptual_score_reward_func/mean": 0.5695576667785645, |
| "rewards/perceptual_score_reward_func/std": 0.07363401353359222, |
| "step": 279, |
| "time_profile/curr_logprobs_and_update": 0.2877699746750295, |
| "time_profile/image_rollout": 77.98259479552507, |
| "time_profile/old_logps": 35.70114668272436, |
| "time_profile/ref_logps": 35.775142496451735, |
| "time_profile/reward": 1.3990372698754072, |
| "train/gen_step": 17676.0, |
| "unified/clip_ratio/high_mean": 0.0005160919226909755, |
| "unified/clip_ratio/low_mean": 0.00017728943566908129, |
| "unified/clip_ratio/region_mean": 0.0006933813556315727, |
| "unified/kl": 0.10497432359261438, |
| "unified/loss": 0.002874232982343017 |
| }, |
| { |
| "epoch": 3.5508406725380306, |
| "grad_norm": 0.0034624349791556597, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "step": 280, |
| "time_profile/curr_logprobs_and_update": 0.2857127044117078, |
| "train/gen_step": 17740.0, |
| "unified/clip_ratio/high_mean": 3.8896130718057975e-05, |
| "unified/clip_ratio/low_mean": 2.6701456590672024e-05, |
| "unified/clip_ratio/region_mean": 6.559758730873e-05, |
| "unified/kl": 0.06620051473146304, |
| "unified/loss": 0.0025284356404426944 |
| }, |
| { |
| "epoch": 3.563650920736589, |
| "gen/advantages_abs_mean": 0.07148271054029465, |
| "gen/advantages_max": 0.29644376039505005, |
| "gen/advantages_mean": 1.3387762010097504e-09, |
| "gen/advantages_min": -0.2896309494972229, |
| "gen/advantages_std": 0.0903797522187233, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 83.5, |
| "gen/prompt/min_length": 24.0, |
| "gen/reward": 0.4678086042404175, |
| "gen/reward_std": 0.24638544023036957, |
| "grad_norm": 0.0029037499334663153, |
| "learning_rate": 1e-05, |
| "loss": 0.0024, |
| "rewards/perceptual_score_reward_func/mean": 0.4608888030052185, |
| "rewards/perceptual_score_reward_func/std": 0.261143296957016, |
| "step": 281, |
| "time_profile/curr_logprobs_and_update": 0.2909189436759334, |
| "time_profile/image_rollout": 95.60725989565253, |
| "time_profile/old_logps": 35.645109789445996, |
| "time_profile/ref_logps": 35.74024346843362, |
| "time_profile/reward": 1.1167160719633102, |
| "train/gen_step": 17804.0, |
| "unified/clip_ratio/high_mean": 7.616222410433693e-05, |
| "unified/clip_ratio/low_mean": 3.8152402339619584e-05, |
| "unified/clip_ratio/region_mean": 0.00011431462644395651, |
| "unified/kl": 0.06412994969286956, |
| "unified/loss": 0.006115963831689442 |
| }, |
| { |
| "epoch": 3.576461168935148, |
| "grad_norm": 0.0027928680647164583, |
| "learning_rate": 1e-05, |
| "loss": 0.0033, |
| "step": 282, |
| "time_profile/curr_logprobs_and_update": 0.2950462262961082, |
| "train/gen_step": 17868.0, |
| "unified/clip_ratio/high_mean": 0.00020154348021605983, |
| "unified/clip_ratio/low_mean": 2.6584201805235352e-05, |
| "unified/clip_ratio/region_mean": 0.00022812768202129519, |
| "unified/kl": 0.08310258729034103, |
| "unified/loss": 0.0033516893672640435 |
| }, |
| { |
| "epoch": 3.5892714171337072, |
| "gen/advantages_abs_mean": 0.07729902863502502, |
| "gen/advantages_max": 0.3134341835975647, |
| "gen/advantages_mean": 2.0372681319713593e-10, |
| "gen/advantages_min": -0.24613593518733978, |
| "gen/advantages_std": 0.09904753416776657, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 31.0, |
| "gen/prompt/mean_length": 28.5, |
| "gen/prompt/min_length": 26.0, |
| "gen/reward": 0.20991452038288116, |
| "gen/reward_std": 0.1033468246459961, |
| "grad_norm": 0.004990903660655022, |
| "learning_rate": 1e-05, |
| "loss": 0.0029, |
| "rewards/perceptual_score_reward_func/mean": 0.19766443967819214, |
| "rewards/perceptual_score_reward_func/std": 0.10781577229499817, |
| "step": 283, |
| "time_profile/curr_logprobs_and_update": 0.28558273872477, |
| "time_profile/image_rollout": 84.52166864462197, |
| "time_profile/old_logps": 35.65723523311317, |
| "time_profile/ref_logps": 35.67234238050878, |
| "time_profile/reward": 1.3341108299791813, |
| "train/gen_step": 17932.0, |
| "unified/clip_ratio/high_mean": 0.0004901750253338832, |
| "unified/clip_ratio/low_mean": 0.0002691970212254091, |
| "unified/clip_ratio/region_mean": 0.0007593720411023241, |
| "unified/kl": 0.07404309092089534, |
| "unified/loss": -0.005812747418531217 |
| }, |
| { |
| "epoch": 3.602081665332266, |
| "grad_norm": 0.007926159538328648, |
| "learning_rate": 1e-05, |
| "loss": 0.0012, |
| "step": 284, |
| "time_profile/curr_logprobs_and_update": 0.2852334416238591, |
| "train/gen_step": 17996.0, |
| "unified/clip_ratio/high_mean": 1.2283804608159699e-05, |
| "unified/clip_ratio/low_mean": 1.4051259313418996e-05, |
| "unified/clip_ratio/region_mean": 2.6335063921578694e-05, |
| "unified/kl": 0.02951611264143139, |
| "unified/loss": 0.0011699605111061828 |
| }, |
| { |
| "epoch": 3.6148919135308244, |
| "gen/advantages_abs_mean": 0.04824373871088028, |
| "gen/advantages_max": 0.16505038738250732, |
| "gen/advantages_mean": -3.4924596548080444e-09, |
| "gen/advantages_min": -0.2584998607635498, |
| "gen/advantages_std": 0.06254779547452927, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 88.5, |
| "gen/prompt/min_length": 34.0, |
| "gen/reward": 0.7271822690963745, |
| "gen/reward_std": 0.0663105696439743, |
| "grad_norm": 0.006434672512114048, |
| "learning_rate": 1e-05, |
| "loss": 0.0023, |
| "rewards/perceptual_score_reward_func/mean": 0.7265818119049072, |
| "rewards/perceptual_score_reward_func/std": 0.06613854318857193, |
| "step": 285, |
| "time_profile/curr_logprobs_and_update": 0.2933608774037566, |
| "time_profile/image_rollout": 88.68789251707494, |
| "time_profile/old_logps": 35.656789338216186, |
| "time_profile/ref_logps": 35.682189056649804, |
| "time_profile/reward": 1.2701308466494083, |
| "train/gen_step": 18060.0, |
| "unified/clip_ratio/high_mean": 0.00017991956792684505, |
| "unified/clip_ratio/low_mean": 1.3469827536027879e-05, |
| "unified/clip_ratio/region_mean": 0.00019338939546287293, |
| "unified/kl": 0.047669891180703416, |
| "unified/loss": 0.011713605775639735 |
| }, |
| { |
| "epoch": 3.6277021617293834, |
| "grad_norm": 0.0027151338290423155, |
| "learning_rate": 1e-05, |
| "loss": 0.0044, |
| "step": 286, |
| "time_profile/curr_logprobs_and_update": 0.2846964886994101, |
| "train/gen_step": 18124.0, |
| "unified/clip_ratio/high_mean": 0.000592122969464981, |
| "unified/clip_ratio/low_mean": 0.0002473404074407881, |
| "unified/clip_ratio/region_mean": 0.0008394633778152638, |
| "unified/kl": 0.1111522851861082, |
| "unified/loss": 0.004183867895335425 |
| }, |
| { |
| "epoch": 3.6405124099279424, |
| "gen/advantages_abs_mean": 0.0678078830242157, |
| "gen/advantages_max": 0.21553650498390198, |
| "gen/advantages_mean": -4.656612873077393e-10, |
| "gen/advantages_min": -0.2846578359603882, |
| "gen/advantages_std": 0.0855150818824768, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 218.5, |
| "gen/prompt/min_length": 143.0, |
| "gen/reward": 0.39473026990890503, |
| "gen/reward_std": 0.24421042203903198, |
| "grad_norm": 0.0041110278107225895, |
| "learning_rate": 1e-05, |
| "loss": 0.0054, |
| "rewards/perceptual_score_reward_func/mean": 0.4020701050758362, |
| "rewards/perceptual_score_reward_func/std": 0.24086683988571167, |
| "step": 287, |
| "time_profile/curr_logprobs_and_update": 0.28402337976149283, |
| "time_profile/image_rollout": 79.04989433661103, |
| "time_profile/old_logps": 35.72478462010622, |
| "time_profile/ref_logps": 35.819537691771984, |
| "time_profile/reward": 0.9356822613626719, |
| "train/gen_step": 18188.0, |
| "unified/clip_ratio/high_mean": 0.0011436262520874152, |
| "unified/clip_ratio/low_mean": 0.00047107959471759386, |
| "unified/clip_ratio/region_mean": 0.0016147058449860197, |
| "unified/kl": 0.12687812180956826, |
| "unified/loss": 0.00387902719558042 |
| }, |
| { |
| "epoch": 3.653322658126501, |
| "grad_norm": 0.004605242982506752, |
| "learning_rate": 1e-05, |
| "loss": 0.0046, |
| "step": 288, |
| "time_profile/curr_logprobs_and_update": 0.2842605091573205, |
| "train/gen_step": 18252.0, |
| "unified/clip_ratio/high_mean": 0.0002273049494760926, |
| "unified/clip_ratio/low_mean": 7.214290326373884e-05, |
| "unified/clip_ratio/region_mean": 0.00029944785183033673, |
| "unified/kl": 0.1161597307655029, |
| "unified/loss": 0.004344568809756311 |
| }, |
| { |
| "epoch": 3.66613290632506, |
| "gen/advantages_abs_mean": 0.06362708657979965, |
| "gen/advantages_max": 0.24109137058258057, |
| "gen/advantages_mean": 1.6298145055770874e-09, |
| "gen/advantages_min": -0.26409316062927246, |
| "gen/advantages_std": 0.07972931861877441, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 137.0, |
| "gen/prompt/mean_length": 137.0, |
| "gen/prompt/min_length": 137.0, |
| "gen/reward": 0.36953791975975037, |
| "gen/reward_std": 0.18224549293518066, |
| "grad_norm": 0.005123942159116268, |
| "learning_rate": 1e-05, |
| "loss": 0.0054, |
| "rewards/perceptual_score_reward_func/mean": 0.35553839802742004, |
| "rewards/perceptual_score_reward_func/std": 0.17481675744056702, |
| "step": 289, |
| "time_profile/curr_logprobs_and_update": 0.29193573028896935, |
| "time_profile/image_rollout": 85.58525138907135, |
| "time_profile/old_logps": 35.699933636933565, |
| "time_profile/ref_logps": 35.74023015238345, |
| "time_profile/reward": 1.5181463621556759, |
| "train/gen_step": 18316.0, |
| "unified/clip_ratio/high_mean": 0.01066688062655885, |
| "unified/clip_ratio/low_mean": 0.0003187119891663315, |
| "unified/clip_ratio/region_mean": 0.010985592674842337, |
| "unified/kl": 0.10665562597569078, |
| "unified/loss": 0.005077178055216791 |
| }, |
| { |
| "epoch": 3.678943154523619, |
| "grad_norm": 0.004347388166934252, |
| "learning_rate": 1e-05, |
| "loss": 0.0027, |
| "step": 290, |
| "time_profile/curr_logprobs_and_update": 0.2874660540255718, |
| "train/gen_step": 18380.0, |
| "unified/clip_ratio/high_mean": 0.00016701010554243112, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.00016701010554243112, |
| "unified/kl": 0.06843785307137296, |
| "unified/loss": 0.0026681148301577196 |
| }, |
| { |
| "epoch": 3.6917534027221777, |
| "gen/advantages_abs_mean": 0.023959465324878693, |
| "gen/advantages_max": 0.1302698850631714, |
| "gen/advantages_mean": -6.984919309616089e-10, |
| "gen/advantages_min": -0.21591639518737793, |
| "gen/advantages_std": 0.042961977422237396, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.453125, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 90.5, |
| "gen/prompt/min_length": 38.0, |
| "gen/reward": 0.3222031593322754, |
| "gen/reward_std": 0.3252832293510437, |
| "grad_norm": 0.0031189958099275827, |
| "learning_rate": 1e-05, |
| "loss": 0.0017, |
| "rewards/perceptual_score_reward_func/mean": 0.3289386034011841, |
| "rewards/perceptual_score_reward_func/std": 0.3337442874908447, |
| "step": 291, |
| "time_profile/curr_logprobs_and_update": 0.28930792823666707, |
| "time_profile/image_rollout": 89.22134350053966, |
| "time_profile/old_logps": 35.68188642151654, |
| "time_profile/ref_logps": 35.70580958202481, |
| "time_profile/reward": 1.0724818594753742, |
| "train/gen_step": 18444.0, |
| "unified/clip_ratio/high_mean": 0.006382234801094455, |
| "unified/clip_ratio/low_mean": 0.0002502836205167114, |
| "unified/clip_ratio/region_mean": 0.006632518420701672, |
| "unified/kl": 0.07385592849459499, |
| "unified/loss": 0.00274933292712376 |
| }, |
| { |
| "epoch": 3.7045636509207367, |
| "grad_norm": 0.0009837422985583544, |
| "learning_rate": 1e-05, |
| "loss": 0.0035, |
| "step": 292, |
| "time_profile/curr_logprobs_and_update": 0.2856500959896948, |
| "train/gen_step": 18508.0, |
| "unified/clip_ratio/high_mean": 0.0016065689296738128, |
| "unified/clip_ratio/low_mean": 5.288676311465679e-05, |
| "unified/clip_ratio/region_mean": 0.001659455691878975, |
| "unified/kl": 0.0873052715905942, |
| "unified/loss": 0.0035271293800178682 |
| }, |
| { |
| "epoch": 3.7173738991192953, |
| "gen/advantages_abs_mean": 0.032585710287094116, |
| "gen/advantages_max": 0.1677337884902954, |
| "gen/advantages_mean": -2.3283064365386963e-09, |
| "gen/advantages_min": -0.19779980182647705, |
| "gen/advantages_std": 0.052031490951776505, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.21875, |
| "gen/prompt/max_length": 228.0, |
| "gen/prompt/mean_length": 126.5, |
| "gen/prompt/min_length": 25.0, |
| "gen/reward": 0.3170202672481537, |
| "gen/reward_std": 0.3172413110733032, |
| "grad_norm": 0.0009276257478632033, |
| "learning_rate": 1e-05, |
| "loss": 0.0036, |
| "rewards/perceptual_score_reward_func/mean": 0.33117929100990295, |
| "rewards/perceptual_score_reward_func/std": 0.32941934466362, |
| "step": 293, |
| "time_profile/curr_logprobs_and_update": 0.2861288280109875, |
| "time_profile/image_rollout": 88.20444161072373, |
| "time_profile/old_logps": 35.684086905792356, |
| "time_profile/ref_logps": 35.73149266280234, |
| "time_profile/reward": 1.3472717106342316, |
| "train/gen_step": 18572.0, |
| "unified/clip_ratio/high_mean": 0.00624829082062206, |
| "unified/clip_ratio/low_mean": 0.00016511899048055056, |
| "unified/clip_ratio/region_mean": 0.0064134098020076635, |
| "unified/kl": 0.07742297963704914, |
| "unified/loss": 0.003311360438601696 |
| }, |
| { |
| "epoch": 3.7301841473178543, |
| "grad_norm": 0.0010312836384400725, |
| "learning_rate": 1e-05, |
| "loss": 0.002, |
| "step": 294, |
| "time_profile/curr_logprobs_and_update": 0.28893115191021934, |
| "train/gen_step": 18636.0, |
| "unified/clip_ratio/high_mean": 0.00042864468014158774, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.00042864468014158774, |
| "unified/kl": 0.051507175172446296, |
| "unified/loss": 0.0020400361800057 |
| }, |
| { |
| "epoch": 3.742994395516413, |
| "gen/advantages_abs_mean": 0.06168805807828903, |
| "gen/advantages_max": 0.2514798939228058, |
| "gen/advantages_mean": -1.6298145055770874e-09, |
| "gen/advantages_min": -0.31687089800834656, |
| "gen/advantages_std": 0.08253538608551025, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 294.0, |
| "gen/prompt/mean_length": 165.5, |
| "gen/prompt/min_length": 37.0, |
| "gen/reward": 0.6195996403694153, |
| "gen/reward_std": 0.17864473164081573, |
| "grad_norm": 0.001671893522143364, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "rewards/perceptual_score_reward_func/mean": 0.6047117114067078, |
| "rewards/perceptual_score_reward_func/std": 0.18849465250968933, |
| "step": 295, |
| "time_profile/curr_logprobs_and_update": 0.2885678320308216, |
| "time_profile/image_rollout": 80.15901150368154, |
| "time_profile/old_logps": 35.669525284320116, |
| "time_profile/ref_logps": 35.711911994963884, |
| "time_profile/reward": 1.5623396970331669, |
| "train/gen_step": 18700.0, |
| "unified/clip_ratio/high_mean": 0.0023336565727731795, |
| "unified/clip_ratio/low_mean": 7.46693549444899e-05, |
| "unified/clip_ratio/region_mean": 0.002408325938631606, |
| "unified/kl": 0.05192991314106621, |
| "unified/loss": 0.00448986226720649 |
| }, |
| { |
| "epoch": 3.755804643714972, |
| "grad_norm": 0.0020732299890369177, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "step": 296, |
| "time_profile/curr_logprobs_and_update": 0.2906850943691097, |
| "train/gen_step": 18764.0, |
| "unified/clip_ratio/high_mean": 0.0004890980990239768, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 0.0004890980990239768, |
| "unified/kl": 0.05222447955748066, |
| "unified/loss": 0.002113723692673375 |
| }, |
| { |
| "epoch": 3.768614891913531, |
| "gen/advantages_abs_mean": 0.05659784376621246, |
| "gen/advantages_max": 0.30854809284210205, |
| "gen/advantages_mean": 2.9103830456733704e-11, |
| "gen/advantages_min": -0.14766725897789001, |
| "gen/advantages_std": 0.0727003961801529, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 31.0, |
| "gen/prompt/mean_length": 30.0, |
| "gen/prompt/min_length": 29.0, |
| "gen/reward": 0.0630592554807663, |
| "gen/reward_std": 0.07766459882259369, |
| "grad_norm": 0.002387274755164981, |
| "learning_rate": 1e-05, |
| "loss": 0.0021, |
| "rewards/perceptual_score_reward_func/mean": 0.07399304956197739, |
| "rewards/perceptual_score_reward_func/std": 0.07755821943283081, |
| "step": 297, |
| "time_profile/curr_logprobs_and_update": 0.28434587141964585, |
| "time_profile/image_rollout": 93.55125546082854, |
| "time_profile/old_logps": 35.63593592122197, |
| "time_profile/ref_logps": 35.681266540661454, |
| "time_profile/reward": 1.2651427760720253, |
| "train/gen_step": 18828.0, |
| "unified/clip_ratio/high_mean": 0.002428019828585093, |
| "unified/clip_ratio/low_mean": 2.390239114902215e-05, |
| "unified/clip_ratio/region_mean": 0.0024519222188246204, |
| "unified/kl": 0.04881385323824361, |
| "unified/loss": -0.0021698455511796055 |
| }, |
| { |
| "epoch": 3.7814251401120895, |
| "grad_norm": 0.0029673695098608732, |
| "learning_rate": 1e-05, |
| "loss": 0.0016, |
| "step": 298, |
| "time_profile/curr_logprobs_and_update": 0.2842209104564972, |
| "train/gen_step": 18892.0, |
| "unified/clip_ratio/high_mean": 2.6307398002245463e-05, |
| "unified/clip_ratio/low_mean": 0.0, |
| "unified/clip_ratio/region_mean": 2.6307398002245463e-05, |
| "unified/kl": 0.041427473712246865, |
| "unified/loss": 0.0016878378992259968 |
| }, |
| { |
| "epoch": 3.7942353883106485, |
| "gen/advantages_abs_mean": 0.039250317960977554, |
| "gen/advantages_max": 0.1470203995704651, |
| "gen/advantages_mean": 4.656612873077393e-10, |
| "gen/advantages_min": -0.23213261365890503, |
| "gen/advantages_std": 0.05306250602006912, |
| "gen/completion/max_length": 1024.0, |
| "gen/completion/mean_length": 1024.0, |
| "gen/completion/min_length": 1024.0, |
| "gen/frac_reward_zero_std": 0.0, |
| "gen/prompt/max_length": 143.0, |
| "gen/prompt/mean_length": 89.0, |
| "gen/prompt/min_length": 35.0, |
| "gen/reward": 0.7142139673233032, |
| "gen/reward_std": 0.13610295951366425, |
| "grad_norm": 0.0023270256351679564, |
| "learning_rate": 1e-05, |
| "loss": 0.0026, |
| "rewards/perceptual_score_reward_func/mean": 0.7127842903137207, |
| "rewards/perceptual_score_reward_func/std": 0.14060144126415253, |
| "step": 299, |
| "time_profile/curr_logprobs_and_update": 0.28757855866570026, |
| "time_profile/image_rollout": 88.57772912271321, |
| "time_profile/old_logps": 35.677918802946806, |
| "time_profile/ref_logps": 35.72353080287576, |
| "time_profile/reward": 1.1193790771067142, |
| "train/gen_step": 18956.0, |
| "unified/clip_ratio/high_mean": 6.198136179591529e-05, |
| "unified/clip_ratio/low_mean": 1.169535971712321e-05, |
| "unified/clip_ratio/region_mean": 7.36767215130385e-05, |
| "unified/kl": 0.05895602604141459, |
| "unified/loss": 0.0031285112600016873 |
| }, |
| { |
| "epoch": 3.8070456365092076, |
| "grad_norm": 0.001265124068595469, |
| "learning_rate": 1e-05, |
| "loss": 0.0047, |
| "step": 300, |
| "time_profile/curr_logprobs_and_update": 0.2926370550703723, |
| "train/gen_step": 19020.0, |
| "unified/clip_ratio/high_mean": 0.00042617201506800484, |
| "unified/clip_ratio/low_mean": 2.537912587285973e-05, |
| "unified/clip_ratio/region_mean": 0.00045155114094086457, |
| "unified/kl": 0.11658903432544321, |
| "unified/loss": 0.004803504161827732 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 790, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 50, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
