{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 3.8070456365092076, "eval_steps": 500, "global_step": 300, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.012810248198558846, "gen/advantages_abs_mean": 0.052109457552433014, "gen/advantages_max": 0.13163888454437256, "gen/advantages_mean": 3.026798367500305e-09, "gen/advantages_min": -0.2830061912536621, "gen/advantages_std": 0.06655973941087723, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 88.0, "gen/prompt/min_length": 33.0, "gen/reward": 0.7407832741737366, "gen/reward_std": 0.07185085862874985, "grad_norm": 0.004070668946951628, "learning_rate": 0.0, "loss": -0.0, "rewards/perceptual_score_reward_func/mean": 0.7447192668914795, "rewards/perceptual_score_reward_func/std": 0.06898166239261627, "step": 1, "time_profile/curr_logprobs_and_update": 0.28362782069598325, "time_profile/image_rollout": 95.28615356422961, "time_profile/old_logps": 35.66134166903794, "time_profile/ref_logps": 35.66421937197447, "time_profile/reward": 4.256949637085199, "train/gen_step": 64.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.0, "unified/loss": -1.0550138540565968e-08 }, { "epoch": 0.025620496397117692, "grad_norm": 0.0038007174152880907, "learning_rate": 1e-05, "loss": -0.0, "step": 2, "time_profile/curr_logprobs_and_update": 0.2893671114288736, "train/gen_step": 128.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.0, "unified/loss": -1.1525116860866547e-08 }, { "epoch": 0.03843074459567654, "gen/advantages_abs_mean": 0.06179381534457207, "gen/advantages_max": 0.2224750518798828, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.27374815940856934, "gen/advantages_std": 0.07891622185707092, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 182.5, "gen/prompt/min_length": 137.0, "gen/reward": 0.4988939166069031, "gen/reward_std": 0.23086503148078918, "grad_norm": 0.004844950512051582, "learning_rate": 1e-05, "loss": 0.0, "rewards/perceptual_score_reward_func/mean": 0.4858173727989197, "rewards/perceptual_score_reward_func/std": 0.2394656240940094, "step": 3, "time_profile/curr_logprobs_and_update": 0.2888016360811889, "time_profile/image_rollout": 85.37720386311412, "time_profile/old_logps": 35.70834754779935, "time_profile/ref_logps": 35.81470891647041, "time_profile/reward": 1.1268615759909153, "train/gen_step": 192.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.0011604064757193555, "unified/loss": 4.6893979742890224e-05 }, { "epoch": 0.051240992794235385, "grad_norm": 0.00484450301155448, "learning_rate": 1e-05, "loss": 0.0, "step": 4, "time_profile/curr_logprobs_and_update": 0.2927547112049069, "train/gen_step": 256.0, "unified/clip_ratio/high_mean": 0.0007498142977055977, "unified/clip_ratio/low_mean": 7.392094084934797e-05, "unified/clip_ratio/region_mean": 0.0008237352385549457, "unified/kl": 0.0015762942311994266, "unified/loss": -1.2426604371285066e-05 }, { "epoch": 0.06405124099279423, "gen/advantages_abs_mean": 0.05059637129306793, "gen/advantages_max": 0.29003840684890747, "gen/advantages_mean": -4.94765117764473e-10, "gen/advantages_min": -0.2061920166015625, "gen/advantages_std": 0.07026641815900803, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 133.5, "gen/prompt/min_length": 39.0, "gen/reward": 0.5134599208831787, "gen/reward_std": 0.2526681423187256, "grad_norm": 0.0043946485966444016, "learning_rate": 1e-05, "loss": 0.0001, "rewards/perceptual_score_reward_func/mean": 0.5262137055397034, "rewards/perceptual_score_reward_func/std": 0.2509992718696594, "step": 5, "time_profile/curr_logprobs_and_update": 0.2874946145748254, "time_profile/image_rollout": 93.83373009413481, "time_profile/old_logps": 35.67024562694132, "time_profile/ref_logps": 35.715081388130784, "time_profile/reward": 5.443569419905543, "train/gen_step": 320.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.0018907583944383077, "unified/loss": 7.613089474034496e-05 }, { "epoch": 0.07686148919135308, "grad_norm": 0.004030726384371519, "learning_rate": 1e-05, "loss": 0.0001, "step": 6, "time_profile/curr_logprobs_and_update": 0.2908289354527369, "train/gen_step": 384.0, "unified/clip_ratio/high_mean": 0.0009645656609791331, "unified/clip_ratio/low_mean": 0.00019809185232588788, "unified/clip_ratio/region_mean": 0.0011626575096670422, "unified/kl": 0.002805404281389201, "unified/loss": 0.00010875017323996872 }, { "epoch": 0.08967173738991192, "gen/advantages_abs_mean": 0.05144927650690079, "gen/advantages_max": 0.15652674436569214, "gen/advantages_mean": 1.1641532182693481e-09, "gen/advantages_min": -0.3314337134361267, "gen/advantages_std": 0.06792663037776947, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 88.0, "gen/prompt/min_length": 33.0, "gen/reward": 0.7259606122970581, "gen/reward_std": 0.11048998683691025, "grad_norm": 0.004164206329733133, "learning_rate": 1e-05, "loss": 0.0004, "rewards/perceptual_score_reward_func/mean": 0.7273693084716797, "rewards/perceptual_score_reward_func/std": 0.10635492205619812, "step": 7, "time_profile/curr_logprobs_and_update": 0.28579148260178044, "time_profile/image_rollout": 93.78808394446969, "time_profile/old_logps": 35.66581052169204, "time_profile/ref_logps": 35.73940089531243, "time_profile/reward": 1.449136609211564, "train/gen_step": 448.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.010006293836340774, "unified/loss": 0.0003717760555446148 }, { "epoch": 0.10248198558847077, "grad_norm": 0.003952718339860439, "learning_rate": 1e-05, "loss": 0.0006, "step": 8, "time_profile/curr_logprobs_and_update": 0.2963303836877458, "train/gen_step": 512.0, "unified/clip_ratio/high_mean": 0.0006130620704425382, "unified/clip_ratio/low_mean": 0.002367428182878939, "unified/clip_ratio/region_mean": 0.0029804902487740037, "unified/kl": 0.015264490411937004, "unified/loss": 0.0006416451851691818 }, { "epoch": 0.11529223378702963, "gen/advantages_abs_mean": 0.08506780862808228, "gen/advantages_max": 0.3090728223323822, "gen/advantages_mean": 1.6880221664905548e-09, "gen/advantages_min": -0.27506160736083984, "gen/advantages_std": 0.10490497946739197, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 137.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.25868576765060425, "gen/reward_std": 0.11191420257091522, "grad_norm": 0.006360508035868406, "learning_rate": 1e-05, "loss": 0.0001, "rewards/perceptual_score_reward_func/mean": 0.25572726130485535, "rewards/perceptual_score_reward_func/std": 0.11308468878269196, "step": 9, "time_profile/curr_logprobs_and_update": 0.2922193466220051, "time_profile/image_rollout": 77.53796414472163, "time_profile/old_logps": 35.67887348122895, "time_profile/ref_logps": 35.77796713076532, "time_profile/reward": 1.6198791358619928, "train/gen_step": 576.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.003742291562957689, "unified/loss": 0.00015398636605823413 }, { "epoch": 0.12810248198558846, "grad_norm": 0.006031422410160303, "learning_rate": 1e-05, "loss": 0.0002, "step": 10, "time_profile/curr_logprobs_and_update": 0.28741057447041385, "train/gen_step": 640.0, "unified/clip_ratio/high_mean": 0.00038488462996610906, "unified/clip_ratio/low_mean": 4.0186700971389655e-05, "unified/clip_ratio/region_mean": 0.0004250713309374987, "unified/kl": 0.004935910466883797, "unified/loss": 0.00012804145080735907 }, { "epoch": 0.14091273018414732, "gen/advantages_abs_mean": 0.06213773041963577, "gen/advantages_max": 0.23558755218982697, "gen/advantages_mean": -1.4260876923799515e-09, "gen/advantages_min": -0.23738695681095123, "gen/advantages_std": 0.0792107805609703, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 30.0, "gen/prompt/mean_length": 30.0, "gen/prompt/min_length": 30.0, "gen/reward": 0.2623980641365051, "gen/reward_std": 0.10127276927232742, "grad_norm": 0.00370898493565619, "learning_rate": 1e-05, "loss": 0.0001, "rewards/perceptual_score_reward_func/mean": 0.27432122826576233, "rewards/perceptual_score_reward_func/std": 0.0917019173502922, "step": 11, "time_profile/curr_logprobs_and_update": 0.29560886629042216, "time_profile/image_rollout": 94.6510800216347, "time_profile/old_logps": 35.63860863447189, "time_profile/ref_logps": 35.675177259370685, "time_profile/reward": 1.475969910621643, "train/gen_step": 704.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.0035910366859752685, "unified/loss": 0.00014697932192575536 }, { "epoch": 0.15372297838270615, "grad_norm": 0.00393424229696393, "learning_rate": 1e-05, "loss": 0.0002, "step": 12, "time_profile/curr_logprobs_and_update": 0.28807597383274697, "train/gen_step": 768.0, "unified/clip_ratio/high_mean": 0.0004313259105401812, "unified/clip_ratio/low_mean": 2.7469435735838488e-05, "unified/clip_ratio/region_mean": 0.0004587953462760197, "unified/kl": 0.0045709875666943844, "unified/loss": 0.00014194081450114027 }, { "epoch": 0.16653322658126501, "gen/advantages_abs_mean": 0.05380730703473091, "gen/advantages_max": 0.32686302065849304, "gen/advantages_mean": 3.259629011154175e-09, "gen/advantages_min": -0.1513519287109375, "gen/advantages_std": 0.0732535719871521, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 36.0, "gen/prompt/mean_length": 32.0, "gen/prompt/min_length": 28.0, "gen/reward": 0.4303140640258789, "gen/reward_std": 0.35259875655174255, "grad_norm": 0.003100387519225478, "learning_rate": 1e-05, "loss": 0.0002, "rewards/perceptual_score_reward_func/mean": 0.43855977058410645, "rewards/perceptual_score_reward_func/std": 0.3576069474220276, "step": 13, "time_profile/curr_logprobs_and_update": 0.2939714658714365, "time_profile/image_rollout": 90.3282467238605, "time_profile/old_logps": 35.644943645223975, "time_profile/ref_logps": 35.6841149572283, "time_profile/reward": 1.4403626210987568, "train/gen_step": 832.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.00593089139147196, "unified/loss": 0.00022935521064937348 }, { "epoch": 0.17934347477982385, "grad_norm": 0.0031230556778609753, "learning_rate": 1e-05, "loss": 0.0003, "step": 14, "time_profile/curr_logprobs_and_update": 0.28601749025983736, "train/gen_step": 896.0, "unified/clip_ratio/high_mean": 0.00032966266735456884, "unified/clip_ratio/low_mean": 1.3754401152255014e-05, "unified/clip_ratio/region_mean": 0.00034341706850682385, "unified/kl": 0.00694357078828034, "unified/loss": 0.00020494235377555015 }, { "epoch": 0.1921537229783827, "gen/advantages_abs_mean": 0.035464800894260406, "gen/advantages_max": 0.18103191256523132, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.16873770952224731, "gen/advantages_std": 0.047589611262083054, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 32.0, "gen/prompt/mean_length": 30.5, "gen/prompt/min_length": 29.0, "gen/reward": 0.4555138349533081, "gen/reward_std": 0.3522937297821045, "grad_norm": 0.0038101491518318653, "learning_rate": 1e-05, "loss": 0.0003, "rewards/perceptual_score_reward_func/mean": 0.4537283182144165, "rewards/perceptual_score_reward_func/std": 0.3564547300338745, "step": 15, "time_profile/curr_logprobs_and_update": 0.2867734569008462, "time_profile/image_rollout": 93.33716715313494, "time_profile/old_logps": 35.64802477508783, "time_profile/ref_logps": 35.67770854756236, "time_profile/reward": 1.3251771815121174, "train/gen_step": 960.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.007522514912125189, "unified/loss": 0.00029024387777099037 }, { "epoch": 0.20496397117694154, "grad_norm": 0.003709500189870596, "learning_rate": 1e-05, "loss": 0.0004, "step": 16, "time_profile/curr_logprobs_and_update": 0.2836496669333428, "train/gen_step": 1024.0, "unified/clip_ratio/high_mean": 0.0001654277248235303, "unified/clip_ratio/low_mean": 0.00014177265802572947, "unified/clip_ratio/region_mean": 0.00030720038193976507, "unified/kl": 0.009284940926590934, "unified/loss": 0.0003907864602297195 }, { "epoch": 0.2177742193755004, "gen/advantages_abs_mean": 0.06693125516176224, "gen/advantages_max": 0.2859118580818176, "gen/advantages_mean": -2.3283064365386963e-09, "gen/advantages_min": -0.248762309551239, "gen/advantages_std": 0.08490201085805893, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 26.0, "gen/prompt/mean_length": 25.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.3253200650215149, "gen/reward_std": 0.12534543871879578, "grad_norm": 0.003804264822974801, "learning_rate": 1e-05, "loss": 0.0004, "rewards/perceptual_score_reward_func/mean": 0.30608081817626953, "rewards/perceptual_score_reward_func/std": 0.1369846761226654, "step": 17, "time_profile/curr_logprobs_and_update": 0.28391146194189787, "time_profile/image_rollout": 91.10511220991611, "time_profile/old_logps": 35.609973045066, "time_profile/ref_logps": 35.6577035933733, "time_profile/reward": 1.3727597426623106, "train/gen_step": 1088.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.00982376610772917, "unified/loss": 0.00041090739432547707 }, { "epoch": 0.23058446757405926, "grad_norm": 0.0037233256734907627, "learning_rate": 1e-05, "loss": 0.0004, "step": 18, "time_profile/curr_logprobs_and_update": 0.2903437889472116, "train/gen_step": 1152.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.01129970328474883, "unified/loss": 0.00045823437358194496 }, { "epoch": 0.2433947157726181, "gen/advantages_abs_mean": 0.055955708026885986, "gen/advantages_max": 0.2999653220176697, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.25927937030792236, "gen/advantages_std": 0.07066643238067627, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 34.0, "gen/prompt/mean_length": 33.0, "gen/prompt/min_length": 32.0, "gen/reward": 0.5881402492523193, "gen/reward_std": 0.16526837646961212, "grad_norm": 0.004073821473866701, "learning_rate": 1e-05, "loss": 0.0005, "rewards/perceptual_score_reward_func/mean": 0.5861481428146362, "rewards/perceptual_score_reward_func/std": 0.16784866154193878, "step": 19, "time_profile/curr_logprobs_and_update": 0.2822369273926597, "time_profile/image_rollout": 88.6840718369931, "time_profile/old_logps": 35.64870150946081, "time_profile/ref_logps": 35.687422059476376, "time_profile/reward": 1.4169904831796885, "train/gen_step": 1216.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.013576237324741669, "unified/loss": 0.0005496463236340787 }, { "epoch": 0.2562049639711769, "grad_norm": 0.0040005045011639595, "learning_rate": 1e-05, "loss": 0.0006, "step": 20, "time_profile/curr_logprobs_and_update": 0.28987074297037907, "train/gen_step": 1280.0, "unified/clip_ratio/high_mean": 0.0002034804583672667, "unified/clip_ratio/low_mean": 9.962243984773522e-05, "unified/clip_ratio/region_mean": 0.0003031028982150019, "unified/kl": 0.014395568068721332, "unified/loss": 0.0005391188487919862 }, { "epoch": 0.2690152121697358, "gen/advantages_abs_mean": 0.06234389543533325, "gen/advantages_max": 0.23057347536087036, "gen/advantages_mean": -1.862645149230957e-09, "gen/advantages_min": -0.2955120801925659, "gen/advantages_std": 0.08112785965204239, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 185.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.5955153703689575, "gen/reward_std": 0.19560492038726807, "grad_norm": 0.006979112047702074, "learning_rate": 1e-05, "loss": 0.0062, "rewards/perceptual_score_reward_func/mean": 0.5835608839988708, "rewards/perceptual_score_reward_func/std": 0.1863791048526764, "step": 21, "time_profile/curr_logprobs_and_update": 0.28707319442764856, "time_profile/image_rollout": 89.13379067741334, "time_profile/old_logps": 35.708857618272305, "time_profile/ref_logps": 35.791965547949076, "time_profile/reward": 1.1867827828973532, "train/gen_step": 1344.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.15566039714030921, "unified/loss": 0.005685318652922433 }, { "epoch": 0.28182546036829464, "grad_norm": 0.00766932126134634, "learning_rate": 1e-05, "loss": 0.0052, "step": 22, "time_profile/curr_logprobs_and_update": 0.2838307637430262, "train/gen_step": 1408.0, "unified/clip_ratio/high_mean": 0.010980058965287753, "unified/clip_ratio/low_mean": 0.00033267143953708, "unified/clip_ratio/region_mean": 0.011312730418467254, "unified/kl": 0.13134970283135772, "unified/loss": 0.005792835259853746 }, { "epoch": 0.2946357085668535, "gen/advantages_abs_mean": 0.05016437917947769, "gen/advantages_max": 0.2511361837387085, "gen/advantages_mean": -2.0372681319713593e-10, "gen/advantages_min": -0.2357330471277237, "gen/advantages_std": 0.06782188266515732, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 49.0, "gen/prompt/mean_length": 38.0, "gen/prompt/min_length": 27.0, "gen/reward": 0.48785874247550964, "gen/reward_std": 0.313375324010849, "grad_norm": 0.003058127360418439, "learning_rate": 1e-05, "loss": 0.0007, "rewards/perceptual_score_reward_func/mean": 0.4932803511619568, "rewards/perceptual_score_reward_func/std": 0.3043711483478546, "step": 23, "time_profile/curr_logprobs_and_update": 0.2894766298995819, "time_profile/image_rollout": 90.673310758546, "time_profile/old_logps": 35.6381083894521, "time_profile/ref_logps": 35.683547265827656, "time_profile/reward": 1.3350308034569025, "train/gen_step": 1472.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.018112926394678652, "unified/loss": 0.0007296701951418072 }, { "epoch": 0.3074459567654123, "grad_norm": 0.003112577134743333, "learning_rate": 1e-05, "loss": 0.0007, "step": 24, "time_profile/curr_logprobs_and_update": 0.28356376514420845, "train/gen_step": 1536.0, "unified/clip_ratio/high_mean": 0.00036169634040561505, "unified/clip_ratio/low_mean": 1.2056327250320464e-05, "unified/clip_ratio/region_mean": 0.0003737526712939143, "unified/kl": 0.018430211246595718, "unified/loss": 0.0007105697004590183 }, { "epoch": 0.32025620496397117, "gen/advantages_abs_mean": 0.06575670838356018, "gen/advantages_max": 0.19591188430786133, "gen/advantages_mean": -3.4924596548080444e-09, "gen/advantages_min": -0.37458205223083496, "gen/advantages_std": 0.08331269770860672, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 90.5, "gen/prompt/min_length": 38.0, "gen/reward": 0.7573187351226807, "gen/reward_std": 0.08858481794595718, "grad_norm": 0.004597505554556847, "learning_rate": 1e-05, "loss": 0.0012, "rewards/perceptual_score_reward_func/mean": 0.7637672424316406, "rewards/perceptual_score_reward_func/std": 0.07734499126672745, "step": 25, "time_profile/curr_logprobs_and_update": 0.28622239985270426, "time_profile/image_rollout": 86.45065759681165, "time_profile/old_logps": 35.67832253314555, "time_profile/ref_logps": 35.702019242569804, "time_profile/reward": 1.359033975750208, "train/gen_step": 1600.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.029347103234613314, "unified/loss": 0.0012196608440717682 }, { "epoch": 0.33306645316253003, "grad_norm": 0.0043387808836996555, "learning_rate": 1e-05, "loss": 0.001, "step": 26, "time_profile/curr_logprobs_and_update": 0.28386400899034925, "train/gen_step": 1664.0, "unified/clip_ratio/high_mean": 0.01954531196224707, "unified/clip_ratio/low_mean": 0.0003240542173443828, "unified/clip_ratio/region_mean": 0.019869366309649195, "unified/kl": 0.023536839376902208, "unified/loss": 0.0010066187054320608 }, { "epoch": 0.3458767013610889, "gen/advantages_abs_mean": 0.057586055248975754, "gen/advantages_max": 0.21371035277843475, "gen/advantages_mean": -1.1641532182693481e-09, "gen/advantages_min": -0.2889488935470581, "gen/advantages_std": 0.08002809435129166, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 40.0, "gen/prompt/mean_length": 33.0, "gen/prompt/min_length": 26.0, "gen/reward": 0.5128387212753296, "gen/reward_std": 0.3075273931026459, "grad_norm": 0.0035108160227537155, "learning_rate": 1e-05, "loss": 0.0007, "rewards/perceptual_score_reward_func/mean": 0.5264440774917603, "rewards/perceptual_score_reward_func/std": 0.3053143322467804, "step": 27, "time_profile/curr_logprobs_and_update": 0.28258850090787746, "time_profile/image_rollout": 93.14971325173974, "time_profile/old_logps": 35.633352022618055, "time_profile/ref_logps": 35.67252997867763, "time_profile/reward": 1.3629055880010128, "train/gen_step": 1728.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.017757456327672116, "unified/loss": 0.0006818042947998038 }, { "epoch": 0.3586869495596477, "grad_norm": 0.003482175525277853, "learning_rate": 1e-05, "loss": 0.0007, "step": 28, "time_profile/curr_logprobs_and_update": 0.29045128886355087, "train/gen_step": 1792.0, "unified/clip_ratio/high_mean": 0.003981999492680188, "unified/clip_ratio/low_mean": 6.156700783321867e-05, "unified/clip_ratio/region_mean": 0.004043566495965933, "unified/kl": 0.016798593336716294, "unified/loss": 0.0006475578447862063 }, { "epoch": 0.37149719775820655, "gen/advantages_abs_mean": 0.020567672327160835, "gen/advantages_max": 0.0832895040512085, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.09890902042388916, "gen/advantages_std": 0.0276055708527565, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 35.0, "gen/prompt/mean_length": 34.5, "gen/prompt/min_length": 34.0, "gen/reward": 0.8026696443557739, "gen/reward_std": 0.035780273377895355, "grad_norm": 0.001524417893961072, "learning_rate": 1e-05, "loss": 0.0008, "rewards/perceptual_score_reward_func/mean": 0.7995024919509888, "rewards/perceptual_score_reward_func/std": 0.039139553904533386, "step": 29, "time_profile/curr_logprobs_and_update": 0.28697867854498327, "time_profile/image_rollout": 94.98372022993863, "time_profile/old_logps": 35.64544406160712, "time_profile/ref_logps": 35.6720716599375, "time_profile/reward": 1.554461432620883, "train/gen_step": 1856.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.020413192978594452, "unified/loss": 0.0008108863465707827 }, { "epoch": 0.3843074459567654, "grad_norm": 0.001496512326411903, "learning_rate": 1e-05, "loss": 0.0008, "step": 30, "time_profile/curr_logprobs_and_update": 0.28794672252843156, "train/gen_step": 1920.0, "unified/clip_ratio/high_mean": 0.00028818798364227405, "unified/clip_ratio/low_mean": 0.0001083762999769533, "unified/clip_ratio/region_mean": 0.00039656428452872206, "unified/kl": 0.020366095268400386, "unified/loss": 0.000806004970286267 }, { "epoch": 0.3971176941553243, "gen/advantages_abs_mean": 0.07551919668912888, "gen/advantages_max": 0.24253734946250916, "gen/advantages_mean": -2.561137080192566e-09, "gen/advantages_min": -0.31094181537628174, "gen/advantages_std": 0.09551571309566498, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 89.0, "gen/prompt/min_length": 35.0, "gen/reward": 0.4983167052268982, "gen/reward_std": 0.2392866164445877, "grad_norm": 0.004808772820979357, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.5010837316513062, "rewards/perceptual_score_reward_func/std": 0.2483944445848465, "step": 31, "time_profile/curr_logprobs_and_update": 0.2927919552021194, "time_profile/image_rollout": 92.86078766733408, "time_profile/old_logps": 35.645628007128835, "time_profile/ref_logps": 35.68816146440804, "time_profile/reward": 1.1220357697457075, "train/gen_step": 1984.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.028540802421048284, "unified/loss": 0.00118683417849752 }, { "epoch": 0.4099279423538831, "grad_norm": 0.004993099253624678, "learning_rate": 1e-05, "loss": 0.0013, "step": 32, "time_profile/curr_logprobs_and_update": 0.2836689332325477, "train/gen_step": 2048.0, "unified/clip_ratio/high_mean": 0.004601992815878475, "unified/clip_ratio/low_mean": 8.766420523897978e-05, "unified/clip_ratio/region_mean": 0.004689657042035833, "unified/kl": 0.0318972848035628, "unified/loss": 0.0012409869095790782 }, { "epoch": 0.42273819055244194, "gen/advantages_abs_mean": 0.056175872683525085, "gen/advantages_max": 0.2621963918209076, "gen/advantages_mean": 1.3969838619232178e-09, "gen/advantages_min": -0.3464314341545105, "gen/advantages_std": 0.07949691265821457, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 85.5, "gen/prompt/min_length": 28.0, "gen/reward": 0.5262117385864258, "gen/reward_std": 0.21805663406848907, "grad_norm": 0.003093303646892309, "learning_rate": 1e-05, "loss": 0.0013, "rewards/perceptual_score_reward_func/mean": 0.5296328663825989, "rewards/perceptual_score_reward_func/std": 0.2171928584575653, "step": 33, "time_profile/curr_logprobs_and_update": 0.28117132117040455, "time_profile/image_rollout": 94.87426543608308, "time_profile/old_logps": 35.6621759980917, "time_profile/ref_logps": 35.68978282995522, "time_profile/reward": 1.123130239546299, "train/gen_step": 2112.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.03187064320081845, "unified/loss": 0.0012426345692801988 }, { "epoch": 0.4355484387510008, "grad_norm": 0.0030948214698582888, "learning_rate": 1e-05, "loss": 0.0013, "step": 34, "time_profile/curr_logprobs_and_update": 0.28984188855974935, "train/gen_step": 2176.0, "unified/clip_ratio/high_mean": 0.0015250134501911816, "unified/clip_ratio/low_mean": 0.00027515896545082796, "unified/clip_ratio/region_mean": 0.0018001724210989778, "unified/kl": 0.032954427908407524, "unified/loss": 0.0013315071737451945 }, { "epoch": 0.44835868694955966, "gen/advantages_abs_mean": 0.06641550362110138, "gen/advantages_max": 0.2938459515571594, "gen/advantages_mean": -2.9103830456733704e-11, "gen/advantages_min": -0.1641228049993515, "gen/advantages_std": 0.0816919133067131, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 126.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.08744116127490997, "gen/reward_std": 0.09168319404125214, "grad_norm": 0.004929918795824051, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.08148351311683655, "rewards/perceptual_score_reward_func/std": 0.09414726495742798, "step": 35, "time_profile/curr_logprobs_and_update": 0.2904790450411383, "time_profile/image_rollout": 92.59001582488418, "time_profile/old_logps": 35.86070014163852, "time_profile/ref_logps": 35.71136962622404, "time_profile/reward": 2.4510285947471857, "train/gen_step": 2240.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.028016642027068883, "unified/loss": 0.0010704244341468439 }, { "epoch": 0.4611689351481185, "grad_norm": 0.004974815063178539, "learning_rate": 1e-05, "loss": 0.0011, "step": 36, "time_profile/curr_logprobs_and_update": 0.2918431573198177, "train/gen_step": 2304.0, "unified/clip_ratio/high_mean": 0.00020790819235116942, "unified/clip_ratio/low_mean": 2.3113905626814812e-05, "unified/clip_ratio/region_mean": 0.00023102209797798423, "unified/kl": 0.029212650420959108, "unified/loss": 0.0011148997673444683 }, { "epoch": 0.4739791833466773, "gen/advantages_abs_mean": 0.019589366391301155, "gen/advantages_max": 0.16522341966629028, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.06063012778759003, "gen/advantages_std": 0.032727889716625214, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.28125, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 132.0, "gen/prompt/min_length": 36.0, "gen/reward": 0.015548557974398136, "gen/reward_std": 0.03653242811560631, "grad_norm": 0.0015641744248569012, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.019582821056246758, "rewards/perceptual_score_reward_func/std": 0.04001577943563461, "step": 37, "time_profile/curr_logprobs_and_update": 0.2838708157360088, "time_profile/image_rollout": 90.96994621679187, "time_profile/old_logps": 35.68323879875243, "time_profile/ref_logps": 35.691848032176495, "time_profile/reward": 1.3393241744488478, "train/gen_step": 2368.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.026530865143286064, "unified/loss": 0.0010725572483352153 }, { "epoch": 0.4867894315452362, "grad_norm": 0.001707454095594585, "learning_rate": 1e-05, "loss": 0.0011, "step": 38, "time_profile/curr_logprobs_and_update": 0.28927692130673677, "train/gen_step": 2432.0, "unified/clip_ratio/high_mean": 2.3696002244832925e-05, "unified/clip_ratio/low_mean": 1.4909351193637121e-05, "unified/clip_ratio/region_mean": 3.860535343847005e-05, "unified/kl": 0.026749580312753096, "unified/loss": 0.0010843233976629563 }, { "epoch": 0.49959967974379504, "gen/advantages_abs_mean": 0.06686871498823166, "gen/advantages_max": 0.2528344690799713, "gen/advantages_mean": 2.3283064365386963e-09, "gen/advantages_min": -0.27862676978111267, "gen/advantages_std": 0.08488749712705612, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 160.5, "gen/prompt/min_length": 27.0, "gen/reward": 0.29595229029655457, "gen/reward_std": 0.09182636439800262, "grad_norm": 0.003406939096748829, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.3082679212093353, "rewards/perceptual_score_reward_func/std": 0.08436276763677597, "step": 39, "time_profile/curr_logprobs_and_update": 0.28643454433768056, "time_profile/image_rollout": 98.25927837193012, "time_profile/old_logps": 35.69824261032045, "time_profile/ref_logps": 35.77457028999925, "time_profile/reward": 6.389842351898551, "train/gen_step": 2496.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.0277218354458455, "unified/loss": 0.0011168291480316839 }, { "epoch": 0.5124099279423538, "grad_norm": 0.0038018571212887764, "learning_rate": 1e-05, "loss": 0.0011, "step": 40, "time_profile/curr_logprobs_and_update": 0.2840952040278353, "train/gen_step": 2560.0, "unified/clip_ratio/high_mean": 0.00011745172923838254, "unified/clip_ratio/low_mean": 2.7233115361013915e-05, "unified/clip_ratio/region_mean": 0.00014468484368990175, "unified/kl": 0.028168133198050782, "unified/loss": 0.001051058910888969 }, { "epoch": 0.5252201761409128, "gen/advantages_abs_mean": 0.06853524595499039, "gen/advantages_max": 0.1936076283454895, "gen/advantages_mean": -6.984919309616089e-09, "gen/advantages_min": -0.3802328109741211, "gen/advantages_std": 0.08733128011226654, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.653662919998169, "gen/reward_std": 0.09350146353244781, "grad_norm": 0.007019991986453533, "learning_rate": 1e-05, "loss": 0.0033, "rewards/perceptual_score_reward_func/mean": 0.6532255411148071, "rewards/perceptual_score_reward_func/std": 0.08254469931125641, "step": 41, "time_profile/curr_logprobs_and_update": 0.2885236190923024, "time_profile/image_rollout": 96.83855919353664, "time_profile/old_logps": 35.68985055014491, "time_profile/ref_logps": 35.7790841050446, "time_profile/reward": 1.0563220214098692, "train/gen_step": 2624.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.08251963119255379, "unified/loss": 0.003462206368567422 }, { "epoch": 0.5380304243394716, "grad_norm": 0.004827939905226231, "learning_rate": 1e-05, "loss": 0.003, "step": 42, "time_profile/curr_logprobs_and_update": 0.28419688864960335, "train/gen_step": 2688.0, "unified/clip_ratio/high_mean": 0.001810626625228906, "unified/clip_ratio/low_mean": 0.002413051798612287, "unified/clip_ratio/region_mean": 0.004223678405651299, "unified/kl": 0.07354312663665041, "unified/loss": 0.00292131217156566 }, { "epoch": 0.5508406725380304, "gen/advantages_abs_mean": 0.03801333159208298, "gen/advantages_max": 0.18501146137714386, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.17977194488048553, "gen/advantages_std": 0.052959226071834564, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 38.0, "gen/prompt/mean_length": 31.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.48641788959503174, "gen/reward_std": 0.30649667978286743, "grad_norm": 0.0027381409890949726, "learning_rate": 1e-05, "loss": 0.0009, "rewards/perceptual_score_reward_func/mean": 0.4760347306728363, "rewards/perceptual_score_reward_func/std": 0.3147166967391968, "step": 43, "time_profile/curr_logprobs_and_update": 0.2863878221542109, "time_profile/image_rollout": 93.18704553879797, "time_profile/old_logps": 35.647348675876856, "time_profile/ref_logps": 35.67785409279168, "time_profile/reward": 1.3080625962466002, "train/gen_step": 2752.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.022025975049473345, "unified/loss": 0.0009260941587854177 }, { "epoch": 0.5636509207365893, "grad_norm": 0.0029822327196598053, "learning_rate": 1e-05, "loss": 0.0009, "step": 44, "time_profile/curr_logprobs_and_update": 0.2840305963472929, "train/gen_step": 2816.0, "unified/clip_ratio/high_mean": 3.7968629840179347e-05, "unified/clip_ratio/low_mean": 6.098106496210676e-05, "unified/clip_ratio/region_mean": 9.89496948022861e-05, "unified/kl": 0.02211095401435159, "unified/loss": 0.0009122132069023792 }, { "epoch": 0.5764611689351481, "gen/advantages_abs_mean": 0.03591323643922806, "gen/advantages_max": 0.09316939115524292, "gen/advantages_mean": -4.190951585769653e-09, "gen/advantages_min": -0.20308387279510498, "gen/advantages_std": 0.047014325857162476, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 85.5, "gen/prompt/min_length": 28.0, "gen/reward": 0.7401652336120605, "gen/reward_std": 0.07190340012311935, "grad_norm": 0.0034790660720318556, "learning_rate": 1e-05, "loss": 0.0014, "rewards/perceptual_score_reward_func/mean": 0.7485611438751221, "rewards/perceptual_score_reward_func/std": 0.06707578152418137, "step": 45, "time_profile/curr_logprobs_and_update": 0.2859906947123818, "time_profile/image_rollout": 91.15467045269907, "time_profile/old_logps": 35.6747612785548, "time_profile/ref_logps": 35.70546899549663, "time_profile/reward": 1.08854722045362, "train/gen_step": 2880.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.03551414527464658, "unified/loss": 0.0014457265460805502 }, { "epoch": 0.589271417133707, "grad_norm": 0.002829823410138488, "learning_rate": 1e-05, "loss": 0.0014, "step": 46, "time_profile/curr_logprobs_and_update": 0.2882837516372092, "train/gen_step": 2944.0, "unified/clip_ratio/high_mean": 0.0007083340506142122, "unified/clip_ratio/low_mean": 0.00391641097212414, "unified/clip_ratio/region_mean": 0.004624744986358564, "unified/kl": 0.035664693103171885, "unified/loss": 0.0014884003412589664 }, { "epoch": 0.6020816653322658, "gen/advantages_abs_mean": 0.05315067991614342, "gen/advantages_max": 0.2091759443283081, "gen/advantages_mean": 4.889443516731262e-09, "gen/advantages_min": -0.3550701141357422, "gen/advantages_std": 0.06988511979579926, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.7280530333518982, "gen/reward_std": 0.07879945635795593, "grad_norm": 0.0043497150763869286, "learning_rate": 1e-05, "loss": 0.0024, "rewards/perceptual_score_reward_func/mean": 0.7486039400100708, "rewards/perceptual_score_reward_func/std": 0.056908342987298965, "step": 47, "time_profile/curr_logprobs_and_update": 0.2845378862693906, "time_profile/image_rollout": 93.7537659406662, "time_profile/old_logps": 35.71550615131855, "time_profile/ref_logps": 35.81924728676677, "time_profile/reward": 0.9302971065044403, "train/gen_step": 3008.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.05950034986017272, "unified/loss": 0.0025013487907017407 }, { "epoch": 0.6148919135308246, "grad_norm": 0.0044915638864040375, "learning_rate": 1e-05, "loss": 0.0026, "step": 48, "time_profile/curr_logprobs_and_update": 0.28826455789385363, "train/gen_step": 3072.0, "unified/clip_ratio/high_mean": 0.0014778335389564745, "unified/clip_ratio/low_mean": 0.009821245124840061, "unified/clip_ratio/region_mean": 0.011299078611045843, "unified/kl": 0.06371590058552101, "unified/loss": 0.002573426672597634 }, { "epoch": 0.6277021617293835, "gen/advantages_abs_mean": 0.03862232714891434, "gen/advantages_max": 0.24333956837654114, "gen/advantages_mean": 1.1641532182693481e-09, "gen/advantages_min": -0.1539299339056015, "gen/advantages_std": 0.05260282754898071, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 38.0, "gen/prompt/mean_length": 33.5, "gen/prompt/min_length": 29.0, "gen/reward": 0.4815671443939209, "gen/reward_std": 0.33772698044776917, "grad_norm": 0.00316769746132195, "learning_rate": 1e-05, "loss": 0.0008, "rewards/perceptual_score_reward_func/mean": 0.490455687046051, "rewards/perceptual_score_reward_func/std": 0.3341163992881775, "step": 49, "time_profile/curr_logprobs_and_update": 0.28858013937133364, "time_profile/image_rollout": 92.26549053192139, "time_profile/old_logps": 35.63793566450477, "time_profile/ref_logps": 35.693480079993606, "time_profile/reward": 2.303144110366702, "train/gen_step": 3136.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.020838526252191514, "unified/loss": 0.0008064835201366805 }, { "epoch": 0.6405124099279423, "grad_norm": 0.0031122236978262663, "learning_rate": 1e-05, "loss": 0.0008, "step": 50, "time_profile/curr_logprobs_and_update": 0.2807842147012707, "train/gen_step": 3200.0, "unified/clip_ratio/high_mean": 0.00040497850750398356, "unified/clip_ratio/low_mean": 5.118025183037389e-05, "unified/clip_ratio/region_mean": 0.00045615876297233626, "unified/kl": 0.02007605423568748, "unified/loss": 0.0008150048051902559 }, { "epoch": 0.6533226581265013, "gen/advantages_abs_mean": 0.045201025903224945, "gen/advantages_max": 0.2404613345861435, "gen/advantages_mean": -2.5320332497358322e-09, "gen/advantages_min": -0.16845963895320892, "gen/advantages_std": 0.06456216424703598, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 36.0, "gen/prompt/mean_length": 33.0, "gen/prompt/min_length": 30.0, "gen/reward": 0.48360586166381836, "gen/reward_std": 0.3726156949996948, "grad_norm": 0.004150118213146925, "learning_rate": 1e-05, "loss": 0.001, "rewards/perceptual_score_reward_func/mean": 0.4762686491012573, "rewards/perceptual_score_reward_func/std": 0.38303321599960327, "step": 51, "time_profile/curr_logprobs_and_update": 0.2836454862554092, "time_profile/image_rollout": 90.03667972795665, "time_profile/old_logps": 35.62950346246362, "time_profile/ref_logps": 35.67210088297725, "time_profile/reward": 1.1959608290344477, "train/gen_step": 3264.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.023927463014842942, "unified/loss": 0.000979490663667093 }, { "epoch": 0.6661329063250601, "grad_norm": 0.00425776606425643, "learning_rate": 1e-05, "loss": 0.0009, "step": 52, "time_profile/curr_logprobs_and_update": 0.2833061977289617, "train/gen_step": 3328.0, "unified/clip_ratio/high_mean": 0.0007207577100416529, "unified/clip_ratio/low_mean": 7.231485324155074e-05, "unified/clip_ratio/region_mean": 0.0007930725596452248, "unified/kl": 0.022969681856920943, "unified/loss": 0.0008664954584673978 }, { "epoch": 0.6789431545236189, "gen/advantages_abs_mean": 0.06953415274620056, "gen/advantages_max": 0.22283008694648743, "gen/advantages_mean": 1.3969838619232178e-09, "gen/advantages_min": -0.2646213173866272, "gen/advantages_std": 0.08586404472589493, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 215.5, "gen/prompt/min_length": 137.0, "gen/reward": 0.4072887897491455, "gen/reward_std": 0.23019105195999146, "grad_norm": 0.003899160074070096, "learning_rate": 1e-05, "loss": 0.0016, "rewards/perceptual_score_reward_func/mean": 0.42112410068511963, "rewards/perceptual_score_reward_func/std": 0.23615583777427673, "step": 53, "time_profile/curr_logprobs_and_update": 0.2833262274216395, "time_profile/image_rollout": 78.63981351442635, "time_profile/old_logps": 35.7274684663862, "time_profile/ref_logps": 35.797853803262115, "time_profile/reward": 1.171833161264658, "train/gen_step": 3392.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.03973888815380633, "unified/loss": 0.0015896564364084043 }, { "epoch": 0.6917534027221778, "grad_norm": 0.0038946911226958036, "learning_rate": 1e-05, "loss": 0.0016, "step": 54, "time_profile/curr_logprobs_and_update": 0.28796653434983455, "train/gen_step": 3456.0, "unified/clip_ratio/high_mean": 0.00015383093978016404, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.00015383093978016404, "unified/kl": 0.03877779032336548, "unified/loss": 0.0015223519985738676 }, { "epoch": 0.7045636509207366, "gen/advantages_abs_mean": 0.09249347448348999, "gen/advantages_max": 0.3164886236190796, "gen/advantages_mean": 4.3655745685100555e-10, "gen/advantages_min": -0.258684903383255, "gen/advantages_std": 0.11256097257137299, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 28.0, "gen/prompt/mean_length": 26.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.2795722782611847, "gen/reward_std": 0.13351306319236755, "grad_norm": 0.004936882294714451, "learning_rate": 1e-05, "loss": 0.0009, "rewards/perceptual_score_reward_func/mean": 0.2909090518951416, "rewards/perceptual_score_reward_func/std": 0.13314305245876312, "step": 55, "time_profile/curr_logprobs_and_update": 0.28386063207290135, "time_profile/image_rollout": 97.24604052305222, "time_profile/old_logps": 35.62250469997525, "time_profile/ref_logps": 35.67051147669554, "time_profile/reward": 1.362681845203042, "train/gen_step": 3520.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.0227790946373716, "unified/loss": 0.0008780551270319847 }, { "epoch": 0.7173738991192954, "grad_norm": 0.00479149492457509, "learning_rate": 1e-05, "loss": 0.0009, "step": 56, "time_profile/curr_logprobs_and_update": 0.28903222834924236, "train/gen_step": 3584.0, "unified/clip_ratio/high_mean": 0.000115000895675621, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.000115000895675621, "unified/kl": 0.024405151838436723, "unified/loss": 0.0010101461557496805 }, { "epoch": 0.7301841473178543, "gen/advantages_abs_mean": 0.058377232402563095, "gen/advantages_max": 0.24894979596138, "gen/advantages_mean": -2.561137080192566e-09, "gen/advantages_min": -0.2390967607498169, "gen/advantages_std": 0.07458589226007462, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 37.0, "gen/prompt/mean_length": 32.0, "gen/prompt/min_length": 27.0, "gen/reward": 0.5388422012329102, "gen/reward_std": 0.1490672528743744, "grad_norm": 0.0033019648399204016, "learning_rate": 1e-05, "loss": 0.0009, "rewards/perceptual_score_reward_func/mean": 0.5304601192474365, "rewards/perceptual_score_reward_func/std": 0.15523570775985718, "step": 57, "time_profile/curr_logprobs_and_update": 0.28454931010492146, "time_profile/image_rollout": 89.89481943659484, "time_profile/old_logps": 35.618444772437215, "time_profile/ref_logps": 35.67245299369097, "time_profile/reward": 1.3785167206078768, "train/gen_step": 3648.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.021809650992508978, "unified/loss": 0.0008770865549649898 }, { "epoch": 0.7429943955164131, "grad_norm": 0.0033817507792264223, "learning_rate": 1e-05, "loss": 0.0009, "step": 58, "time_profile/curr_logprobs_and_update": 0.28436136766686104, "train/gen_step": 3712.0, "unified/clip_ratio/high_mean": 0.00040959819307317957, "unified/clip_ratio/low_mean": 2.537267664592946e-05, "unified/clip_ratio/region_mean": 0.00043497086971910903, "unified/kl": 0.023068611073540524, "unified/loss": 0.0008947431465458067 }, { "epoch": 0.755804643714972, "gen/advantages_abs_mean": 0.06415918469429016, "gen/advantages_max": 0.3439922332763672, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.25892210006713867, "gen/advantages_std": 0.08247826993465424, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 28.0, "gen/prompt/mean_length": 27.0, "gen/prompt/min_length": 26.0, "gen/reward": 0.23676720261573792, "gen/reward_std": 0.08878710865974426, "grad_norm": 0.002476656809449196, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.23685407638549805, "rewards/perceptual_score_reward_func/std": 0.09089451283216476, "step": 59, "time_profile/curr_logprobs_and_update": 0.2890419715840835, "time_profile/image_rollout": 91.48738839104772, "time_profile/old_logps": 35.63207217492163, "time_profile/ref_logps": 35.67341075837612, "time_profile/reward": 1.305101539939642, "train/gen_step": 3776.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.027545947610633448, "unified/loss": 0.0011357355069776531 }, { "epoch": 0.7686148919135308, "grad_norm": 0.002608807757496834, "learning_rate": 1e-05, "loss": 0.0011, "step": 60, "time_profile/curr_logprobs_and_update": 0.28312389296479523, "train/gen_step": 3840.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 1.302083364862483e-05, "unified/clip_ratio/region_mean": 1.302083364862483e-05, "unified/kl": 0.027554543688893318, "unified/loss": 0.0010848145793715958 }, { "epoch": 0.7814251401120896, "gen/advantages_abs_mean": 0.03390387073159218, "gen/advantages_max": 0.13214147090911865, "gen/advantages_mean": -2.561137080192566e-09, "gen/advantages_min": -0.3367220163345337, "gen/advantages_std": 0.04783477634191513, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 87.0, "gen/prompt/min_length": 31.0, "gen/reward": 0.6992359757423401, "gen/reward_std": 0.09428206086158752, "grad_norm": 0.0026902405079454184, "learning_rate": 1e-05, "loss": 0.0018, "rewards/perceptual_score_reward_func/mean": 0.6967364549636841, "rewards/perceptual_score_reward_func/std": 0.09561172127723694, "step": 61, "time_profile/curr_logprobs_and_update": 0.2933660880953539, "time_profile/image_rollout": 90.9877971354872, "time_profile/old_logps": 35.695311322808266, "time_profile/ref_logps": 35.70469231158495, "time_profile/reward": 1.2580257393419743, "train/gen_step": 3904.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.04532510880380869, "unified/loss": 0.001845876574407157 }, { "epoch": 0.7942353883106485, "grad_norm": 0.002456333488225937, "learning_rate": 1e-05, "loss": 0.0017, "step": 62, "time_profile/curr_logprobs_and_update": 0.289033788634697, "train/gen_step": 3968.0, "unified/clip_ratio/high_mean": 0.0044582416012417525, "unified/clip_ratio/low_mean": 0.00011452383205323713, "unified/clip_ratio/region_mean": 0.004572765432385495, "unified/kl": 0.04306245065527037, "unified/loss": 0.0016709059446498031 }, { "epoch": 0.8070456365092074, "gen/advantages_abs_mean": 0.04794721305370331, "gen/advantages_max": 0.16062045097351074, "gen/advantages_mean": -2.3283064365386963e-09, "gen/advantages_min": -0.2264614701271057, "gen/advantages_std": 0.06226880103349686, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 33.0, "gen/prompt/mean_length": 31.5, "gen/prompt/min_length": 30.0, "gen/reward": 0.7169865965843201, "gen/reward_std": 0.08426859974861145, "grad_norm": 0.0035603002179414034, "learning_rate": 1e-05, "loss": 0.001, "rewards/perceptual_score_reward_func/mean": 0.7329443693161011, "rewards/perceptual_score_reward_func/std": 0.07624883949756622, "step": 63, "time_profile/curr_logprobs_and_update": 0.2918489087896887, "time_profile/image_rollout": 90.09776932001114, "time_profile/old_logps": 35.6436334438622, "time_profile/ref_logps": 35.685858277603984, "time_profile/reward": 1.3784125726670027, "train/gen_step": 4032.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.024762474262388423, "unified/loss": 0.000963904687523609 }, { "epoch": 0.8198558847077662, "grad_norm": 0.003421743866056204, "learning_rate": 1e-05, "loss": 0.001, "step": 64, "time_profile/curr_logprobs_and_update": 0.28897311072796583, "train/gen_step": 4096.0, "unified/clip_ratio/high_mean": 0.0002471591642461135, "unified/clip_ratio/low_mean": 0.00021569021282630274, "unified/clip_ratio/region_mean": 0.00046284937798191095, "unified/kl": 0.024078681715764105, "unified/loss": 0.0009506012538622599 }, { "epoch": 0.8326661329063251, "gen/advantages_abs_mean": 0.03795839846134186, "gen/advantages_max": 0.19034850597381592, "gen/advantages_mean": 2.3283064365386963e-09, "gen/advantages_min": -0.11913624405860901, "gen/advantages_std": 0.05202922970056534, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 90.5, "gen/prompt/min_length": 44.0, "gen/reward": 0.4587080478668213, "gen/reward_std": 0.3764707148075104, "grad_norm": 0.0019860498141497374, "learning_rate": 1e-05, "loss": 0.0009, "rewards/perceptual_score_reward_func/mean": 0.45268112421035767, "rewards/perceptual_score_reward_func/std": 0.3850635290145874, "step": 65, "time_profile/curr_logprobs_and_update": 0.2939559489605017, "time_profile/image_rollout": 87.89024894498289, "time_profile/old_logps": 35.630313439294696, "time_profile/ref_logps": 35.68208645284176, "time_profile/reward": 1.5224378574639559, "train/gen_step": 4160.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.023230015503941104, "unified/loss": 0.0009148021113105642 }, { "epoch": 0.8454763811048839, "grad_norm": 0.0018545385682955384, "learning_rate": 1e-05, "loss": 0.0009, "step": 66, "time_profile/curr_logprobs_and_update": 0.2903638135176152, "train/gen_step": 4224.0, "unified/clip_ratio/high_mean": 0.00017140993986686226, "unified/clip_ratio/low_mean": 0.00045493132347473875, "unified/clip_ratio/region_mean": 0.0006263412597036222, "unified/kl": 0.02263786207186058, "unified/loss": 0.0009286232896101865 }, { "epoch": 0.8582866293034428, "gen/advantages_abs_mean": 0.06805029511451721, "gen/advantages_max": 0.33124828338623047, "gen/advantages_mean": -3.055902197957039e-09, "gen/advantages_min": -0.2604416608810425, "gen/advantages_std": 0.08757373690605164, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 34.0, "gen/prompt/mean_length": 30.5, "gen/prompt/min_length": 27.0, "gen/reward": 0.4795072078704834, "gen/reward_std": 0.2444414645433426, "grad_norm": 0.00417951587587595, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.48338961601257324, "rewards/perceptual_score_reward_func/std": 0.2435249537229538, "step": 67, "time_profile/curr_logprobs_and_update": 0.2845795691537205, "time_profile/image_rollout": 94.08473618142307, "time_profile/old_logps": 35.653244607150555, "time_profile/ref_logps": 35.68242741189897, "time_profile/reward": 1.570898663252592, "train/gen_step": 4288.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.027502871002070606, "unified/loss": 0.001087280929368717 }, { "epoch": 0.8710968775020016, "grad_norm": 0.004231288097798824, "learning_rate": 1e-05, "loss": 0.001, "step": 68, "time_profile/curr_logprobs_and_update": 0.28916991435107775, "train/gen_step": 4352.0, "unified/clip_ratio/high_mean": 0.000318356032948941, "unified/clip_ratio/low_mean": 0.000416163186855556, "unified/clip_ratio/region_mean": 0.0007345192207139917, "unified/kl": 0.026457387197297066, "unified/loss": 0.0010340961512156355 }, { "epoch": 0.8839071257005604, "gen/advantages_abs_mean": 0.04543104022741318, "gen/advantages_max": 0.23743367195129395, "gen/advantages_mean": -3.026798367500305e-09, "gen/advantages_min": -0.31966233253479004, "gen/advantages_std": 0.06301993876695633, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 166.5, "gen/prompt/min_length": 39.0, "gen/reward": 0.6233011484146118, "gen/reward_std": 0.24330009520053864, "grad_norm": 0.00291136815212667, "learning_rate": 1e-05, "loss": 0.0018, "rewards/perceptual_score_reward_func/mean": 0.6394872665405273, "rewards/perceptual_score_reward_func/std": 0.22583803534507751, "step": 69, "time_profile/curr_logprobs_and_update": 0.29470567259704694, "time_profile/image_rollout": 83.45683548785746, "time_profile/old_logps": 35.69185835123062, "time_profile/ref_logps": 35.72108877636492, "time_profile/reward": 1.4233777895569801, "train/gen_step": 4416.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.043804147600894794, "unified/loss": 0.0017766948712960584 }, { "epoch": 0.8967173738991193, "grad_norm": 0.003228710265830159, "learning_rate": 1e-05, "loss": 0.0016, "step": 70, "time_profile/curr_logprobs_and_update": 0.28153255736106075, "train/gen_step": 4480.0, "unified/clip_ratio/high_mean": 0.00013945894897915423, "unified/clip_ratio/low_mean": 0.000410210202971939, "unified/clip_ratio/region_mean": 0.0005496691501321038, "unified/kl": 0.04113790258998051, "unified/loss": 0.001632484994843253 }, { "epoch": 0.9095276220976781, "gen/advantages_abs_mean": 0.04988550767302513, "gen/advantages_max": 0.25346121191978455, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.20711418986320496, "gen/advantages_std": 0.06387721002101898, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 165.0, "gen/prompt/min_length": 36.0, "gen/reward": 0.510562539100647, "gen/reward_std": 0.2098800390958786, "grad_norm": 0.002879665931686759, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.5041624903678894, "rewards/perceptual_score_reward_func/std": 0.20727090537548065, "step": 71, "time_profile/curr_logprobs_and_update": 0.2848870683228597, "time_profile/image_rollout": 89.13787977769971, "time_profile/old_logps": 35.68902938440442, "time_profile/ref_logps": 35.7388895843178, "time_profile/reward": 1.2322177048772573, "train/gen_step": 4544.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.026298649958334863, "unified/loss": 0.0010413604650238995 }, { "epoch": 0.922337870296237, "grad_norm": 0.002842222573235631, "learning_rate": 1e-05, "loss": 0.001, "step": 72, "time_profile/curr_logprobs_and_update": 0.287293476780178, "train/gen_step": 4608.0, "unified/clip_ratio/high_mean": 0.00032580364859313704, "unified/clip_ratio/low_mean": 0.0001650192261877237, "unified/clip_ratio/region_mean": 0.0004908228756903554, "unified/kl": 0.025436352443648502, "unified/loss": 0.0009967807036446175 }, { "epoch": 0.9351481184947958, "gen/advantages_abs_mean": 0.06981154531240463, "gen/advantages_max": 0.2795065641403198, "gen/advantages_mean": 9.313225746154785e-10, "gen/advantages_min": -0.2961052358150482, "gen/advantages_std": 0.08908019214868546, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 129.0, "gen/prompt/min_length": 30.0, "gen/reward": 0.3487090468406677, "gen/reward_std": 0.10480765253305435, "grad_norm": 0.0054478757083415985, "learning_rate": 1e-05, "loss": 0.0012, "rewards/perceptual_score_reward_func/mean": 0.34865331649780273, "rewards/perceptual_score_reward_func/std": 0.09679755568504333, "step": 73, "time_profile/curr_logprobs_and_update": 0.2976631856581662, "time_profile/image_rollout": 93.59027141705155, "time_profile/old_logps": 35.697798715904355, "time_profile/ref_logps": 35.81994019635022, "time_profile/reward": 1.8824999630451202, "train/gen_step": 4672.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.030644683924037963, "unified/loss": 0.0012316222382651176 }, { "epoch": 0.9479583666933546, "grad_norm": 0.005177585408091545, "learning_rate": 1e-05, "loss": 0.0012, "step": 74, "time_profile/curr_logprobs_and_update": 0.28828327282099053, "train/gen_step": 4736.0, "unified/clip_ratio/high_mean": 0.00013896780728828162, "unified/clip_ratio/low_mean": 0.00019588950817706063, "unified/clip_ratio/region_mean": 0.00033485731455584755, "unified/kl": 0.03049655826180242, "unified/loss": 0.0014046990941096738 }, { "epoch": 0.9607686148919136, "gen/advantages_abs_mean": 0.05187247693538666, "gen/advantages_max": 0.17589330673217773, "gen/advantages_mean": -1.3969838619232178e-09, "gen/advantages_min": -0.3043435215950012, "gen/advantages_std": 0.07163926959037781, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 88.0, "gen/prompt/min_length": 33.0, "gen/reward": 0.7588982582092285, "gen/reward_std": 0.09124431014060974, "grad_norm": 0.008605101145803928, "learning_rate": 1e-05, "loss": 0.003, "rewards/perceptual_score_reward_func/mean": 0.7651864290237427, "rewards/perceptual_score_reward_func/std": 0.07482659071683884, "step": 75, "time_profile/curr_logprobs_and_update": 0.28617306941305287, "time_profile/image_rollout": 95.36229601316154, "time_profile/old_logps": 35.670409236103296, "time_profile/ref_logps": 35.708794893696904, "time_profile/reward": 1.2329577188938856, "train/gen_step": 4800.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.07468988915206864, "unified/loss": 0.0028182062202404268 }, { "epoch": 0.9735788630904724, "grad_norm": 0.003986031282693148, "learning_rate": 1e-05, "loss": 0.0028, "step": 76, "time_profile/curr_logprobs_and_update": 0.29819186983513646, "train/gen_step": 4864.0, "unified/clip_ratio/high_mean": 0.003613574607697956, "unified/clip_ratio/low_mean": 0.0007360392692135065, "unified/clip_ratio/region_mean": 0.0043496139069247874, "unified/kl": 0.06856736988993362, "unified/loss": 0.0028428112407254957 }, { "epoch": 0.9863891112890312, "gen/advantages_abs_mean": 0.04775109142065048, "gen/advantages_max": 0.19789379835128784, "gen/advantages_mean": 3.4924596548080444e-09, "gen/advantages_min": -0.39248591661453247, "gen/advantages_std": 0.06537375599145889, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 87.5, "gen/prompt/min_length": 32.0, "gen/reward": 0.6827777028083801, "gen/reward_std": 0.09045189619064331, "grad_norm": 0.005181001964956522, "learning_rate": 1e-05, "loss": 0.0022, "rewards/perceptual_score_reward_func/mean": 0.6811036467552185, "rewards/perceptual_score_reward_func/std": 0.09361086785793304, "step": 77, "time_profile/curr_logprobs_and_update": 0.29179857825511135, "time_profile/image_rollout": 83.27612288482487, "time_profile/old_logps": 35.658640841022134, "time_profile/ref_logps": 35.68707458116114, "time_profile/reward": 1.1409052349627018, "train/gen_step": 4928.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.05598265668959357, "unified/loss": 0.0022124095494291396 }, { "epoch": 0.9991993594875901, "grad_norm": 0.005045454483479261, "learning_rate": 1e-05, "loss": 0.002, "step": 78, "time_profile/curr_logprobs_and_update": 0.2866454735340085, "train/gen_step": 4992.0, "unified/clip_ratio/high_mean": 0.001705825293356611, "unified/clip_ratio/low_mean": 0.002124744443790405, "unified/clip_ratio/region_mean": 0.0038305697316900478, "unified/kl": 0.04944189221714623, "unified/loss": 0.0019907314253941877 }, { "epoch": 1.0, "gen/advantages_abs_mean": 0.05695275217294693, "gen/advantages_max": 0.29835107922554016, "gen/advantages_mean": -1.862645149230957e-09, "gen/advantages_min": -0.24371516704559326, "gen/advantages_std": 0.07456254214048386, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 45.0, "gen/prompt/mean_length": 38.0, "gen/prompt/min_length": 31.0, "gen/reward": 0.5770522356033325, "gen/reward_std": 0.21481432020664215, "grad_norm": 0.0033948994241654873, "learning_rate": 1e-05, "loss": 0.0, "rewards/perceptual_score_reward_func/mean": 0.5808098316192627, "rewards/perceptual_score_reward_func/std": 0.21286322176456451, "step": 79, "time_profile/curr_logprobs_and_update": 0.28097593411803246, "time_profile/image_rollout": 86.5219391360879, "time_profile/old_logps": 35.649459190666676, "time_profile/ref_logps": 35.691718278452754, "time_profile/reward": 1.29204579629004, "train/gen_step": 4996.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.025911119766533375, "unified/loss": 0.008625521790236235 }, { "epoch": 1.0128102481985588, "grad_norm": 0.002015733392909169, "learning_rate": 1e-05, "loss": 0.001, "step": 80, "time_profile/curr_logprobs_and_update": 0.2885962183645461, "train/gen_step": 5060.0, "unified/clip_ratio/high_mean": 0.0003266334970248863, "unified/clip_ratio/low_mean": 0.00028056274277332705, "unified/clip_ratio/region_mean": 0.000607196237979224, "unified/kl": 0.02535266784252599, "unified/loss": 0.00100777412262687 }, { "epoch": 1.0256204963971176, "gen/advantages_abs_mean": 0.04768690466880798, "gen/advantages_max": 0.23502177000045776, "gen/advantages_mean": -1.6298145055770874e-09, "gen/advantages_min": -0.17850369215011597, "gen/advantages_std": 0.06418629735708237, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 39.0, "gen/prompt/mean_length": 32.0, "gen/prompt/min_length": 25.0, "gen/reward": 0.5296216011047363, "gen/reward_std": 0.24720948934555054, "grad_norm": 0.002834862098097801, "learning_rate": 1e-05, "loss": 0.0013, "rewards/perceptual_score_reward_func/mean": 0.5289547443389893, "rewards/perceptual_score_reward_func/std": 0.24988876283168793, "step": 81, "time_profile/curr_logprobs_and_update": 0.28801219374872744, "time_profile/image_rollout": 86.72076600790024, "time_profile/old_logps": 35.62672356516123, "time_profile/ref_logps": 35.670366356149316, "time_profile/reward": 1.4512761607766151, "train/gen_step": 5124.0, "unified/clip_ratio/high_mean": 0.0009658800263423473, "unified/clip_ratio/low_mean": 0.0008977456272987183, "unified/clip_ratio/region_mean": 0.001863625655460055, "unified/kl": 0.025290006422437727, "unified/loss": -0.0006758762265235418 }, { "epoch": 1.0384307445956766, "grad_norm": 0.003795901546254754, "learning_rate": 1e-05, "loss": 0.0013, "step": 82, "time_profile/curr_logprobs_and_update": 0.29509411737672053, "train/gen_step": 5188.0, "unified/clip_ratio/high_mean": 0.0003557014515536139, "unified/clip_ratio/low_mean": 0.00017478797963121906, "unified/clip_ratio/region_mean": 0.0005304894348228117, "unified/kl": 0.03130401810631156, "unified/loss": 0.000909826174163797 }, { "epoch": 1.0512409927942354, "gen/advantages_abs_mean": 0.050498440861701965, "gen/advantages_max": 0.19424709677696228, "gen/advantages_mean": 3.026798367500305e-09, "gen/advantages_min": -0.28340137004852295, "gen/advantages_std": 0.06539153307676315, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 32.0, "gen/prompt/mean_length": 28.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.4895917773246765, "gen/reward_std": 0.17331120371818542, "grad_norm": 0.003306619357317686, "learning_rate": 1e-05, "loss": -0.0001, "rewards/perceptual_score_reward_func/mean": 0.497758686542511, "rewards/perceptual_score_reward_func/std": 0.1741664856672287, "step": 83, "time_profile/curr_logprobs_and_update": 0.28760485889506526, "time_profile/image_rollout": 88.75942911952734, "time_profile/old_logps": 35.62598751485348, "time_profile/ref_logps": 35.6756409779191, "time_profile/reward": 1.3580076191574335, "train/gen_step": 5252.0, "unified/clip_ratio/high_mean": 0.0012764136354235234, "unified/clip_ratio/low_mean": 0.001044765193000785, "unified/clip_ratio/region_mean": 0.0023211788338812767, "unified/kl": 0.021809721365571022, "unified/loss": 0.0047350469174034515 }, { "epoch": 1.0640512409927942, "grad_norm": 0.004482661839574575, "learning_rate": 1e-05, "loss": 0.001, "step": 84, "time_profile/curr_logprobs_and_update": 0.28422601052443497, "train/gen_step": 5316.0, "unified/clip_ratio/high_mean": 0.00010350447519158479, "unified/clip_ratio/low_mean": 6.248103727557464e-05, "unified/clip_ratio/region_mean": 0.00016598551337665413, "unified/kl": 0.02575801726197824, "unified/loss": 0.0009556097375025274 }, { "epoch": 1.076861489191353, "gen/advantages_abs_mean": 0.07610887289047241, "gen/advantages_max": 0.279594749212265, "gen/advantages_mean": 4.656612873077393e-10, "gen/advantages_min": -0.2552163898944855, "gen/advantages_std": 0.09314155578613281, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 28.0, "gen/prompt/mean_length": 26.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.16894452273845673, "gen/reward_std": 0.12381462752819061, "grad_norm": 0.005054818466305733, "learning_rate": 1e-05, "loss": 0.0005, "rewards/perceptual_score_reward_func/mean": 0.1732819378376007, "rewards/perceptual_score_reward_func/std": 0.11441599577665329, "step": 85, "time_profile/curr_logprobs_and_update": 0.2935908046492841, "time_profile/image_rollout": 80.12255467288196, "time_profile/old_logps": 35.63555760681629, "time_profile/ref_logps": 35.66839297674596, "time_profile/reward": 1.3038444705307484, "train/gen_step": 5380.0, "unified/clip_ratio/high_mean": 0.0005621371064989944, "unified/clip_ratio/low_mean": 0.0003427702713452163, "unified/clip_ratio/region_mean": 0.0009049073787537054, "unified/kl": 0.02579947459162213, "unified/loss": -0.0009912145637827052 }, { "epoch": 1.0896717373899119, "grad_norm": 0.00472763879224658, "learning_rate": 1e-05, "loss": 0.0009, "step": 86, "time_profile/curr_logprobs_and_update": 0.2931293906294741, "train/gen_step": 5444.0, "unified/clip_ratio/high_mean": 2.467482136125909e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 2.467482136125909e-05, "unified/kl": 0.023300431785173714, "unified/loss": 0.0010191962392127607 }, { "epoch": 1.1024819855884709, "gen/advantages_abs_mean": 0.059635188430547714, "gen/advantages_max": 0.23380088806152344, "gen/advantages_mean": 2.0954757928848267e-09, "gen/advantages_min": -0.3752295970916748, "gen/advantages_std": 0.0778651088476181, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.4627537727355957, "gen/reward_std": 0.2263425588607788, "grad_norm": 0.005005163140594959, "learning_rate": 1e-05, "loss": 0.0024, "rewards/perceptual_score_reward_func/mean": 0.46124348044395447, "rewards/perceptual_score_reward_func/std": 0.2309817224740982, "step": 87, "time_profile/curr_logprobs_and_update": 0.2806922975287307, "time_profile/image_rollout": 86.29171478375793, "time_profile/old_logps": 35.68952482007444, "time_profile/ref_logps": 35.76360684260726, "time_profile/reward": 1.391890512779355, "train/gen_step": 5508.0, "unified/clip_ratio/high_mean": 0.000248023759922944, "unified/clip_ratio/low_mean": 5.1719837756536435e-05, "unified/clip_ratio/region_mean": 0.00029974359676998574, "unified/kl": 0.02937340398784727, "unified/loss": -0.0005479482424561866 }, { "epoch": 1.1152922337870297, "grad_norm": 0.004413130227476358, "learning_rate": 1e-05, "loss": 0.003, "step": 88, "time_profile/curr_logprobs_and_update": 0.281406976893777, "train/gen_step": 5572.0, "unified/clip_ratio/high_mean": 0.0038563079042432946, "unified/clip_ratio/low_mean": 9.112708903558087e-05, "unified/clip_ratio/region_mean": 0.003947434987821907, "unified/kl": 0.07321834639878944, "unified/loss": 0.002794792841086746 }, { "epoch": 1.1281024819855885, "gen/advantages_abs_mean": 0.08485985547304153, "gen/advantages_max": 0.37464210391044617, "gen/advantages_mean": 9.604264050722122e-10, "gen/advantages_min": -0.2651618421077728, "gen/advantages_std": 0.10387808084487915, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 161.0, "gen/prompt/min_length": 28.0, "gen/reward": 0.3185094892978668, "gen/reward_std": 0.18651913106441498, "grad_norm": 0.006158351898193359, "learning_rate": 1e-05, "loss": 0.0032, "rewards/perceptual_score_reward_func/mean": 0.3045373857021332, "rewards/perceptual_score_reward_func/std": 0.19984664022922516, "step": 89, "time_profile/curr_logprobs_and_update": 0.2874557306349743, "time_profile/image_rollout": 89.15939953178167, "time_profile/old_logps": 35.65223306231201, "time_profile/ref_logps": 35.69842333719134, "time_profile/reward": 1.4332980029284954, "train/gen_step": 5636.0, "unified/clip_ratio/high_mean": 0.034317739496145805, "unified/clip_ratio/low_mean": 0.0007901403296273202, "unified/clip_ratio/region_mean": 0.03510787995310238, "unified/kl": 0.057693745475262403, "unified/loss": 0.006083310350277316 }, { "epoch": 1.1409127301841473, "grad_norm": 0.006367347203195095, "learning_rate": 1e-05, "loss": 0.0013, "step": 90, "time_profile/curr_logprobs_and_update": 0.2880125379888341, "train/gen_step": 5700.0, "unified/clip_ratio/high_mean": 0.001398666523527936, "unified/clip_ratio/low_mean": 0.002986333717672096, "unified/clip_ratio/region_mean": 0.004385000211186707, "unified/kl": 0.03243849166028667, "unified/loss": 0.0013116523468852392 }, { "epoch": 1.153722978382706, "gen/advantages_abs_mean": 0.05420481413602829, "gen/advantages_max": 0.20414644479751587, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.27419042587280273, "gen/advantages_std": 0.07089695334434509, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 85.0, "gen/prompt/min_length": 27.0, "gen/reward": 0.5731308460235596, "gen/reward_std": 0.16429921984672546, "grad_norm": 0.005784476175904274, "learning_rate": 1e-05, "loss": 0.0017, "rewards/perceptual_score_reward_func/mean": 0.5847713947296143, "rewards/perceptual_score_reward_func/std": 0.1503916084766388, "step": 91, "time_profile/curr_logprobs_and_update": 0.29128942391253076, "time_profile/image_rollout": 91.42569714412093, "time_profile/old_logps": 35.68290564417839, "time_profile/ref_logps": 35.72888129577041, "time_profile/reward": 1.2796872407197952, "train/gen_step": 5764.0, "unified/clip_ratio/high_mean": 0.003734815680218162, "unified/clip_ratio/low_mean": 0.009340950593468733, "unified/clip_ratio/region_mean": 0.01307576622639317, "unified/kl": 0.03101163225073833, "unified/loss": 0.00124256549815982 }, { "epoch": 1.1665332265812651, "grad_norm": 0.004937492776662111, "learning_rate": 1e-05, "loss": 0.005, "step": 92, "time_profile/curr_logprobs_and_update": 0.2811797432950698, "train/gen_step": 5828.0, "unified/clip_ratio/high_mean": 0.007572715853711998, "unified/clip_ratio/low_mean": 3.8088402106950525e-05, "unified/clip_ratio/region_mean": 0.007610804260366422, "unified/kl": 0.12355734975426458, "unified/loss": 0.004843149222779175 }, { "epoch": 1.179343474779824, "gen/advantages_abs_mean": 0.03607391566038132, "gen/advantages_max": 0.2704744338989258, "gen/advantages_mean": -2.3283064365386963e-10, "gen/advantages_min": -0.10774393379688263, "gen/advantages_std": 0.0513719767332077, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.015625, "gen/prompt/max_length": 46.0, "gen/prompt/mean_length": 35.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.43351101875305176, "gen/reward_std": 0.38293835520744324, "grad_norm": 0.004236552864313126, "learning_rate": 1e-05, "loss": 0.0027, "rewards/perceptual_score_reward_func/mean": 0.43139415979385376, "rewards/perceptual_score_reward_func/std": 0.3872426748275757, "step": 93, "time_profile/curr_logprobs_and_update": 0.28935843985527754, "time_profile/image_rollout": 90.9723764192313, "time_profile/old_logps": 35.630882170051336, "time_profile/ref_logps": 35.66354006715119, "time_profile/reward": 1.4163836408406496, "train/gen_step": 5892.0, "unified/clip_ratio/high_mean": 0.02233631252693158, "unified/clip_ratio/low_mean": 0.00026049184361909283, "unified/clip_ratio/region_mean": 0.022596804475142562, "unified/kl": 0.11128485442895908, "unified/loss": 0.0008558991967220209 }, { "epoch": 1.1921537229783827, "grad_norm": 0.003623092547059059, "learning_rate": 1e-05, "loss": 0.001, "step": 94, "time_profile/curr_logprobs_and_update": 0.29402141278842464, "train/gen_step": 5956.0, "unified/clip_ratio/high_mean": 0.0002511533584765857, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0002511533584765857, "unified/kl": 0.024947728699771687, "unified/loss": 0.0009469758740578982 }, { "epoch": 1.2049639711769415, "gen/advantages_abs_mean": 0.061276111751794815, "gen/advantages_max": 0.27673643827438354, "gen/advantages_mean": -1.3969838619232178e-09, "gen/advantages_min": -0.24275538325309753, "gen/advantages_std": 0.07711324840784073, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 126.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.29714375734329224, "gen/reward_std": 0.09595662355422974, "grad_norm": 0.004491707310080528, "learning_rate": 1e-05, "loss": -0.0, "rewards/perceptual_score_reward_func/mean": 0.2941877245903015, "rewards/perceptual_score_reward_func/std": 0.10239575803279877, "step": 95, "time_profile/curr_logprobs_and_update": 0.2845701384358108, "time_profile/image_rollout": 94.30383717268705, "time_profile/old_logps": 35.69289446435869, "time_profile/ref_logps": 35.735097793862224, "time_profile/reward": 1.5108983255922794, "train/gen_step": 6020.0, "unified/clip_ratio/high_mean": 0.0010795590387715492, "unified/clip_ratio/low_mean": 2.6937110305880196e-05, "unified/clip_ratio/region_mean": 0.0011064961481679347, "unified/kl": 0.024600972144980915, "unified/loss": 0.0015992923219982913 }, { "epoch": 1.2177742193755003, "grad_norm": 0.004685665015131235, "learning_rate": 1e-05, "loss": 0.001, "step": 96, "time_profile/curr_logprobs_and_update": 0.2889473946997896, "train/gen_step": 6084.0, "unified/clip_ratio/high_mean": 0.00010233268585579935, "unified/clip_ratio/low_mean": 2.49623572017299e-05, "unified/clip_ratio/region_mean": 0.00012729504305752926, "unified/kl": 0.02420059047290124, "unified/loss": 0.0009602482255104405 }, { "epoch": 1.2305844675740594, "gen/advantages_abs_mean": 0.07217675447463989, "gen/advantages_max": 0.22240379452705383, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.36782652139663696, "gen/advantages_std": 0.09157998859882355, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.606522798538208, "gen/reward_std": 0.1440057009458542, "grad_norm": 0.0029499332886189222, "learning_rate": 1e-05, "loss": 0.002, "rewards/perceptual_score_reward_func/mean": 0.6199157238006592, "rewards/perceptual_score_reward_func/std": 0.13665904104709625, "step": 97, "time_profile/curr_logprobs_and_update": 0.29385396288125776, "time_profile/image_rollout": 97.1500741597265, "time_profile/old_logps": 35.72174118645489, "time_profile/ref_logps": 35.81614971533418, "time_profile/reward": 1.2525407243520021, "train/gen_step": 6148.0, "unified/clip_ratio/high_mean": 0.0007662127018193132, "unified/clip_ratio/low_mean": 0.00015251044169417582, "unified/clip_ratio/region_mean": 0.0009187231389660155, "unified/kl": 0.026670520936022513, "unified/loss": -0.0007936091969895642 }, { "epoch": 1.2433947157726182, "grad_norm": 0.004023274406790733, "learning_rate": 1e-05, "loss": 0.0041, "step": 98, "time_profile/curr_logprobs_and_update": 0.286594680743292, "train/gen_step": 6212.0, "unified/clip_ratio/high_mean": 0.001166129409284622, "unified/clip_ratio/low_mean": 2.7221584787184838e-05, "unified/clip_ratio/region_mean": 0.001193350993162312, "unified/kl": 0.1031805292586796, "unified/loss": 0.0034758291503749206 }, { "epoch": 1.256204963971177, "gen/advantages_abs_mean": 0.05734759569168091, "gen/advantages_max": 0.1668534278869629, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.24680942296981812, "gen/advantages_std": 0.0720229223370552, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.4030495882034302, "gen/reward_std": 0.30972352623939514, "grad_norm": 0.005159635562449694, "learning_rate": 1e-05, "loss": 0.0054, "rewards/perceptual_score_reward_func/mean": 0.40351754426956177, "rewards/perceptual_score_reward_func/std": 0.30575186014175415, "step": 99, "time_profile/curr_logprobs_and_update": 0.2843075899290852, "time_profile/image_rollout": 90.91732196509838, "time_profile/old_logps": 35.70482533983886, "time_profile/ref_logps": 35.787240678444505, "time_profile/reward": 1.3005148079246283, "train/gen_step": 6276.0, "unified/clip_ratio/high_mean": 0.005653069780237274, "unified/clip_ratio/low_mean": 0.0001404002250637859, "unified/clip_ratio/region_mean": 0.005793470003482071, "unified/kl": 0.10310261632548645, "unified/loss": 0.008152539892307686 }, { "epoch": 1.2690152121697358, "grad_norm": 0.005166908260434866, "learning_rate": 1e-05, "loss": 0.0044, "step": 100, "time_profile/curr_logprobs_and_update": 0.28935047722188756, "train/gen_step": 6340.0, "unified/clip_ratio/high_mean": 0.0013726951256103348, "unified/clip_ratio/low_mean": 0.00010761726116470527, "unified/clip_ratio/region_mean": 0.0014803123895035242, "unified/kl": 0.11148487494210713, "unified/loss": 0.005707373344193911 }, { "epoch": 1.2818254603682946, "gen/advantages_abs_mean": 0.05996637046337128, "gen/advantages_max": 0.22481316328048706, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.28761735558509827, "gen/advantages_std": 0.0771557167172432, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 37.0, "gen/prompt/mean_length": 33.5, "gen/prompt/min_length": 30.0, "gen/reward": 0.4563029110431671, "gen/reward_std": 0.1505608707666397, "grad_norm": 0.0037838416174054146, "learning_rate": 1e-05, "loss": 0.0034, "rewards/perceptual_score_reward_func/mean": 0.45866450667381287, "rewards/perceptual_score_reward_func/std": 0.15732455253601074, "step": 101, "time_profile/curr_logprobs_and_update": 0.2890072492300533, "time_profile/image_rollout": 89.5529919564724, "time_profile/old_logps": 35.6248143799603, "time_profile/ref_logps": 35.6683790050447, "time_profile/reward": 1.463647324591875, "train/gen_step": 6404.0, "unified/clip_ratio/high_mean": 0.00770890203784802, "unified/clip_ratio/low_mean": 0.0010239453413305455, "unified/clip_ratio/region_mean": 0.008732847421924816, "unified/kl": 0.10659060714533553, "unified/loss": 0.001588718660059385 }, { "epoch": 1.2946357085668536, "grad_norm": 0.0035527099389582872, "learning_rate": 1e-05, "loss": 0.0012, "step": 102, "time_profile/curr_logprobs_and_update": 0.29435791762080044, "train/gen_step": 6468.0, "unified/clip_ratio/high_mean": 0.00019829840948659694, "unified/clip_ratio/low_mean": 1.2361550943751354e-05, "unified/clip_ratio/region_mean": 0.0002106599595208536, "unified/kl": 0.029273734951857477, "unified/loss": 0.0010736352372759939 }, { "epoch": 1.3074459567654122, "gen/advantages_abs_mean": 0.05255986750125885, "gen/advantages_max": 0.26030582189559937, "gen/advantages_mean": 2.9103830456733704e-11, "gen/advantages_min": -0.1432579606771469, "gen/advantages_std": 0.06540312618017197, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 159.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.0877436026930809, "gen/reward_std": 0.07358266413211823, "grad_norm": 0.004366860259324312, "learning_rate": 1e-05, "loss": 0.002, "rewards/perceptual_score_reward_func/mean": 0.07803992182016373, "rewards/perceptual_score_reward_func/std": 0.07350844889879227, "step": 103, "time_profile/curr_logprobs_and_update": 0.2862840360903647, "time_profile/image_rollout": 92.44728139974177, "time_profile/old_logps": 35.6767134424299, "time_profile/ref_logps": 35.706137884408236, "time_profile/reward": 1.3225576486438513, "train/gen_step": 6532.0, "unified/clip_ratio/high_mean": 0.0006179989059091895, "unified/clip_ratio/low_mean": 2.4732017664064188e-05, "unified/clip_ratio/region_mean": 0.0006427309235732537, "unified/kl": 0.030823647743090987, "unified/loss": 0.0035945791969425045 }, { "epoch": 1.3202562049639712, "grad_norm": 0.0035135592333972454, "learning_rate": 1e-05, "loss": 0.0013, "step": 104, "time_profile/curr_logprobs_and_update": 0.28596270937123336, "train/gen_step": 6596.0, "unified/clip_ratio/high_mean": 4.980298490409041e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 4.980298490409041e-05, "unified/kl": 0.032854080345714465, "unified/loss": 0.001322075961070368 }, { "epoch": 1.33306645316253, "gen/advantages_abs_mean": 0.0665295422077179, "gen/advantages_max": 0.19884777069091797, "gen/advantages_mean": 2.3283064365386963e-09, "gen/advantages_min": -0.34449654817581177, "gen/advantages_std": 0.08466022461652756, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.7172415256500244, "gen/reward_std": 0.09059640765190125, "grad_norm": 0.0032354136928915977, "learning_rate": 1e-05, "loss": 0.0001, "rewards/perceptual_score_reward_func/mean": 0.707398533821106, "rewards/perceptual_score_reward_func/std": 0.0924367904663086, "step": 105, "time_profile/curr_logprobs_and_update": 0.2953199516341556, "time_profile/image_rollout": 88.1711419057101, "time_profile/old_logps": 35.69154427945614, "time_profile/ref_logps": 35.82033765874803, "time_profile/reward": 0.9439821243286133, "train/gen_step": 6660.0, "unified/clip_ratio/high_mean": 0.0004632073678294546, "unified/clip_ratio/low_mean": 0.0002651929771673167, "unified/clip_ratio/region_mean": 0.0007284003477252554, "unified/kl": 0.0403441485541407, "unified/loss": 0.0012512003850133624 }, { "epoch": 1.3458767013610888, "grad_norm": 0.003011377528309822, "learning_rate": 1e-05, "loss": 0.0065, "step": 106, "time_profile/curr_logprobs_and_update": 0.2940919303218834, "train/gen_step": 6724.0, "unified/clip_ratio/high_mean": 0.00030953719215176534, "unified/clip_ratio/low_mean": 8.872071430232609e-05, "unified/clip_ratio/region_mean": 0.00039825790736358613, "unified/kl": 0.1624628723366186, "unified/loss": 0.006346872902213363 }, { "epoch": 1.3586869495596476, "gen/advantages_abs_mean": 0.028656376525759697, "gen/advantages_max": 0.07740437984466553, "gen/advantages_mean": 4.889443516731262e-09, "gen/advantages_min": -0.1487799882888794, "gen/advantages_std": 0.03779109567403793, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 39.0, "gen/prompt/mean_length": 36.5, "gen/prompt/min_length": 34.0, "gen/reward": 0.7954732179641724, "gen/reward_std": 0.042653292417526245, "grad_norm": 0.0025170508306473494, "learning_rate": 1e-05, "loss": 0.0067, "rewards/perceptual_score_reward_func/mean": 0.7931220531463623, "rewards/perceptual_score_reward_func/std": 0.038766633719205856, "step": 107, "time_profile/curr_logprobs_and_update": 0.28722655613091774, "time_profile/image_rollout": 85.07742638885975, "time_profile/old_logps": 35.61971877142787, "time_profile/ref_logps": 35.68408760428429, "time_profile/reward": 1.3477066438645124, "train/gen_step": 6788.0, "unified/clip_ratio/high_mean": 0.0007559659843536792, "unified/clip_ratio/low_mean": 0.00031796366874914384, "unified/clip_ratio/region_mean": 0.0010739296567408019, "unified/kl": 0.14353025154559873, "unified/loss": 0.004765539683660336 }, { "epoch": 1.3714971977582064, "grad_norm": 0.00173460494261235, "learning_rate": 1e-05, "loss": 0.0013, "step": 108, "time_profile/curr_logprobs_and_update": 0.2885909783653915, "train/gen_step": 6852.0, "unified/clip_ratio/high_mean": 0.00010551881587161915, "unified/clip_ratio/low_mean": 1.3563368156610522e-05, "unified/clip_ratio/region_mean": 0.00011908218402822968, "unified/kl": 0.033308936399407685, "unified/loss": 0.001296318120239448 }, { "epoch": 1.3843074459567655, "gen/advantages_abs_mean": 0.0636034607887268, "gen/advantages_max": 0.3713875412940979, "gen/advantages_mean": 1.6298145055770874e-09, "gen/advantages_min": -0.274211585521698, "gen/advantages_std": 0.09082217514514923, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 32.0, "gen/prompt/mean_length": 29.5, "gen/prompt/min_length": 27.0, "gen/reward": 0.5426168441772461, "gen/reward_std": 0.2932545244693756, "grad_norm": 0.001911607920192182, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.5345706343650818, "rewards/perceptual_score_reward_func/std": 0.29058483242988586, "step": 109, "time_profile/curr_logprobs_and_update": 0.28650623420253396, "time_profile/image_rollout": 91.26650758460164, "time_profile/old_logps": 35.63152730278671, "time_profile/ref_logps": 35.694553695619106, "time_profile/reward": 1.3631694037467241, "train/gen_step": 6916.0, "unified/clip_ratio/high_mean": 0.0006030555850884411, "unified/clip_ratio/low_mean": 0.0001932560098794056, "unified/clip_ratio/region_mean": 0.0007963115995153203, "unified/kl": 0.032178554829442874, "unified/loss": 0.0007431395524690743 }, { "epoch": 1.3971176941553243, "grad_norm": 0.002165607176721096, "learning_rate": 1e-05, "loss": 0.0012, "step": 110, "time_profile/curr_logprobs_and_update": 0.28561883803922683, "train/gen_step": 6980.0, "unified/clip_ratio/high_mean": 0.00024020071941777132, "unified/clip_ratio/low_mean": 5.193096149014309e-05, "unified/clip_ratio/region_mean": 0.0002921316809079144, "unified/kl": 0.031306360120652243, "unified/loss": 0.0011849405273096636 }, { "epoch": 1.409927942353883, "gen/advantages_abs_mean": 0.07240858674049377, "gen/advantages_max": 0.26393023133277893, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.3260378837585449, "gen/advantages_std": 0.0921124517917633, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 159.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.422931969165802, "gen/reward_std": 0.16759854555130005, "grad_norm": 0.0034576484467834234, "learning_rate": 1e-05, "loss": 0.0, "rewards/perceptual_score_reward_func/mean": 0.4374734163284302, "rewards/perceptual_score_reward_func/std": 0.15218466520309448, "step": 111, "time_profile/curr_logprobs_and_update": 0.2904707919806242, "time_profile/image_rollout": 94.59738933853805, "time_profile/old_logps": 35.6568603720516, "time_profile/ref_logps": 35.68659848533571, "time_profile/reward": 1.427611980587244, "train/gen_step": 7044.0, "unified/clip_ratio/high_mean": 0.0007604033617099049, "unified/clip_ratio/low_mean": 0.00046580425623687916, "unified/clip_ratio/region_mean": 0.0012262076288607204, "unified/kl": 0.03141445969231427, "unified/loss": -0.0003618052833189722 }, { "epoch": 1.4227381905524419, "grad_norm": 0.004003474954515696, "learning_rate": 1e-05, "loss": 0.0016, "step": 112, "time_profile/curr_logprobs_and_update": 0.2907797862426378, "train/gen_step": 7108.0, "unified/clip_ratio/high_mean": 0.00041981838876381516, "unified/clip_ratio/low_mean": 0.00023284091003006324, "unified/clip_ratio/region_mean": 0.0006526592924274155, "unified/kl": 0.04061797269969247, "unified/loss": 0.0016009146888791292 }, { "epoch": 1.4355484387510007, "gen/advantages_abs_mean": 0.03790595009922981, "gen/advantages_max": 0.1838318109512329, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.1129482239484787, "gen/advantages_std": 0.04889063909649849, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 30.0, "gen/prompt/mean_length": 27.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.03915974497795105, "gen/reward_std": 0.05308791622519493, "grad_norm": 0.003365787910297513, "learning_rate": 1e-05, "loss": 0.0033, "rewards/perceptual_score_reward_func/mean": 0.04437080770730972, "rewards/perceptual_score_reward_func/std": 0.055337294936180115, "step": 113, "time_profile/curr_logprobs_and_update": 0.2875677521515172, "time_profile/image_rollout": 93.94608847610652, "time_profile/old_logps": 35.64288152009249, "time_profile/ref_logps": 35.68590772897005, "time_profile/reward": 1.4800133537501097, "train/gen_step": 7172.0, "unified/clip_ratio/high_mean": 0.0013470338944898685, "unified/clip_ratio/low_mean": 0.00042899202344415244, "unified/clip_ratio/region_mean": 0.0017760259106580634, "unified/kl": 0.03961936090490781, "unified/loss": 0.0038841106043037144 }, { "epoch": 1.4483586869495597, "grad_norm": 0.0020599612034857273, "learning_rate": 1e-05, "loss": 0.0013, "step": 114, "time_profile/curr_logprobs_and_update": 0.29330829688115045, "train/gen_step": 7236.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 1.4256386748456862e-05, "unified/clip_ratio/region_mean": 1.4256386748456862e-05, "unified/kl": 0.033574721310287714, "unified/loss": 0.001462343894672813 }, { "epoch": 1.4611689351481185, "gen/advantages_abs_mean": 0.044344015419483185, "gen/advantages_max": 0.2023053765296936, "gen/advantages_mean": 1.1641532182693481e-09, "gen/advantages_min": -0.2494000792503357, "gen/advantages_std": 0.06300931423902512, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 33.0, "gen/prompt/mean_length": 29.5, "gen/prompt/min_length": 26.0, "gen/reward": 0.643923819065094, "gen/reward_std": 0.17081858217716217, "grad_norm": 0.004796881694346666, "learning_rate": 1e-05, "loss": 0.0012, "rewards/perceptual_score_reward_func/mean": 0.6462793350219727, "rewards/perceptual_score_reward_func/std": 0.174828439950943, "step": 115, "time_profile/curr_logprobs_and_update": 0.2918398874462582, "time_profile/image_rollout": 90.91947788372636, "time_profile/old_logps": 35.63139848783612, "time_profile/ref_logps": 35.679666850715876, "time_profile/reward": 1.5054155122488737, "train/gen_step": 7300.0, "unified/clip_ratio/high_mean": 5.238996800471796e-05, "unified/clip_ratio/low_mean": 5.107643664814532e-05, "unified/clip_ratio/region_mean": 0.00010346640465286328, "unified/kl": 0.03278353181667626, "unified/loss": 0.003379457935807295 }, { "epoch": 1.4739791833466773, "grad_norm": 0.0059644728899002075, "learning_rate": 1e-05, "loss": 0.0012, "step": 116, "time_profile/curr_logprobs_and_update": 0.2902962447842583, "train/gen_step": 7364.0, "unified/clip_ratio/high_mean": 0.00014891066621203208, "unified/clip_ratio/low_mean": 0.00020414480513863964, "unified/clip_ratio/region_mean": 0.0003530554713506717, "unified/kl": 0.03125500594615005, "unified/loss": 0.0011424217318563024 }, { "epoch": 1.4867894315452361, "gen/advantages_abs_mean": 0.056864481419324875, "gen/advantages_max": 0.19812063872814178, "gen/advantages_mean": -2.3283064365386963e-09, "gen/advantages_min": -0.19689124822616577, "gen/advantages_std": 0.06950084120035172, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 126.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.36290961503982544, "gen/reward_std": 0.2883332371711731, "grad_norm": 0.00461900420486927, "learning_rate": 1e-05, "loss": 0.0006, "rewards/perceptual_score_reward_func/mean": 0.3551962971687317, "rewards/perceptual_score_reward_func/std": 0.2728025019168854, "step": 117, "time_profile/curr_logprobs_and_update": 0.28732451432733797, "time_profile/image_rollout": 100.65153944864869, "time_profile/old_logps": 35.655188797041774, "time_profile/ref_logps": 35.70040735602379, "time_profile/reward": 1.2644598968327045, "train/gen_step": 7428.0, "unified/clip_ratio/high_mean": 0.00042217884856654564, "unified/clip_ratio/low_mean": 0.0009703336363600101, "unified/clip_ratio/region_mean": 0.0013925124894740293, "unified/kl": 0.031373919220641255, "unified/loss": -0.001355293581582373 }, { "epoch": 1.499599679743795, "grad_norm": 0.0028887134976685047, "learning_rate": 1e-05, "loss": 0.0018, "step": 118, "time_profile/curr_logprobs_and_update": 0.28666515793884173, "train/gen_step": 7492.0, "unified/clip_ratio/high_mean": 9.078754828806268e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 9.078754828806268e-05, "unified/kl": 0.04551572597119957, "unified/loss": 0.0018532902213337366 }, { "epoch": 1.512409927942354, "gen/advantages_abs_mean": 0.0431256927549839, "gen/advantages_max": 0.1255958080291748, "gen/advantages_mean": 2.3283064365386963e-09, "gen/advantages_min": -0.21674871444702148, "gen/advantages_std": 0.05766208469867706, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.7449733018875122, "gen/reward_std": 0.06443611532449722, "grad_norm": 0.007539812941104174, "learning_rate": 1e-05, "loss": 0.0024, "rewards/perceptual_score_reward_func/mean": 0.7542487978935242, "rewards/perceptual_score_reward_func/std": 0.061425261199474335, "step": 119, "time_profile/curr_logprobs_and_update": 0.2824957420816645, "time_profile/image_rollout": 95.6516018435359, "time_profile/old_logps": 35.68137123994529, "time_profile/ref_logps": 35.80113796517253, "time_profile/reward": 0.9460577331483364, "train/gen_step": 7556.0, "unified/clip_ratio/high_mean": 0.0005107632741783164, "unified/clip_ratio/low_mean": 0.00025477437520748936, "unified/clip_ratio/region_mean": 0.000765537648476311, "unified/kl": 0.054741554835345596, "unified/loss": 0.002122770885762293 }, { "epoch": 1.5252201761409128, "grad_norm": 0.002637062221765518, "learning_rate": 1e-05, "loss": 0.0053, "step": 120, "time_profile/curr_logprobs_and_update": 0.2882592770329211, "train/gen_step": 7620.0, "unified/clip_ratio/high_mean": 0.0011127292445962667, "unified/clip_ratio/low_mean": 0.0010107289936058805, "unified/clip_ratio/region_mean": 0.0021234582272882108, "unified/kl": 0.13152603071648628, "unified/loss": 0.005372763577724982 }, { "epoch": 1.5380304243394716, "gen/advantages_abs_mean": 0.05986716225743294, "gen/advantages_max": 0.13977259397506714, "gen/advantages_mean": -4.190951585769653e-09, "gen/advantages_min": -0.3370237946510315, "gen/advantages_std": 0.07795548439025879, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.7155458927154541, "gen/reward_std": 0.09138889610767365, "grad_norm": 0.002922995714470744, "learning_rate": 1e-05, "loss": 0.0055, "rewards/perceptual_score_reward_func/mean": 0.7051100730895996, "rewards/perceptual_score_reward_func/std": 0.1026398092508316, "step": 121, "time_profile/curr_logprobs_and_update": 0.2949224690091796, "time_profile/image_rollout": 93.05562633834779, "time_profile/old_logps": 35.69093172252178, "time_profile/ref_logps": 35.76676655560732, "time_profile/reward": 0.9614674616605043, "train/gen_step": 7684.0, "unified/clip_ratio/high_mean": 0.0037278791087373975, "unified/clip_ratio/low_mean": 0.00411738019920449, "unified/clip_ratio/region_mean": 0.007845259402529337, "unified/kl": 0.13341835048049688, "unified/loss": 0.0043207566068304 }, { "epoch": 1.5508406725380304, "grad_norm": 0.0042500728741288185, "learning_rate": 1e-05, "loss": 0.004, "step": 122, "time_profile/curr_logprobs_and_update": 0.2829219346458558, "train/gen_step": 7748.0, "unified/clip_ratio/high_mean": 0.0016792991782494937, "unified/clip_ratio/low_mean": 0.0006439993148887879, "unified/clip_ratio/region_mean": 0.0023232984931382816, "unified/kl": 0.09978884411975741, "unified/loss": 0.0038766780953665148 }, { "epoch": 1.5636509207365892, "gen/advantages_abs_mean": 0.05314895510673523, "gen/advantages_max": 0.28923487663269043, "gen/advantages_mean": -1.5133991837501526e-09, "gen/advantages_min": -0.27263307571411133, "gen/advantages_std": 0.07695531100034714, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 27.0, "gen/prompt/mean_length": 25.5, "gen/prompt/min_length": 24.0, "gen/reward": 0.5273290276527405, "gen/reward_std": 0.3176587224006653, "grad_norm": 0.005965746007859707, "learning_rate": 1e-05, "loss": 0.0063, "rewards/perceptual_score_reward_func/mean": 0.528980553150177, "rewards/perceptual_score_reward_func/std": 0.31866586208343506, "step": 123, "time_profile/curr_logprobs_and_update": 0.28925655310740694, "time_profile/image_rollout": 85.26398288831115, "time_profile/old_logps": 35.60682420618832, "time_profile/ref_logps": 35.67962844111025, "time_profile/reward": 1.2899844255298376, "train/gen_step": 7812.0, "unified/clip_ratio/high_mean": 0.006747730705683352, "unified/clip_ratio/low_mean": 0.0036483843277892447, "unified/clip_ratio/region_mean": 0.010396115059847943, "unified/kl": 0.09209522936725989, "unified/loss": 0.003992128267327644 }, { "epoch": 1.5764611689351482, "grad_norm": 0.006018125917762518, "learning_rate": 1e-05, "loss": 0.0025, "step": 124, "time_profile/curr_logprobs_and_update": 0.2918457875493914, "train/gen_step": 7876.0, "unified/clip_ratio/high_mean": 0.00030464172141364543, "unified/clip_ratio/low_mean": 0.0022733521536792978, "unified/clip_ratio/region_mean": 0.0025779938796404167, "unified/kl": 0.06117875926429406, "unified/loss": 0.0025565675814966937 }, { "epoch": 1.589271417133707, "gen/advantages_abs_mean": 0.061115965247154236, "gen/advantages_max": 0.1915382742881775, "gen/advantages_mean": -9.313225746154785e-10, "gen/advantages_min": -0.31723618507385254, "gen/advantages_std": 0.07902835309505463, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 185.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.5624587535858154, "gen/reward_std": 0.09737126529216766, "grad_norm": 0.01172794308513403, "learning_rate": 1e-05, "loss": 0.0009, "rewards/perceptual_score_reward_func/mean": 0.5736739635467529, "rewards/perceptual_score_reward_func/std": 0.07946990430355072, "step": 125, "time_profile/curr_logprobs_and_update": 0.2807448592793662, "time_profile/image_rollout": 84.8983690943569, "time_profile/old_logps": 35.744687812402844, "time_profile/ref_logps": 35.80844911374152, "time_profile/reward": 1.2122203204780817, "train/gen_step": 7940.0, "unified/clip_ratio/high_mean": 0.0011306394399070996, "unified/clip_ratio/low_mean": 0.011417139139666688, "unified/clip_ratio/region_mean": 0.01254777849317179, "unified/kl": 0.07883448613574728, "unified/loss": 0.0028802082304082433 }, { "epoch": 1.6020816653322658, "grad_norm": 0.00505675608292222, "learning_rate": 1e-05, "loss": 0.0041, "step": 126, "time_profile/curr_logprobs_and_update": 0.28650646412279457, "train/gen_step": 8004.0, "unified/clip_ratio/high_mean": 0.07535966531486338, "unified/clip_ratio/low_mean": 0.0053497172903007595, "unified/clip_ratio/region_mean": 0.08070938309538178, "unified/kl": 0.08585448877420276, "unified/loss": 0.004471837332857831 }, { "epoch": 1.6148919135308246, "gen/advantages_abs_mean": 0.07567422091960907, "gen/advantages_max": 0.2743659019470215, "gen/advantages_mean": -1.6298145055770874e-09, "gen/advantages_min": -0.23675748705863953, "gen/advantages_std": 0.09291587769985199, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 83.5, "gen/prompt/min_length": 30.0, "gen/reward": 0.264026939868927, "gen/reward_std": 0.15199002623558044, "grad_norm": 0.00418127654120326, "learning_rate": 1e-05, "loss": 0.0059, "rewards/perceptual_score_reward_func/mean": 0.24559339880943298, "rewards/perceptual_score_reward_func/std": 0.15700195729732513, "step": 127, "time_profile/curr_logprobs_and_update": 0.287341259769164, "time_profile/image_rollout": 85.78850851021707, "time_profile/old_logps": 35.69921772927046, "time_profile/ref_logps": 35.74098035879433, "time_profile/reward": 1.3971008583903313, "train/gen_step": 8068.0, "unified/clip_ratio/high_mean": 0.10724178397231299, "unified/clip_ratio/low_mean": 0.015299416099878727, "unified/clip_ratio/region_mean": 0.12254120000216062, "unified/kl": 0.06688741553807631, "unified/loss": 0.008423287796631485 }, { "epoch": 1.6277021617293834, "grad_norm": 0.004291560500860214, "learning_rate": 1e-05, "loss": 0.0016, "step": 128, "time_profile/curr_logprobs_and_update": 0.2904924996837508, "train/gen_step": 8132.0, "unified/clip_ratio/high_mean": 3.791005929087987e-05, "unified/clip_ratio/low_mean": 2.5746619030542206e-05, "unified/clip_ratio/region_mean": 6.365667832142208e-05, "unified/kl": 0.04011628415901214, "unified/loss": 0.0016968616482699872 }, { "epoch": 1.6405124099279424, "gen/advantages_abs_mean": 0.07780550420284271, "gen/advantages_max": 0.33822619915008545, "gen/advantages_mean": 2.0372681319713593e-09, "gen/advantages_min": -0.32445240020751953, "gen/advantages_std": 0.10094477981328964, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 91.5, "gen/prompt/min_length": 40.0, "gen/reward": 0.38963890075683594, "gen/reward_std": 0.27400505542755127, "grad_norm": 0.0064746844582259655, "learning_rate": 1e-05, "loss": 0.0025, "rewards/perceptual_score_reward_func/mean": 0.39440298080444336, "rewards/perceptual_score_reward_func/std": 0.2802719473838806, "step": 129, "time_profile/curr_logprobs_and_update": 0.29365084203891456, "time_profile/image_rollout": 92.23527741245925, "time_profile/old_logps": 35.69436052814126, "time_profile/ref_logps": 35.73389621451497, "time_profile/reward": 1.062905428931117, "train/gen_step": 8196.0, "unified/clip_ratio/high_mean": 0.0001702761464912328, "unified/clip_ratio/low_mean": 0.0010935042027995223, "unified/clip_ratio/region_mean": 0.0012637803538382286, "unified/kl": 0.0411043684289325, "unified/loss": 0.004106405973288929 }, { "epoch": 1.6533226581265013, "grad_norm": 0.006903781555593014, "learning_rate": 1e-05, "loss": 0.0035, "step": 130, "time_profile/curr_logprobs_and_update": 0.28863042485318147, "train/gen_step": 8260.0, "unified/clip_ratio/high_mean": 0.01118387773840368, "unified/clip_ratio/low_mean": 0.00042925817706418457, "unified/clip_ratio/region_mean": 0.011613136058258533, "unified/kl": 0.08873327681794763, "unified/loss": 0.0033595796521694865 }, { "epoch": 1.66613290632506, "gen/advantages_abs_mean": 0.02932814322412014, "gen/advantages_max": 0.15440410375595093, "gen/advantages_mean": 1.1641532182693481e-09, "gen/advantages_min": -0.2082865834236145, "gen/advantages_std": 0.05175040289759636, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.5, "gen/prompt/max_length": 36.0, "gen/prompt/mean_length": 35.0, "gen/prompt/min_length": 34.0, "gen/reward": 0.35747069120407104, "gen/reward_std": 0.36191922426223755, "grad_norm": 0.00403651362285018, "learning_rate": 1e-05, "loss": 0.0036, "rewards/perceptual_score_reward_func/mean": 0.35938212275505066, "rewards/perceptual_score_reward_func/std": 0.36509737372398376, "step": 131, "time_profile/curr_logprobs_and_update": 0.2961919621739071, "time_profile/image_rollout": 96.8908116389066, "time_profile/old_logps": 35.64496449753642, "time_profile/ref_logps": 35.696318335831165, "time_profile/reward": 1.4984830934554338, "train/gen_step": 8324.0, "unified/clip_ratio/high_mean": 0.03772879852567712, "unified/clip_ratio/low_mean": 0.003505356321511499, "unified/clip_ratio/region_mean": 0.041234154683479574, "unified/kl": 0.09730945219052956, "unified/loss": 0.0015052031849336345 }, { "epoch": 1.6789431545236189, "grad_norm": 0.0016463575884699821, "learning_rate": 1e-05, "loss": 0.0017, "step": 132, "time_profile/curr_logprobs_and_update": 0.2821661222842522, "train/gen_step": 8388.0, "unified/clip_ratio/high_mean": 0.0008666838857607218, "unified/clip_ratio/low_mean": 1.3754401152255014e-05, "unified/clip_ratio/region_mean": 0.0008804382905509556, "unified/kl": 0.042414410738274455, "unified/loss": 0.0016847542519826675 }, { "epoch": 1.6917534027221777, "gen/advantages_abs_mean": 0.05056470260024071, "gen/advantages_max": 0.18678975105285645, "gen/advantages_mean": 2.0954757928848267e-09, "gen/advantages_min": -0.365351140499115, "gen/advantages_std": 0.06941158324480057, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.7006023526191711, "gen/reward_std": 0.07870075106620789, "grad_norm": 0.00302127399481833, "learning_rate": 1e-05, "loss": 0.002, "rewards/perceptual_score_reward_func/mean": 0.715668261051178, "rewards/perceptual_score_reward_func/std": 0.05759961158037186, "step": 133, "time_profile/curr_logprobs_and_update": 0.29358167183818296, "time_profile/image_rollout": 89.95654336549342, "time_profile/old_logps": 35.73023318499327, "time_profile/ref_logps": 35.7922641094774, "time_profile/reward": 0.9506809506565332, "train/gen_step": 8452.0, "unified/clip_ratio/high_mean": 0.003891350916092051, "unified/clip_ratio/low_mean": 7.476141399820335e-05, "unified/clip_ratio/region_mean": 0.003966112333728233, "unified/kl": 0.053177921363385394, "unified/loss": 0.000371901216567494 }, { "epoch": 1.7045636509207367, "grad_norm": 0.0037558316253125668, "learning_rate": 1e-05, "loss": 0.0091, "step": 134, "time_profile/curr_logprobs_and_update": 0.29027548429439776, "train/gen_step": 8516.0, "unified/clip_ratio/high_mean": 0.0017724815052133636, "unified/clip_ratio/low_mean": 3.741087857633829e-05, "unified/clip_ratio/region_mean": 0.001809892386518186, "unified/kl": 0.22769839805550873, "unified/loss": 0.009044620092936384 }, { "epoch": 1.7173738991192953, "gen/advantages_abs_mean": 0.06249502673745155, "gen/advantages_max": 0.18479061126708984, "gen/advantages_mean": -1.862645149230957e-09, "gen/advantages_min": -0.28166067600250244, "gen/advantages_std": 0.0770028680562973, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 215.5, "gen/prompt/min_length": 137.0, "gen/reward": 0.4174192249774933, "gen/reward_std": 0.23804545402526855, "grad_norm": 0.003767220536246896, "learning_rate": 1e-05, "loss": 0.0071, "rewards/perceptual_score_reward_func/mean": 0.41816747188568115, "rewards/perceptual_score_reward_func/std": 0.2535645663738251, "step": 135, "time_profile/curr_logprobs_and_update": 0.28429412076366134, "time_profile/image_rollout": 102.21069337241352, "time_profile/old_logps": 35.70307997614145, "time_profile/ref_logps": 35.79909221455455, "time_profile/reward": 1.4969095662236214, "train/gen_step": 8580.0, "unified/clip_ratio/high_mean": 0.0030967974716986646, "unified/clip_ratio/low_mean": 0.0007302079957298702, "unified/clip_ratio/region_mean": 0.003827005463790556, "unified/kl": 0.21062234381679446, "unified/loss": 0.010004522626331891 }, { "epoch": 1.7301841473178543, "grad_norm": 0.004301386885344982, "learning_rate": 1e-05, "loss": 0.0022, "step": 136, "time_profile/curr_logprobs_and_update": 0.29434079653583467, "train/gen_step": 8644.0, "unified/clip_ratio/high_mean": 0.0004625109258995508, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0004625109258995508, "unified/kl": 0.05430893413722515, "unified/loss": 0.0022077317435105215 }, { "epoch": 1.742994395516413, "gen/advantages_abs_mean": 0.06403308361768723, "gen/advantages_max": 0.33869343996047974, "gen/advantages_mean": 2.0954757928848267e-09, "gen/advantages_min": -0.20772579312324524, "gen/advantages_std": 0.08191482722759247, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.40738505125045776, "gen/reward_std": 0.30709385871887207, "grad_norm": 0.004496556706726551, "learning_rate": 1e-05, "loss": 0.0056, "rewards/perceptual_score_reward_func/mean": 0.4147990345954895, "rewards/perceptual_score_reward_func/std": 0.3243771195411682, "step": 137, "time_profile/curr_logprobs_and_update": 0.2963087991520297, "time_profile/image_rollout": 91.59672148153186, "time_profile/old_logps": 35.74044441990554, "time_profile/ref_logps": 35.80338528752327, "time_profile/reward": 2.680176628753543, "train/gen_step": 8708.0, "unified/clip_ratio/high_mean": 0.014924995422916254, "unified/clip_ratio/low_mean": 0.0006889987589602242, "unified/clip_ratio/region_mean": 0.01561399412821629, "unified/kl": 0.05471922585275024, "unified/loss": 0.001671851314313244 }, { "epoch": 1.755804643714972, "grad_norm": 0.006244179792702198, "learning_rate": 1e-05, "loss": 0.004, "step": 138, "time_profile/curr_logprobs_and_update": 0.29202507858281024, "train/gen_step": 8772.0, "unified/clip_ratio/high_mean": 0.000596910118474625, "unified/clip_ratio/low_mean": 0.0009496516950093792, "unified/clip_ratio/region_mean": 0.0015465618116650148, "unified/kl": 0.09889116248814389, "unified/loss": 0.003862968309476855 }, { "epoch": 1.768614891913531, "gen/advantages_abs_mean": 0.06389250606298447, "gen/advantages_max": 0.23165170848369598, "gen/advantages_mean": 4.656612873077393e-10, "gen/advantages_min": -0.26901906728744507, "gen/advantages_std": 0.07976555079221725, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 137.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.18621030449867249, "gen/reward_std": 0.11706754565238953, "grad_norm": 0.006589157041162252, "learning_rate": 1e-05, "loss": 0.001, "rewards/perceptual_score_reward_func/mean": 0.18768814206123352, "rewards/perceptual_score_reward_func/std": 0.1021382138133049, "step": 139, "time_profile/curr_logprobs_and_update": 0.2866535442008171, "time_profile/image_rollout": 93.9379496704787, "time_profile/old_logps": 35.6809767074883, "time_profile/ref_logps": 35.78240888006985, "time_profile/reward": 5.514092801138759, "train/gen_step": 8836.0, "unified/clip_ratio/high_mean": 0.001998860437197436, "unified/clip_ratio/low_mean": 0.0026950046894853585, "unified/clip_ratio/region_mean": 0.0046938651212258264, "unified/kl": 0.09669042826863006, "unified/loss": 0.0023558811117254663 }, { "epoch": 1.7814251401120895, "grad_norm": 0.0057668304070830345, "learning_rate": 1e-05, "loss": 0.0014, "step": 140, "time_profile/curr_logprobs_and_update": 0.2863264895277098, "train/gen_step": 8900.0, "unified/clip_ratio/high_mean": 0.00048513619458390167, "unified/clip_ratio/low_mean": 2.7048740776081104e-05, "unified/clip_ratio/region_mean": 0.0005121849326314987, "unified/kl": 0.03493243327829987, "unified/loss": 0.001315825039455376 }, { "epoch": 1.7942353883106485, "gen/advantages_abs_mean": 0.048334699124097824, "gen/advantages_max": 0.19973474740982056, "gen/advantages_mean": -1.6298145055770874e-09, "gen/advantages_min": -0.2896121144294739, "gen/advantages_std": 0.0633920356631279, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.35955485701560974, "gen/reward_std": 0.25457412004470825, "grad_norm": 0.00502097187563777, "learning_rate": 1e-05, "loss": 0.0032, "rewards/perceptual_score_reward_func/mean": 0.3785892724990845, "rewards/perceptual_score_reward_func/std": 0.2508572041988373, "step": 141, "time_profile/curr_logprobs_and_update": 0.29334245243808255, "time_profile/image_rollout": 94.66671554744244, "time_profile/old_logps": 35.725636603310704, "time_profile/ref_logps": 35.787481896579266, "time_profile/reward": 1.2374453376978636, "train/gen_step": 8964.0, "unified/clip_ratio/high_mean": 0.008519951419657446, "unified/clip_ratio/low_mean": 0.00013759005014435388, "unified/clip_ratio/region_mean": 0.008657541471620789, "unified/kl": 0.034399580414174125, "unified/loss": 0.00496306180002648 }, { "epoch": 1.8070456365092074, "grad_norm": 0.0034979088231921196, "learning_rate": 1e-05, "loss": 0.0062, "step": 142, "time_profile/curr_logprobs_and_update": 0.28757468741969205, "train/gen_step": 9028.0, "unified/clip_ratio/high_mean": 0.0002629072369018104, "unified/clip_ratio/low_mean": 2.528555523895193e-05, "unified/clip_ratio/region_mean": 0.000288192793050257, "unified/kl": 0.1550389884505421, "unified/loss": 0.006169562449940713 }, { "epoch": 1.8198558847077662, "gen/advantages_abs_mean": 0.0765346884727478, "gen/advantages_max": 0.26568108797073364, "gen/advantages_mean": -1.3969838619232178e-09, "gen/advantages_min": -0.24493470788002014, "gen/advantages_std": 0.09568614512681961, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 126.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.2732863426208496, "gen/reward_std": 0.1075901985168457, "grad_norm": 0.003704902483150363, "learning_rate": 1e-05, "loss": 0.006, "rewards/perceptual_score_reward_func/mean": 0.2819960117340088, "rewards/perceptual_score_reward_func/std": 0.10969275236129761, "step": 143, "time_profile/curr_logprobs_and_update": 0.29061931784963235, "time_profile/image_rollout": 102.85954966209829, "time_profile/old_logps": 35.6852895244956, "time_profile/ref_logps": 35.699950790032744, "time_profile/reward": 5.994517290964723, "train/gen_step": 9092.0, "unified/clip_ratio/high_mean": 0.0024094630516628968, "unified/clip_ratio/low_mean": 0.00028702112922474043, "unified/clip_ratio/region_mean": 0.0026964841772496584, "unified/kl": 0.1615976605389733, "unified/loss": 0.0051123992066095525 }, { "epoch": 1.8326661329063252, "grad_norm": 0.00441337563097477, "learning_rate": 1e-05, "loss": 0.0016, "step": 144, "time_profile/curr_logprobs_and_update": 0.29150564002338797, "train/gen_step": 9156.0, "unified/clip_ratio/high_mean": 5.041367967351107e-05, "unified/clip_ratio/low_mean": 1.2934602636960335e-05, "unified/clip_ratio/region_mean": 6.33482823104714e-05, "unified/kl": 0.03936847756267525, "unified/loss": 0.0016968364725471474 }, { "epoch": 1.8454763811048838, "gen/advantages_abs_mean": 0.05427496135234833, "gen/advantages_max": 0.16081571578979492, "gen/advantages_mean": -1.3969838619232178e-09, "gen/advantages_min": -0.27252915501594543, "gen/advantages_std": 0.07097128033638, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 140.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.6301897764205933, "gen/reward_std": 0.10973169654607773, "grad_norm": 0.005718636326491833, "learning_rate": 1e-05, "loss": 0.0008, "rewards/perceptual_score_reward_func/mean": 0.6228070259094238, "rewards/perceptual_score_reward_func/std": 0.11832652240991592, "step": 145, "time_profile/curr_logprobs_and_update": 0.29172540063154884, "time_profile/image_rollout": 95.14329688437283, "time_profile/old_logps": 35.679359233006835, "time_profile/ref_logps": 35.740789292380214, "time_profile/reward": 1.1167409159243107, "train/gen_step": 9220.0, "unified/clip_ratio/high_mean": 0.000587068720960815, "unified/clip_ratio/low_mean": 0.000173021161572251, "unified/clip_ratio/region_mean": 0.0007600898861710448, "unified/kl": 0.04188113062991761, "unified/loss": -0.0002558948690420948 }, { "epoch": 1.8582866293034428, "grad_norm": 0.005676935892552137, "learning_rate": 1e-05, "loss": 0.0036, "step": 146, "time_profile/curr_logprobs_and_update": 0.28573712857905775, "train/gen_step": 9284.0, "unified/clip_ratio/high_mean": 0.0002970744308186113, "unified/clip_ratio/low_mean": 8.519378025084734e-05, "unified/clip_ratio/region_mean": 0.00038226821015996393, "unified/kl": 0.09014151419978589, "unified/loss": 0.003581891072826693 }, { "epoch": 1.8710968775020016, "gen/advantages_abs_mean": 0.051070600748062134, "gen/advantages_max": 0.13527166843414307, "gen/advantages_mean": -5.122274160385132e-09, "gen/advantages_min": -0.3856954574584961, "gen/advantages_std": 0.06852003186941147, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.7004803419113159, "gen/reward_std": 0.07513634115457535, "grad_norm": 0.00409558555111289, "learning_rate": 1e-05, "loss": 0.0036, "rewards/perceptual_score_reward_func/mean": 0.6796281337738037, "rewards/perceptual_score_reward_func/std": 0.08104143291711807, "step": 147, "time_profile/curr_logprobs_and_update": 0.285855452704709, "time_profile/image_rollout": 85.7736628819257, "time_profile/old_logps": 35.67286063730717, "time_profile/ref_logps": 35.779720950871706, "time_profile/reward": 1.0234551653265953, "train/gen_step": 9348.0, "unified/clip_ratio/high_mean": 0.0008755821590966661, "unified/clip_ratio/low_mean": 0.0004622845581252477, "unified/clip_ratio/region_mean": 0.0013378667035794933, "unified/kl": 0.09117380547104403, "unified/loss": 0.002188693568314193 }, { "epoch": 1.8839071257005604, "grad_norm": 0.0033700531348586082, "learning_rate": 1e-05, "loss": 0.0044, "step": 148, "time_profile/curr_logprobs_and_update": 0.2985293152742088, "train/gen_step": 9412.0, "unified/clip_ratio/high_mean": 0.002091699916491052, "unified/clip_ratio/low_mean": 6.537473291245988e-05, "unified/clip_ratio/region_mean": 0.002157074643037049, "unified/kl": 0.11110673216171563, "unified/loss": 0.004461315405933419 }, { "epoch": 1.8967173738991194, "gen/advantages_abs_mean": 0.06669476628303528, "gen/advantages_max": 0.29541581869125366, "gen/advantages_mean": -1.1932570487260818e-09, "gen/advantages_min": -0.31305474042892456, "gen/advantages_std": 0.08556267619132996, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 86.0, "gen/prompt/min_length": 35.0, "gen/reward": 0.442577600479126, "gen/reward_std": 0.3167572021484375, "grad_norm": 0.0039604133926332, "learning_rate": 1e-05, "loss": 0.0039, "rewards/perceptual_score_reward_func/mean": 0.41718146204948425, "rewards/perceptual_score_reward_func/std": 0.31259584426879883, "step": 149, "time_profile/curr_logprobs_and_update": 0.291258782352088, "time_profile/image_rollout": 85.42934694699943, "time_profile/old_logps": 35.6700346339494, "time_profile/ref_logps": 35.718164034187794, "time_profile/reward": 3.843767935410142, "train/gen_step": 9476.0, "unified/clip_ratio/high_mean": 0.005506029001480783, "unified/clip_ratio/low_mean": 0.0008064525482041063, "unified/clip_ratio/region_mean": 0.0063124815624178154, "unified/kl": 0.10582851071376354, "unified/loss": 0.005097084878798341 }, { "epoch": 1.909527622097678, "grad_norm": 0.0031276314985007048, "learning_rate": 1e-05, "loss": 0.0019, "step": 150, "time_profile/curr_logprobs_and_update": 0.2896409893874079, "train/gen_step": 9540.0, "unified/clip_ratio/high_mean": 7.758394167467486e-05, "unified/clip_ratio/low_mean": 4.173634351900546e-05, "unified/clip_ratio/region_mean": 0.00011932028519368032, "unified/kl": 0.04871381289558485, "unified/loss": 0.002004596136885084 }, { "epoch": 1.922337870296237, "gen/advantages_abs_mean": 0.06253403425216675, "gen/advantages_max": 0.22776862978935242, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.24407082796096802, "gen/advantages_std": 0.07795893400907516, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 80.5, "gen/prompt/min_length": 24.0, "gen/reward": 0.2212839424610138, "gen/reward_std": 0.10819225758314133, "grad_norm": 0.0028782610315829515, "learning_rate": 1e-05, "loss": 0.0037, "rewards/perceptual_score_reward_func/mean": 0.20518934726715088, "rewards/perceptual_score_reward_func/std": 0.11293523758649826, "step": 151, "time_profile/curr_logprobs_and_update": 0.2882380739611108, "time_profile/image_rollout": 94.66589397005737, "time_profile/old_logps": 35.84556386433542, "time_profile/ref_logps": 35.70588610507548, "time_profile/reward": 1.3458904176950455, "train/gen_step": 9604.0, "unified/clip_ratio/high_mean": 0.0004462642273210804, "unified/clip_ratio/low_mean": 0.0003407068516025902, "unified/clip_ratio/region_mean": 0.0007869710789236706, "unified/kl": 0.04785197728779167, "unified/loss": 0.005950479607236048 }, { "epoch": 1.9351481184947958, "grad_norm": 0.003339177230373025, "learning_rate": 1e-05, "loss": 0.0014, "step": 152, "time_profile/curr_logprobs_and_update": 0.28296438709367067, "train/gen_step": 9668.0, "unified/clip_ratio/high_mean": 2.6393581720185466e-05, "unified/clip_ratio/low_mean": 5.193366814637557e-05, "unified/clip_ratio/region_mean": 7.832724986656103e-05, "unified/kl": 0.03657471065525897, "unified/loss": 0.0014128756847640034 }, { "epoch": 1.9479583666933546, "gen/advantages_abs_mean": 0.027933739125728607, "gen/advantages_max": 0.09872519969940186, "gen/advantages_mean": -3.4924596548080444e-09, "gen/advantages_min": -0.21751081943511963, "gen/advantages_std": 0.039107732474803925, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 92.5, "gen/prompt/min_length": 42.0, "gen/reward": 0.780127227306366, "gen/reward_std": 0.0778065100312233, "grad_norm": 0.0027934254612773657, "learning_rate": 1e-05, "loss": 0.0018, "rewards/perceptual_score_reward_func/mean": 0.7786700129508972, "rewards/perceptual_score_reward_func/std": 0.07780160754919052, "step": 153, "time_profile/curr_logprobs_and_update": 0.28734079509740695, "time_profile/image_rollout": 94.81195741146803, "time_profile/old_logps": 35.69596145488322, "time_profile/ref_logps": 35.74558510072529, "time_profile/reward": 1.2448325883597136, "train/gen_step": 9732.0, "unified/clip_ratio/high_mean": 0.00020744220455526374, "unified/clip_ratio/low_mean": 0.0002956635626105708, "unified/clip_ratio/region_mean": 0.0005031057635278557, "unified/kl": 0.040840220666723326, "unified/loss": 0.000912024426725111 }, { "epoch": 1.9607686148919137, "grad_norm": 0.0028558243066072464, "learning_rate": 1e-05, "loss": 0.0026, "step": 154, "time_profile/curr_logprobs_and_update": 0.29257669125217944, "train/gen_step": 9796.0, "unified/clip_ratio/high_mean": 0.0008039099420784623, "unified/clip_ratio/low_mean": 0.0016663962333041127, "unified/clip_ratio/region_mean": 0.002470306179020554, "unified/kl": 0.0655196302686818, "unified/loss": 0.0026364892204355783 }, { "epoch": 1.9735788630904723, "gen/advantages_abs_mean": 0.06897905468940735, "gen/advantages_max": 0.25565266609191895, "gen/advantages_mean": -3.6670826375484467e-09, "gen/advantages_min": -0.2908056974411011, "gen/advantages_std": 0.08907780051231384, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 84.0, "gen/prompt/min_length": 25.0, "gen/reward": 0.5265546441078186, "gen/reward_std": 0.23570306599140167, "grad_norm": 0.002455601003021002, "learning_rate": 1e-05, "loss": 0.0017, "rewards/perceptual_score_reward_func/mean": 0.5150833129882812, "rewards/perceptual_score_reward_func/std": 0.25435975193977356, "step": 155, "time_profile/curr_logprobs_and_update": 0.28119892042013817, "time_profile/image_rollout": 83.67556969821453, "time_profile/old_logps": 35.684941904619336, "time_profile/ref_logps": 35.68470383249223, "time_profile/reward": 1.1146302074193954, "train/gen_step": 9860.0, "unified/clip_ratio/high_mean": 0.0033073803624574794, "unified/clip_ratio/low_mean": 0.0065782893370851525, "unified/clip_ratio/region_mean": 0.009885669787763618, "unified/kl": 0.06662523333216086, "unified/loss": -0.000262459849636798 }, { "epoch": 1.9863891112890313, "grad_norm": 0.0022808618377894163, "learning_rate": 1e-05, "loss": 0.0032, "step": 156, "time_profile/curr_logprobs_and_update": 0.2870417326630559, "train/gen_step": 9924.0, "unified/clip_ratio/high_mean": 0.00015197796437860234, "unified/clip_ratio/low_mean": 0.00020154173307673773, "unified/clip_ratio/region_mean": 0.00035351969836483477, "unified/kl": 0.07948597773793153, "unified/loss": 0.00320796342975882 }, { "epoch": 1.99919935948759, "gen/advantages_abs_mean": 0.04813893511891365, "gen/advantages_max": 0.19551599025726318, "gen/advantages_mean": 4.656612873077393e-10, "gen/advantages_min": -0.24950778484344482, "gen/advantages_std": 0.06611930578947067, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.6326351761817932, "gen/reward_std": 0.12095728516578674, "grad_norm": 0.0022853959817439318, "learning_rate": 1e-05, "loss": 0.0031, "rewards/perceptual_score_reward_func/mean": 0.6287431716918945, "rewards/perceptual_score_reward_func/std": 0.1246950700879097, "step": 157, "time_profile/curr_logprobs_and_update": 0.2884384596545715, "time_profile/image_rollout": 99.05054134689271, "time_profile/old_logps": 35.70570847764611, "time_profile/ref_logps": 35.81650428660214, "time_profile/reward": 1.1606343444436789, "train/gen_step": 9988.0, "unified/clip_ratio/high_mean": 0.0007348101480602054, "unified/clip_ratio/low_mean": 0.0016179103122340166, "unified/clip_ratio/region_mean": 0.002352720430280897, "unified/kl": 0.07506306233699434, "unified/loss": 0.0051671818937393255 }, { "epoch": 2.0, "grad_norm": 0.0020002492237836123, "learning_rate": 1e-05, "loss": 0.0003, "step": 158, "time_profile/curr_logprobs_and_update": 0.2812675591558218, "train/gen_step": 9992.0, "unified/clip_ratio/high_mean": 0.001988460251595825, "unified/clip_ratio/low_mean": 0.003396499145310372, "unified/clip_ratio/region_mean": 0.005384959629736841, "unified/kl": 0.07596240937709808, "unified/loss": -0.0008202246390283108 }, { "epoch": 2.012810248198559, "grad_norm": 0.0026500935200601816, "learning_rate": 1e-05, "loss": 0.0027, "step": 159, "time_profile/curr_logprobs_and_update": 0.28652132936986163, "train/gen_step": 10056.0, "unified/clip_ratio/high_mean": 0.005975998764370161, "unified/clip_ratio/low_mean": 0.008422538764534693, "unified/clip_ratio/region_mean": 0.014398537565284641, "unified/kl": 0.06567471823655069, "unified/loss": 0.002683598070007065 }, { "epoch": 2.0256204963971176, "gen/advantages_abs_mean": 0.056865792721509933, "gen/advantages_max": 0.25363510847091675, "gen/advantages_mean": -4.656612873077393e-10, "gen/advantages_min": -0.25625455379486084, "gen/advantages_std": 0.07274879515171051, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 84.5, "gen/prompt/min_length": 26.0, "gen/reward": 0.41545069217681885, "gen/reward_std": 0.32199838757514954, "grad_norm": 0.00292989076115191, "learning_rate": 1e-05, "loss": 0.0036, "rewards/perceptual_score_reward_func/mean": 0.4275937080383301, "rewards/perceptual_score_reward_func/std": 0.3249584436416626, "step": 160, "time_profile/curr_logprobs_and_update": 0.29021278861910105, "time_profile/image_rollout": 90.17475041560829, "time_profile/old_logps": 35.65966880507767, "time_profile/ref_logps": 35.711894784122705, "time_profile/reward": 1.1305186413228512, "train/gen_step": 10120.0, "unified/clip_ratio/high_mean": 0.008071905474025698, "unified/clip_ratio/low_mean": 0.010683667308512668, "unified/clip_ratio/region_mean": 0.01875557277980988, "unified/kl": 0.08813725365325809, "unified/loss": 0.0036637967293700058 }, { "epoch": 2.0384307445956766, "grad_norm": 0.005531059578061104, "learning_rate": 1e-05, "loss": 0.0061, "step": 161, "time_profile/curr_logprobs_and_update": 0.29515198740409687, "train/gen_step": 10184.0, "unified/clip_ratio/high_mean": 0.00033335702755721286, "unified/clip_ratio/low_mean": 0.00017928345914697275, "unified/clip_ratio/region_mean": 0.0005126404903421644, "unified/kl": 0.1533038376364857, "unified/loss": 0.005756833081250079 }, { "epoch": 2.051240992794235, "gen/advantages_abs_mean": 0.03855127468705177, "gen/advantages_max": 0.1778925061225891, "gen/advantages_mean": -3.026798367500305e-09, "gen/advantages_min": -0.22928708791732788, "gen/advantages_std": 0.05448167398571968, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 86.5, "gen/prompt/min_length": 36.0, "gen/reward": 0.7568866014480591, "gen/reward_std": 0.10604741424322128, "grad_norm": 0.0052727325819432735, "learning_rate": 1e-05, "loss": 0.0048, "rewards/perceptual_score_reward_func/mean": 0.7629588842391968, "rewards/perceptual_score_reward_func/std": 0.11004883050918579, "step": 162, "time_profile/curr_logprobs_and_update": 0.28484814503462985, "time_profile/image_rollout": 88.95044229365885, "time_profile/old_logps": 35.69716415554285, "time_profile/ref_logps": 35.729007912799716, "time_profile/reward": 1.3262334875762463, "train/gen_step": 10248.0, "unified/clip_ratio/high_mean": 0.004101160412574245, "unified/clip_ratio/low_mean": 0.003542088237736607, "unified/clip_ratio/region_mean": 0.007643248653948831, "unified/kl": 0.11847320757806301, "unified/loss": 0.004571665815092274 }, { "epoch": 2.0640512409927942, "grad_norm": 0.003853349946439266, "learning_rate": 1e-05, "loss": 0.0028, "step": 163, "time_profile/curr_logprobs_and_update": 0.29358074415358715, "train/gen_step": 10312.0, "unified/clip_ratio/high_mean": 0.0007447073858202202, "unified/clip_ratio/low_mean": 2.7229518309468403e-05, "unified/clip_ratio/region_mean": 0.0007719369050391833, "unified/kl": 0.0694197312113829, "unified/loss": 0.0026930213625746546 }, { "epoch": 2.0768614891913533, "gen/advantages_abs_mean": 0.05062304437160492, "gen/advantages_max": 0.23533660173416138, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.3021351099014282, "gen/advantages_std": 0.06885077804327011, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 83.5, "gen/prompt/min_length": 24.0, "gen/reward": 0.5297470092773438, "gen/reward_std": 0.20814448595046997, "grad_norm": 0.00271556805819273, "learning_rate": 1e-05, "loss": 0.0028, "rewards/perceptual_score_reward_func/mean": 0.5205060243606567, "rewards/perceptual_score_reward_func/std": 0.22350168228149414, "step": 164, "time_profile/curr_logprobs_and_update": 0.29248257720610127, "time_profile/image_rollout": 88.6062811203301, "time_profile/old_logps": 35.64170744456351, "time_profile/ref_logps": 35.71029465831816, "time_profile/reward": 1.0758190751075745, "train/gen_step": 10376.0, "unified/clip_ratio/high_mean": 0.011243771637964528, "unified/clip_ratio/low_mean": 0.00031519579079031246, "unified/clip_ratio/region_mean": 0.011558967485143512, "unified/kl": 0.06918471836252138, "unified/loss": 0.002829152402227919 }, { "epoch": 2.089671737389912, "grad_norm": 0.004280099645256996, "learning_rate": 1e-05, "loss": 0.0019, "step": 165, "time_profile/curr_logprobs_and_update": 0.28327618300681934, "train/gen_step": 10440.0, "unified/clip_ratio/high_mean": 0.004382391351100523, "unified/clip_ratio/low_mean": 0.00019320666888233973, "unified/clip_ratio/region_mean": 0.0045755980208923575, "unified/kl": 0.04727714671753347, "unified/loss": 0.0019169329607393593 }, { "epoch": 2.102481985588471, "gen/advantages_abs_mean": 0.035958182066679, "gen/advantages_max": 0.17724670469760895, "gen/advantages_mean": 1.6298145055770874e-09, "gen/advantages_min": -0.12050622701644897, "gen/advantages_std": 0.04613741859793663, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 31.0, "gen/prompt/mean_length": 31.0, "gen/prompt/min_length": 31.0, "gen/reward": 0.44015035033226013, "gen/reward_std": 0.3897111415863037, "grad_norm": 0.0035524514969438314, "learning_rate": 1e-05, "loss": 0.0018, "rewards/perceptual_score_reward_func/mean": 0.4246736764907837, "rewards/perceptual_score_reward_func/std": 0.3995981812477112, "step": 166, "time_profile/curr_logprobs_and_update": 0.29225988683174364, "time_profile/image_rollout": 89.83664705976844, "time_profile/old_logps": 35.64525066129863, "time_profile/ref_logps": 35.67439313046634, "time_profile/reward": 1.3415015526115894, "train/gen_step": 10504.0, "unified/clip_ratio/high_mean": 0.010256347398353682, "unified/clip_ratio/low_mean": 0.0009354562380394782, "unified/clip_ratio/region_mean": 0.01119180366731598, "unified/kl": 0.043152328697033226, "unified/loss": 0.0016656069524287886 }, { "epoch": 2.1152922337870295, "grad_norm": 0.003104103496298194, "learning_rate": 1e-05, "loss": 0.0017, "step": 167, "time_profile/curr_logprobs_and_update": 0.28839108292595483, "train/gen_step": 10568.0, "unified/clip_ratio/high_mean": 0.0014482807009699172, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0014482807009699172, "unified/kl": 0.042221105250064284, "unified/loss": 0.0016389742395404028 }, { "epoch": 2.1281024819855885, "gen/advantages_abs_mean": 0.04718589410185814, "gen/advantages_max": 0.25266826152801514, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.3306066393852234, "gen/advantages_std": 0.07132115215063095, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.09375, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 130.0, "gen/prompt/min_length": 32.0, "gen/reward": 0.20786593854427338, "gen/reward_std": 0.20858700573444366, "grad_norm": 0.0031762246508151293, "learning_rate": 1e-05, "loss": 0.0016, "rewards/perceptual_score_reward_func/mean": 0.21164503693580627, "rewards/perceptual_score_reward_func/std": 0.22171460092067719, "step": 168, "time_profile/curr_logprobs_and_update": 0.28383773076348007, "time_profile/image_rollout": 86.03059590421617, "time_profile/old_logps": 35.66852024383843, "time_profile/ref_logps": 35.73095652461052, "time_profile/reward": 1.5308976825326681, "train/gen_step": 10632.0, "unified/clip_ratio/high_mean": 0.007134736979423906, "unified/clip_ratio/low_mean": 0.0001799682322598528, "unified/clip_ratio/region_mean": 0.007314705258977483, "unified/kl": 0.03931887086946517, "unified/loss": 0.0015712360254838131 }, { "epoch": 2.1409127301841475, "grad_norm": 0.0041104997508227825, "learning_rate": 1e-05, "loss": 0.0016, "step": 169, "time_profile/curr_logprobs_and_update": 0.29093144927173853, "train/gen_step": 10696.0, "unified/clip_ratio/high_mean": 2.531525296944892e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 2.531525296944892e-05, "unified/kl": 0.041141612542560324, "unified/loss": 0.0015832010067242663 }, { "epoch": 2.153722978382706, "gen/advantages_abs_mean": 0.07721328735351562, "gen/advantages_max": 0.3477572798728943, "gen/advantages_mean": 2.7939677238464355e-09, "gen/advantages_min": -0.2912459969520569, "gen/advantages_std": 0.10059475153684616, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 137.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.4667072296142578, "gen/reward_std": 0.25318169593811035, "grad_norm": 0.004474426619708538, "learning_rate": 1e-05, "loss": 0.0018, "rewards/perceptual_score_reward_func/mean": 0.4682084918022156, "rewards/perceptual_score_reward_func/std": 0.24427475035190582, "step": 170, "time_profile/curr_logprobs_and_update": 0.28858580024098046, "time_profile/image_rollout": 89.55203330516815, "time_profile/old_logps": 35.70584703609347, "time_profile/ref_logps": 35.79972547106445, "time_profile/reward": 1.1563408393412828, "train/gen_step": 10760.0, "unified/clip_ratio/high_mean": 0.0006694951052850229, "unified/clip_ratio/low_mean": 6.273121380218072e-05, "unified/clip_ratio/region_mean": 0.0007322263181777089, "unified/kl": 0.046405625878833234, "unified/loss": 0.0018544251106504817 }, { "epoch": 2.166533226581265, "grad_norm": 0.0052252947352826595, "learning_rate": 1e-05, "loss": 0.0025, "step": 171, "time_profile/curr_logprobs_and_update": 0.28510314726736397, "train/gen_step": 10824.0, "unified/clip_ratio/high_mean": 3.648532583611086e-05, "unified/clip_ratio/low_mean": 1.2934602636960335e-05, "unified/clip_ratio/region_mean": 4.94199293825659e-05, "unified/kl": 0.06382056430447847, "unified/loss": 0.0025927637943823356 }, { "epoch": 2.1793434747798237, "gen/advantages_abs_mean": 0.0301031656563282, "gen/advantages_max": 0.11702626943588257, "gen/advantages_mean": -4.889443516731262e-09, "gen/advantages_min": -0.1823127269744873, "gen/advantages_std": 0.04142240807414055, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 88.5, "gen/prompt/min_length": 34.0, "gen/reward": 0.7539215087890625, "gen/reward_std": 0.08662469685077667, "grad_norm": 0.0047385720536112785, "learning_rate": 1e-05, "loss": 0.0023, "rewards/perceptual_score_reward_func/mean": 0.7586275339126587, "rewards/perceptual_score_reward_func/std": 0.08558635413646698, "step": 172, "time_profile/curr_logprobs_and_update": 0.29184379530488513, "time_profile/image_rollout": 92.46348539367318, "time_profile/old_logps": 35.65855384618044, "time_profile/ref_logps": 35.705090867355466, "time_profile/reward": 1.2204638700932264, "train/gen_step": 10888.0, "unified/clip_ratio/high_mean": 0.0006851561292933184, "unified/clip_ratio/low_mean": 0.000610612833952473, "unified/clip_ratio/region_mean": 0.0012957689750692225, "unified/kl": 0.058761145861353725, "unified/loss": 0.0022805339922342682 }, { "epoch": 2.1921537229783827, "grad_norm": 0.0027263937518000603, "learning_rate": 1e-05, "loss": 0.003, "step": 173, "time_profile/curr_logprobs_and_update": 0.2959218217874877, "train/gen_step": 10952.0, "unified/clip_ratio/high_mean": 0.0005940043447481003, "unified/clip_ratio/low_mean": 1.3377568393480033e-05, "unified/clip_ratio/region_mean": 0.0006073819131415803, "unified/kl": 0.07492666487814859, "unified/loss": 0.0030623077946074773 }, { "epoch": 2.2049639711769418, "gen/advantages_abs_mean": 0.01619063690304756, "gen/advantages_max": 0.048489928245544434, "gen/advantages_mean": -1.862645149230957e-09, "gen/advantages_min": -0.06617963314056396, "gen/advantages_std": 0.020318645983934402, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 39.0, "gen/prompt/mean_length": 36.5, "gen/prompt/min_length": 34.0, "gen/reward": 0.8497151732444763, "gen/reward_std": 0.023617252707481384, "grad_norm": 0.002491321414709091, "learning_rate": 1e-05, "loss": 0.0029, "rewards/perceptual_score_reward_func/mean": 0.8468820452690125, "rewards/perceptual_score_reward_func/std": 0.025489442050457, "step": 174, "time_profile/curr_logprobs_and_update": 0.2864244414377026, "time_profile/image_rollout": 95.55869417823851, "time_profile/old_logps": 35.637094331905246, "time_profile/ref_logps": 35.696372866630554, "time_profile/reward": 1.4515288285911083, "train/gen_step": 11016.0, "unified/clip_ratio/high_mean": 0.0013642299363709753, "unified/clip_ratio/low_mean": 0.00038164848774613347, "unified/clip_ratio/region_mean": 0.0017458784241171088, "unified/kl": 0.07242366363061592, "unified/loss": 0.002966343375192082 }, { "epoch": 2.2177742193755003, "grad_norm": 0.0011181713780388236, "learning_rate": 1e-05, "loss": 0.0015, "step": 175, "time_profile/curr_logprobs_and_update": 0.2914645765267778, "train/gen_step": 11080.0, "unified/clip_ratio/high_mean": 0.00037641884682670934, "unified/clip_ratio/low_mean": 3.8014602978364564e-05, "unified/clip_ratio/region_mean": 0.0004144334498050739, "unified/kl": 0.03686668295995332, "unified/loss": 0.0014178659844787944 }, { "epoch": 2.2305844675740594, "gen/advantages_abs_mean": 0.06131240725517273, "gen/advantages_max": 0.21112197637557983, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.2674327492713928, "gen/advantages_std": 0.07824128866195679, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 215.5, "gen/prompt/min_length": 137.0, "gen/reward": 0.5548129081726074, "gen/reward_std": 0.13008329272270203, "grad_norm": 0.0041183605790138245, "learning_rate": 1e-05, "loss": 0.0015, "rewards/perceptual_score_reward_func/mean": 0.5545451641082764, "rewards/perceptual_score_reward_func/std": 0.11709090322256088, "step": 176, "time_profile/curr_logprobs_and_update": 0.29137875014566816, "time_profile/image_rollout": 91.59421370364726, "time_profile/old_logps": 35.71455822326243, "time_profile/ref_logps": 35.81172662228346, "time_profile/reward": 1.4883500207215548, "train/gen_step": 11144.0, "unified/clip_ratio/high_mean": 0.002236859237200406, "unified/clip_ratio/low_mean": 0.0003157618521072436, "unified/clip_ratio/region_mean": 0.0025526210829411866, "unified/kl": 0.03860792936757207, "unified/loss": 0.0014944951885809132 }, { "epoch": 2.243394715772618, "grad_norm": 0.006271969992667437, "learning_rate": 1e-05, "loss": 0.0022, "step": 177, "time_profile/curr_logprobs_and_update": 0.2987609440460801, "train/gen_step": 11208.0, "unified/clip_ratio/high_mean": 9.015938485390507e-05, "unified/clip_ratio/low_mean": 9.958242844732013e-05, "unified/clip_ratio/region_mean": 0.0001897418133012252, "unified/kl": 0.056187750946264714, "unified/loss": 0.002266170584334759 }, { "epoch": 2.256204963971177, "gen/advantages_abs_mean": 0.05737482011318207, "gen/advantages_max": 0.26223427057266235, "gen/advantages_mean": -1.1641532182693481e-09, "gen/advantages_min": -0.2832987308502197, "gen/advantages_std": 0.07611891627311707, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.6172780990600586, "gen/reward_std": 0.1598697006702423, "grad_norm": 0.006493598222732544, "learning_rate": 1e-05, "loss": 0.0022, "rewards/perceptual_score_reward_func/mean": 0.6214098930358887, "rewards/perceptual_score_reward_func/std": 0.1510515958070755, "step": 178, "time_profile/curr_logprobs_and_update": 0.288628595037153, "time_profile/image_rollout": 93.41255523450673, "time_profile/old_logps": 35.712708877399564, "time_profile/ref_logps": 35.819618970155716, "time_profile/reward": 1.2369329910725355, "train/gen_step": 11272.0, "unified/clip_ratio/high_mean": 0.0009131154902206617, "unified/clip_ratio/low_mean": 0.0013802611756545957, "unified/clip_ratio/region_mean": 0.002293376676789194, "unified/kl": 0.05572476948145777, "unified/loss": 0.002130313092493452 }, { "epoch": 2.2690152121697356, "grad_norm": 0.00450574466958642, "learning_rate": 1e-05, "loss": 0.003, "step": 179, "time_profile/curr_logprobs_and_update": 0.2841369793168269, "train/gen_step": 11336.0, "unified/clip_ratio/high_mean": 0.0005733701354984078, "unified/clip_ratio/low_mean": 0.0003713260139193153, "unified/clip_ratio/region_mean": 0.0009446961430512602, "unified/kl": 0.07562511297874153, "unified/loss": 0.002866666169211385 }, { "epoch": 2.2818254603682946, "gen/advantages_abs_mean": 0.03693922609090805, "gen/advantages_max": 0.1345970630645752, "gen/advantages_mean": 9.313225746154785e-10, "gen/advantages_min": -0.3643248677253723, "gen/advantages_std": 0.05376308783888817, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 89.0, "gen/prompt/min_length": 35.0, "gen/reward": 0.7307989597320557, "gen/reward_std": 0.12450557202100754, "grad_norm": 0.0036411313340067863, "learning_rate": 1e-05, "loss": 0.0034, "rewards/perceptual_score_reward_func/mean": 0.7366804480552673, "rewards/perceptual_score_reward_func/std": 0.12658695876598358, "step": 180, "time_profile/curr_logprobs_and_update": 0.29716587750590406, "time_profile/image_rollout": 88.12802308052778, "time_profile/old_logps": 35.651363495737314, "time_profile/ref_logps": 35.73251064121723, "time_profile/reward": 1.130325747653842, "train/gen_step": 11400.0, "unified/clip_ratio/high_mean": 0.006377894053912314, "unified/clip_ratio/low_mean": 0.0008435255413132836, "unified/clip_ratio/region_mean": 0.007221419669804163, "unified/kl": 0.08534415764734149, "unified/loss": 0.003086884494223341 }, { "epoch": 2.2946357085668536, "grad_norm": 0.0029174236115068197, "learning_rate": 1e-05, "loss": 0.0039, "step": 181, "time_profile/curr_logprobs_and_update": 0.29214068927103654, "train/gen_step": 11464.0, "unified/clip_ratio/high_mean": 0.0026058121047753957, "unified/clip_ratio/low_mean": 0.00016552963006688515, "unified/clip_ratio/region_mean": 0.002771341710285924, "unified/kl": 0.09713484643725678, "unified/loss": 0.003910853625711752 }, { "epoch": 2.307445956765412, "gen/advantages_abs_mean": 0.08743979036808014, "gen/advantages_max": 0.3487345278263092, "gen/advantages_mean": 6.693881005048752e-10, "gen/advantages_min": -0.3564333915710449, "gen/advantages_std": 0.11450637131929398, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 85.0, "gen/prompt/min_length": 27.0, "gen/reward": 0.5280884504318237, "gen/reward_std": 0.21100233495235443, "grad_norm": 0.0019279182888567448, "learning_rate": 1e-05, "loss": 0.0036, "rewards/perceptual_score_reward_func/mean": 0.4943096935749054, "rewards/perceptual_score_reward_func/std": 0.23395422101020813, "step": 182, "time_profile/curr_logprobs_and_update": 0.2942618557426613, "time_profile/image_rollout": 91.16302826814353, "time_profile/old_logps": 35.6648780554533, "time_profile/ref_logps": 35.72973041422665, "time_profile/reward": 1.0635294429957867, "train/gen_step": 11528.0, "unified/clip_ratio/high_mean": 0.01493707289773738, "unified/clip_ratio/low_mean": 0.0015484474943150417, "unified/clip_ratio/region_mean": 0.01648552050573926, "unified/kl": 0.08991632401011884, "unified/loss": 0.003767051275644917 }, { "epoch": 2.3202562049639712, "grad_norm": 0.004897973965853453, "learning_rate": 1e-05, "loss": 0.0025, "step": 183, "time_profile/curr_logprobs_and_update": 0.2830031361372676, "train/gen_step": 11592.0, "unified/clip_ratio/high_mean": 0.013359589624087675, "unified/clip_ratio/low_mean": 0.0002670826233952539, "unified/clip_ratio/region_mean": 0.01362667221292213, "unified/kl": 0.061459658725652844, "unified/loss": 0.002509741234462126 }, { "epoch": 2.3330664531625303, "gen/advantages_abs_mean": 0.04273279756307602, "gen/advantages_max": 0.18777477741241455, "gen/advantages_mean": 6.984919309616089e-10, "gen/advantages_min": -0.27026069164276123, "gen/advantages_std": 0.06035928428173065, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 90.0, "gen/prompt/min_length": 37.0, "gen/reward": 0.716698408126831, "gen/reward_std": 0.09943482279777527, "grad_norm": 0.005198562052100897, "learning_rate": 1e-05, "loss": 0.0028, "rewards/perceptual_score_reward_func/mean": 0.719241201877594, "rewards/perceptual_score_reward_func/std": 0.09961231797933578, "step": 184, "time_profile/curr_logprobs_and_update": 0.28848187151015736, "time_profile/image_rollout": 97.41653303615749, "time_profile/old_logps": 35.6586435046047, "time_profile/ref_logps": 35.720301274210215, "time_profile/reward": 1.2692248299717903, "train/gen_step": 11656.0, "unified/clip_ratio/high_mean": 0.027763162452174583, "unified/clip_ratio/low_mean": 0.0007210646463136072, "unified/clip_ratio/region_mean": 0.028484226852015126, "unified/kl": 0.06450571143068373, "unified/loss": 0.0026937655838992214 }, { "epoch": 2.345876701361089, "grad_norm": 0.0029400514904409647, "learning_rate": 1e-05, "loss": 0.0027, "step": 185, "time_profile/curr_logprobs_and_update": 0.29348416050197557, "train/gen_step": 11720.0, "unified/clip_ratio/high_mean": 0.001847269874815538, "unified/clip_ratio/low_mean": 8.904416790755931e-05, "unified/clip_ratio/region_mean": 0.0019363140418136027, "unified/kl": 0.06811343174194917, "unified/loss": 0.00265883334084549 }, { "epoch": 2.358686949559648, "gen/advantages_abs_mean": 0.060479603707790375, "gen/advantages_max": 0.2555360794067383, "gen/advantages_mean": -2.3283064365386963e-10, "gen/advantages_min": -0.20094865560531616, "gen/advantages_std": 0.07668271660804749, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 80.0, "gen/prompt/min_length": 23.0, "gen/reward": 0.2623538672924042, "gen/reward_std": 0.11184389144182205, "grad_norm": 0.0025075250305235386, "learning_rate": 1e-05, "loss": 0.0024, "rewards/perceptual_score_reward_func/mean": 0.2549823820590973, "rewards/perceptual_score_reward_func/std": 0.11117707192897797, "step": 186, "time_profile/curr_logprobs_and_update": 0.2875999791431241, "time_profile/image_rollout": 95.74755357392132, "time_profile/old_logps": 35.969283543527126, "time_profile/ref_logps": 35.6872977539897, "time_profile/reward": 1.2131715137511492, "train/gen_step": 11784.0, "unified/clip_ratio/high_mean": 0.0022697771437378833, "unified/clip_ratio/low_mean": 0.0003853140624414664, "unified/clip_ratio/region_mean": 0.002655091193446424, "unified/kl": 0.059487957099918276, "unified/loss": 0.002435175606251505 }, { "epoch": 2.3714971977582064, "grad_norm": 0.003168602241203189, "learning_rate": 1e-05, "loss": 0.0016, "step": 187, "time_profile/curr_logprobs_and_update": 0.28446206089574844, "train/gen_step": 11848.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.039400996727636084, "unified/loss": 0.0016360291774617508 }, { "epoch": 2.3843074459567655, "gen/advantages_abs_mean": 0.05380699038505554, "gen/advantages_max": 0.18637949228286743, "gen/advantages_mean": -3.958120942115784e-09, "gen/advantages_min": -0.247217059135437, "gen/advantages_std": 0.06866227835416794, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 90.5, "gen/prompt/min_length": 38.0, "gen/reward": 0.6251561045646667, "gen/reward_std": 0.0872267559170723, "grad_norm": 0.0032522021792829037, "learning_rate": 1e-05, "loss": 0.0019, "rewards/perceptual_score_reward_func/mean": 0.6136674880981445, "rewards/perceptual_score_reward_func/std": 0.08085624128580093, "step": 188, "time_profile/curr_logprobs_and_update": 0.29212979626026936, "time_profile/image_rollout": 90.07975675538182, "time_profile/old_logps": 35.646439569070935, "time_profile/ref_logps": 35.706678591668606, "time_profile/reward": 1.2349300608038902, "train/gen_step": 11912.0, "unified/clip_ratio/high_mean": 0.00013954428322904278, "unified/clip_ratio/low_mean": 1.3196790860092733e-05, "unified/clip_ratio/region_mean": 0.0001527410740891355, "unified/kl": 0.048110857111169025, "unified/loss": 0.0018662362217582995 }, { "epoch": 2.397117694155324, "grad_norm": 0.002924871863797307, "learning_rate": 1e-05, "loss": 0.0028, "step": 189, "time_profile/curr_logprobs_and_update": 0.29138450731988996, "train/gen_step": 11976.0, "unified/clip_ratio/high_mean": 0.00043166354043933097, "unified/clip_ratio/low_mean": 0.00023087494355422677, "unified/clip_ratio/region_mean": 0.000662538483084063, "unified/kl": 0.0693313748924993, "unified/loss": 0.0026826070568404248 }, { "epoch": 2.409927942353883, "gen/advantages_abs_mean": 0.036966800689697266, "gen/advantages_max": 0.19963926076889038, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.17272275686264038, "gen/advantages_std": 0.0572461299598217, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.125, "gen/prompt/max_length": 30.0, "gen/prompt/mean_length": 27.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.11605678498744965, "gen/reward_std": 0.12552930414676666, "grad_norm": 0.003882054705172777, "learning_rate": 1e-05, "loss": 0.0026, "rewards/perceptual_score_reward_func/mean": 0.10083503276109695, "rewards/perceptual_score_reward_func/std": 0.11439882963895798, "step": 190, "time_profile/curr_logprobs_and_update": 0.2918980286340229, "time_profile/image_rollout": 91.15622867643833, "time_profile/old_logps": 35.61673030629754, "time_profile/ref_logps": 35.67608836479485, "time_profile/reward": 1.4936586115509272, "train/gen_step": 12040.0, "unified/clip_ratio/high_mean": 0.0009299532603108673, "unified/clip_ratio/low_mean": 0.0018241363704873947, "unified/clip_ratio/region_mean": 0.0027540896044229157, "unified/kl": 0.06421229938860051, "unified/loss": 0.0030946720908104908 }, { "epoch": 2.422738190552442, "grad_norm": 0.003422857029363513, "learning_rate": 1e-05, "loss": 0.0016, "step": 191, "time_profile/curr_logprobs_and_update": 0.2808142671419773, "train/gen_step": 12104.0, "unified/clip_ratio/high_mean": 1.4575559362128843e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 1.4575559362128843e-05, "unified/kl": 0.04138320931815542, "unified/loss": 0.0015813396021258086 }, { "epoch": 2.4355484387510007, "gen/advantages_abs_mean": 0.045888688415288925, "gen/advantages_max": 0.15211229026317596, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.3356465697288513, "gen/advantages_std": 0.060004960745573044, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 84.0, "gen/prompt/min_length": 25.0, "gen/reward": 0.39801621437072754, "gen/reward_std": 0.1737685650587082, "grad_norm": 0.002050932962447405, "learning_rate": 1e-05, "loss": 0.0017, "rewards/perceptual_score_reward_func/mean": 0.40392911434173584, "rewards/perceptual_score_reward_func/std": 0.17653802037239075, "step": 192, "time_profile/curr_logprobs_and_update": 0.2883164933009539, "time_profile/image_rollout": 92.29078873991966, "time_profile/old_logps": 35.65081191062927, "time_profile/ref_logps": 35.689905505627394, "time_profile/reward": 1.0758352540433407, "train/gen_step": 12168.0, "unified/clip_ratio/high_mean": 8.769621945248218e-05, "unified/clip_ratio/low_mean": 9.131907700066222e-05, "unified/clip_ratio/region_mean": 0.0001790152964531444, "unified/kl": 0.04167587656411342, "unified/loss": 0.0017008707181958016 }, { "epoch": 2.4483586869495597, "grad_norm": 0.00368260545656085, "learning_rate": 1e-05, "loss": 0.0041, "step": 193, "time_profile/curr_logprobs_and_update": 0.2881558191438671, "train/gen_step": 12232.0, "unified/clip_ratio/high_mean": 0.00012643504942388972, "unified/clip_ratio/low_mean": 2.5052377168321982e-05, "unified/clip_ratio/region_mean": 0.0001514874265922117, "unified/kl": 0.1030606998829171, "unified/loss": 0.004149410695390543 }, { "epoch": 2.4611689351481187, "gen/advantages_abs_mean": 0.04738696664571762, "gen/advantages_max": 0.15458166599273682, "gen/advantages_mean": -3.725290298461914e-09, "gen/advantages_min": -0.3881222605705261, "gen/advantages_std": 0.06603256613016129, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 88.0, "gen/prompt/min_length": 39.0, "gen/reward": 0.7343631982803345, "gen/reward_std": 0.10265038162469864, "grad_norm": 0.003888089442625642, "learning_rate": 1e-05, "loss": 0.0045, "rewards/perceptual_score_reward_func/mean": 0.7463352680206299, "rewards/perceptual_score_reward_func/std": 0.09803234785795212, "step": 194, "time_profile/curr_logprobs_and_update": 0.287883290235186, "time_profile/image_rollout": 90.74564089998603, "time_profile/old_logps": 35.664300525560975, "time_profile/ref_logps": 35.708490991964936, "time_profile/reward": 1.5299178566783667, "train/gen_step": 12296.0, "unified/clip_ratio/high_mean": 0.0008944852061176789, "unified/clip_ratio/low_mean": 0.0011902720243597287, "unified/clip_ratio/region_mean": 0.002084757244119828, "unified/kl": 0.11233595450175926, "unified/loss": 0.004423787195264595 }, { "epoch": 2.4739791833466773, "grad_norm": 0.0031231113243848085, "learning_rate": 1e-05, "loss": 0.004, "step": 195, "time_profile/curr_logprobs_and_update": 0.29107131843920797, "train/gen_step": 12360.0, "unified/clip_ratio/high_mean": 0.00021144409492990235, "unified/clip_ratio/low_mean": 7.923106568341609e-05, "unified/clip_ratio/region_mean": 0.00029067516061331844, "unified/kl": 0.098656052199658, "unified/loss": 0.004180431839358789 }, { "epoch": 2.4867894315452364, "gen/advantages_abs_mean": 0.0698113888502121, "gen/advantages_max": 0.27165061235427856, "gen/advantages_mean": -3.958120942115784e-09, "gen/advantages_min": -0.3109276294708252, "gen/advantages_std": 0.0891900584101677, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 140.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.6346923112869263, "gen/reward_std": 0.11421144008636475, "grad_norm": 0.004553945269435644, "learning_rate": 1e-05, "loss": 0.0033, "rewards/perceptual_score_reward_func/mean": 0.6285253763198853, "rewards/perceptual_score_reward_func/std": 0.11653388291597366, "step": 196, "time_profile/curr_logprobs_and_update": 0.2902447068481706, "time_profile/image_rollout": 96.71165009960532, "time_profile/old_logps": 35.683688862249255, "time_profile/ref_logps": 35.757541788741946, "time_profile/reward": 1.2395988013595343, "train/gen_step": 12424.0, "unified/clip_ratio/high_mean": 0.003897015763868694, "unified/clip_ratio/low_mean": 0.004954910057676898, "unified/clip_ratio/region_mean": 0.008851925810631656, "unified/kl": 0.08233964705141261, "unified/loss": 0.003243297023004743 }, { "epoch": 2.499599679743795, "grad_norm": 0.004913492128252983, "learning_rate": 1e-05, "loss": 0.0041, "step": 197, "time_profile/curr_logprobs_and_update": 0.2921755504794419, "train/gen_step": 12488.0, "unified/clip_ratio/high_mean": 0.0013023042602071655, "unified/clip_ratio/low_mean": 0.0032974790565276635, "unified/clip_ratio/region_mean": 0.0045997833303772495, "unified/kl": 0.1036716791568324, "unified/loss": 0.004193980672425823 }, { "epoch": 2.512409927942354, "gen/advantages_abs_mean": 0.07193036377429962, "gen/advantages_max": 0.28605392575263977, "gen/advantages_mean": 6.984919309616089e-10, "gen/advantages_min": -0.32319527864456177, "gen/advantages_std": 0.09065697342157364, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 43.0, "gen/prompt/mean_length": 35.5, "gen/prompt/min_length": 28.0, "gen/reward": 0.5142920017242432, "gen/reward_std": 0.1796254962682724, "grad_norm": 0.004819054156541824, "learning_rate": 1e-05, "loss": 0.0038, "rewards/perceptual_score_reward_func/mean": 0.515720009803772, "rewards/perceptual_score_reward_func/std": 0.1884678304195404, "step": 198, "time_profile/curr_logprobs_and_update": 0.2872099138912745, "time_profile/image_rollout": 90.45758785679936, "time_profile/old_logps": 35.63970177434385, "time_profile/ref_logps": 35.68582937121391, "time_profile/reward": 1.3650418017059565, "train/gen_step": 12552.0, "unified/clip_ratio/high_mean": 0.00796576717220887, "unified/clip_ratio/low_mean": 0.03178466257486434, "unified/clip_ratio/region_mean": 0.039750429803461884, "unified/kl": 0.08939005044521764, "unified/loss": 0.004010563134215772 }, { "epoch": 2.5252201761409125, "grad_norm": 0.00535630714148283, "learning_rate": 1e-05, "loss": 0.0017, "step": 199, "time_profile/curr_logprobs_and_update": 0.2841974784096237, "train/gen_step": 12616.0, "unified/clip_ratio/high_mean": 0.0009237372660209076, "unified/clip_ratio/low_mean": 0.00028113492589909583, "unified/clip_ratio/region_mean": 0.0012048721910105087, "unified/kl": 0.0417316788516473, "unified/loss": 0.0017045280983438715 }, { "epoch": 2.5380304243394716, "gen/advantages_abs_mean": 0.037127502262592316, "gen/advantages_max": 0.1399545967578888, "gen/advantages_mean": 1.3969838619232178e-09, "gen/advantages_min": -0.2307451367378235, "gen/advantages_std": 0.050681211054325104, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.015625, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 228.0, "gen/prompt/min_length": 228.0, "gen/reward": 0.2671893835067749, "gen/reward_std": 0.24783937633037567, "grad_norm": 0.004151855129748583, "learning_rate": 1e-05, "loss": 0.0019, "rewards/perceptual_score_reward_func/mean": 0.2690585255622864, "rewards/perceptual_score_reward_func/std": 0.24623475968837738, "step": 200, "time_profile/curr_logprobs_and_update": 0.2871844826149754, "time_profile/image_rollout": 95.29160342365503, "time_profile/old_logps": 35.766085494309664, "time_profile/ref_logps": 35.8024538513273, "time_profile/reward": 1.1446097139269114, "train/gen_step": 12680.0, "unified/clip_ratio/high_mean": 0.0077927707197886775, "unified/clip_ratio/low_mean": 0.0029419622351269936, "unified/clip_ratio/region_mean": 0.010734732993114449, "unified/kl": 0.0463491125847213, "unified/loss": 0.0019507477118168026 }, { "epoch": 2.5508406725380306, "grad_norm": 0.002670120680704713, "learning_rate": 1e-05, "loss": 0.0025, "step": 201, "time_profile/curr_logprobs_and_update": 0.28650486533297226, "train/gen_step": 12744.0, "unified/clip_ratio/high_mean": 0.00013077845142106526, "unified/clip_ratio/low_mean": 8.67168810145813e-05, "unified/clip_ratio/region_mean": 0.00021749533243564656, "unified/kl": 0.06202875357121229, "unified/loss": 0.002423393003482488 }, { "epoch": 2.563650920736589, "gen/advantages_abs_mean": 0.09604242444038391, "gen/advantages_max": 0.28835737705230713, "gen/advantages_mean": 2.6193447411060333e-10, "gen/advantages_min": -0.29365015029907227, "gen/advantages_std": 0.11690658330917358, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 32.0, "gen/prompt/mean_length": 29.0, "gen/prompt/min_length": 26.0, "gen/reward": 0.25929340720176697, "gen/reward_std": 0.12897181510925293, "grad_norm": 0.003636396722868085, "learning_rate": 1e-05, "loss": 0.0025, "rewards/perceptual_score_reward_func/mean": 0.26449474692344666, "rewards/perceptual_score_reward_func/std": 0.1299288123846054, "step": 202, "time_profile/curr_logprobs_and_update": 0.29194199573248625, "time_profile/image_rollout": 94.50923268683255, "time_profile/old_logps": 35.63929962925613, "time_profile/ref_logps": 35.69228541851044, "time_profile/reward": 1.3253274355083704, "train/gen_step": 12808.0, "unified/clip_ratio/high_mean": 0.001574381493810506, "unified/clip_ratio/low_mean": 0.003362546233802277, "unified/clip_ratio/region_mean": 0.004936927789458423, "unified/kl": 0.062456131854560226, "unified/loss": 0.002444851220388955 }, { "epoch": 2.576461168935148, "grad_norm": 0.004620482213795185, "learning_rate": 1e-05, "loss": 0.0021, "step": 203, "time_profile/curr_logprobs_and_update": 0.29166353173786774, "train/gen_step": 12872.0, "unified/clip_ratio/high_mean": 1.220703143189894e-05, "unified/clip_ratio/low_mean": 3.780533188546542e-05, "unified/clip_ratio/region_mean": 5.001236331736436e-05, "unified/kl": 0.05314113886561245, "unified/loss": 0.001977884821826592 }, { "epoch": 2.5892714171337072, "gen/advantages_abs_mean": 0.04947504773736, "gen/advantages_max": 0.18993930518627167, "gen/advantages_mean": 9.604264050722122e-10, "gen/advantages_min": -0.25990980863571167, "gen/advantages_std": 0.06558328121900558, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 84.5, "gen/prompt/min_length": 26.0, "gen/reward": 0.44555872678756714, "gen/reward_std": 0.27785491943359375, "grad_norm": 0.004160444252192974, "learning_rate": 1e-05, "loss": 0.0024, "rewards/perceptual_score_reward_func/mean": 0.453355610370636, "rewards/perceptual_score_reward_func/std": 0.28221702575683594, "step": 204, "time_profile/curr_logprobs_and_update": 0.2906250981905032, "time_profile/image_rollout": 87.65974761359394, "time_profile/old_logps": 35.657574282959104, "time_profile/ref_logps": 35.68245464004576, "time_profile/reward": 1.1088725440204144, "train/gen_step": 12936.0, "unified/clip_ratio/high_mean": 0.00038497244440804934, "unified/clip_ratio/low_mean": 0.0012731670121866046, "unified/clip_ratio/region_mean": 0.0016581394465902122, "unified/kl": 0.06184308050433174, "unified/loss": 0.0024940579969552346 }, { "epoch": 2.602081665332266, "grad_norm": 0.0026204350870102644, "learning_rate": 1e-05, "loss": 0.0031, "step": 205, "time_profile/curr_logprobs_and_update": 0.29234114653081633, "train/gen_step": 13000.0, "unified/clip_ratio/high_mean": 0.0005670754262609989, "unified/clip_ratio/low_mean": 0.0005148732343513984, "unified/clip_ratio/region_mean": 0.001081948661521892, "unified/kl": 0.07684545189840719, "unified/loss": 0.0031584093303536065 }, { "epoch": 2.6148919135308244, "gen/advantages_abs_mean": 0.056859977543354034, "gen/advantages_max": 0.2253839373588562, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.2270420789718628, "gen/advantages_std": 0.07238562405109406, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 46.0, "gen/prompt/mean_length": 35.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.6065893769264221, "gen/reward_std": 0.19950497150421143, "grad_norm": 0.003692910773679614, "learning_rate": 1e-05, "loss": 0.0029, "rewards/perceptual_score_reward_func/mean": 0.6051943302154541, "rewards/perceptual_score_reward_func/std": 0.20580682158470154, "step": 206, "time_profile/curr_logprobs_and_update": 0.29456248341011815, "time_profile/image_rollout": 89.29932282492518, "time_profile/old_logps": 35.62615482322872, "time_profile/ref_logps": 35.67942949384451, "time_profile/reward": 1.7956994157284498, "train/gen_step": 13064.0, "unified/clip_ratio/high_mean": 0.0021543388202189817, "unified/clip_ratio/low_mean": 0.002322014155652141, "unified/clip_ratio/region_mean": 0.0044763529740521335, "unified/kl": 0.07198869198327884, "unified/loss": 0.00294267701337958 }, { "epoch": 2.6277021617293834, "grad_norm": 0.0037548041436821222, "learning_rate": 1e-05, "loss": 0.0019, "step": 207, "time_profile/curr_logprobs_and_update": 0.28642980358563364, "train/gen_step": 13128.0, "unified/clip_ratio/high_mean": 7.485922742489493e-05, "unified/clip_ratio/low_mean": 1.198236168420408e-05, "unified/clip_ratio/region_mean": 8.6841589109099e-05, "unified/kl": 0.04835753160296008, "unified/loss": 0.0018652965663932264 }, { "epoch": 2.6405124099279424, "gen/advantages_abs_mean": 0.07412971556186676, "gen/advantages_max": 0.34676653146743774, "gen/advantages_mean": -9.604264050722122e-10, "gen/advantages_min": -0.23221302032470703, "gen/advantages_std": 0.09449668973684311, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 140.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.46234989166259766, "gen/reward_std": 0.30892524123191833, "grad_norm": 0.0039537930861115456, "learning_rate": 1e-05, "loss": 0.002, "rewards/perceptual_score_reward_func/mean": 0.4601737856864929, "rewards/perceptual_score_reward_func/std": 0.31586548686027527, "step": 208, "time_profile/curr_logprobs_and_update": 0.28737615986028686, "time_profile/image_rollout": 97.1835206206888, "time_profile/old_logps": 35.69620050303638, "time_profile/ref_logps": 35.75604062154889, "time_profile/reward": 1.0410658437758684, "train/gen_step": 13192.0, "unified/clip_ratio/high_mean": 0.0007957720790727763, "unified/clip_ratio/low_mean": 2.688549557205988e-05, "unified/clip_ratio/region_mean": 0.0008226575746448361, "unified/kl": 0.049520838365424424, "unified/loss": 0.0019604504323069705 }, { "epoch": 2.653322658126501, "grad_norm": 0.005656134337186813, "learning_rate": 1e-05, "loss": 0.0047, "step": 209, "time_profile/curr_logprobs_and_update": 0.287895544199273, "train/gen_step": 13256.0, "unified/clip_ratio/high_mean": 0.00036484682641457766, "unified/clip_ratio/low_mean": 2.5205809834005777e-05, "unified/clip_ratio/region_mean": 0.00039005263442959404, "unified/kl": 0.1169839627109468, "unified/loss": 0.00449724481424596 }, { "epoch": 2.66613290632506, "gen/advantages_abs_mean": 0.05343227833509445, "gen/advantages_max": 0.2956569492816925, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.29374969005584717, "gen/advantages_std": 0.07137347757816315, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 88.5, "gen/prompt/min_length": 34.0, "gen/reward": 0.3955264389514923, "gen/reward_std": 0.3297499120235443, "grad_norm": 0.0059889634139835835, "learning_rate": 1e-05, "loss": 0.0043, "rewards/perceptual_score_reward_func/mean": 0.3783816695213318, "rewards/perceptual_score_reward_func/std": 0.3314151465892792, "step": 210, "time_profile/curr_logprobs_and_update": 0.2907968986837659, "time_profile/image_rollout": 92.30888234451413, "time_profile/old_logps": 35.662212174385786, "time_profile/ref_logps": 35.70151925459504, "time_profile/reward": 1.0841166283935308, "train/gen_step": 13320.0, "unified/clip_ratio/high_mean": 0.00975874760933948, "unified/clip_ratio/low_mean": 0.00022591120068682358, "unified/clip_ratio/region_mean": 0.009984658883695374, "unified/kl": 0.10625315242214128, "unified/loss": 0.003979327900481167 }, { "epoch": 2.678943154523619, "grad_norm": 0.006428052205592394, "learning_rate": 1e-05, "loss": 0.0029, "step": 211, "time_profile/curr_logprobs_and_update": 0.2929955120489467, "train/gen_step": 13384.0, "unified/clip_ratio/high_mean": 0.005091632628136722, "unified/clip_ratio/low_mean": 0.0001600822142791003, "unified/clip_ratio/region_mean": 0.005251714813311992, "unified/kl": 0.07312529330374673, "unified/loss": 0.002942810344393365 }, { "epoch": 2.6917534027221777, "gen/advantages_abs_mean": 0.05805864930152893, "gen/advantages_max": 0.19847756624221802, "gen/advantages_mean": -2.7939677238464355e-09, "gen/advantages_min": -0.28302890062332153, "gen/advantages_std": 0.07244789600372314, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 261.0, "gen/prompt/min_length": 228.0, "gen/reward": 0.4525802731513977, "gen/reward_std": 0.13093777000904083, "grad_norm": 0.0053682648576796055, "learning_rate": 1e-05, "loss": 0.0028, "rewards/perceptual_score_reward_func/mean": 0.4573948085308075, "rewards/perceptual_score_reward_func/std": 0.13452716171741486, "step": 212, "time_profile/curr_logprobs_and_update": 0.2902073429140728, "time_profile/image_rollout": 95.18189113773406, "time_profile/old_logps": 35.68532197549939, "time_profile/ref_logps": 35.79711914435029, "time_profile/reward": 1.238876884803176, "train/gen_step": 13448.0, "unified/clip_ratio/high_mean": 0.01146003782923799, "unified/clip_ratio/low_mean": 0.005478247154314886, "unified/clip_ratio/region_mean": 0.01693828505813144, "unified/kl": 0.06899716431507841, "unified/loss": 0.0027768009167630225 }, { "epoch": 2.7045636509207367, "grad_norm": 0.004297909326851368, "learning_rate": 1e-05, "loss": 0.0024, "step": 213, "time_profile/curr_logprobs_and_update": 0.28509608868625946, "train/gen_step": 13512.0, "unified/clip_ratio/high_mean": 0.0002028881699516205, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0002028881699516205, "unified/kl": 0.05968526436481625, "unified/loss": 0.0023397608256345848 }, { "epoch": 2.7173738991192953, "gen/advantages_abs_mean": 0.04815659299492836, "gen/advantages_max": 0.1666778326034546, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.26088178157806396, "gen/advantages_std": 0.06279948353767395, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.6226699948310852, "gen/reward_std": 0.10022295266389847, "grad_norm": 0.008419697172939777, "learning_rate": 1e-05, "loss": 0.0025, "rewards/perceptual_score_reward_func/mean": 0.6231578588485718, "rewards/perceptual_score_reward_func/std": 0.09478815644979477, "step": 214, "time_profile/curr_logprobs_and_update": 0.28961416336824186, "time_profile/image_rollout": 86.57739455625415, "time_profile/old_logps": 35.68044064939022, "time_profile/ref_logps": 35.7602315954864, "time_profile/reward": 0.8626856170594692, "train/gen_step": 13576.0, "unified/clip_ratio/high_mean": 0.0016434026474598795, "unified/clip_ratio/low_mean": 0.0001981396180781303, "unified/clip_ratio/region_mean": 0.001841542259171547, "unified/kl": 0.06322463700780645, "unified/loss": 0.002458966589983902 }, { "epoch": 2.7301841473178543, "grad_norm": 0.004779498558491468, "learning_rate": 1e-05, "loss": 0.0044, "step": 215, "time_profile/curr_logprobs_and_update": 0.2893037685425952, "train/gen_step": 13640.0, "unified/clip_ratio/high_mean": 0.004174642702309939, "unified/clip_ratio/low_mean": 0.006409092700778274, "unified/clip_ratio/region_mean": 0.01058373525575007, "unified/kl": 0.10860151320230216, "unified/loss": 0.004160132786637405 }, { "epoch": 2.742994395516413, "gen/advantages_abs_mean": 0.04880675673484802, "gen/advantages_max": 0.1898176670074463, "gen/advantages_mean": 1.6298145055770874e-09, "gen/advantages_min": -0.25191187858581543, "gen/advantages_std": 0.06369016319513321, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 140.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.5992552042007446, "gen/reward_std": 0.06945131719112396, "grad_norm": 0.00413332786411047, "learning_rate": 1e-05, "loss": 0.0041, "rewards/perceptual_score_reward_func/mean": 0.6106417179107666, "rewards/perceptual_score_reward_func/std": 0.06783042103052139, "step": 216, "time_profile/curr_logprobs_and_update": 0.2896010137919802, "time_profile/image_rollout": 83.2615770790726, "time_profile/old_logps": 35.682660933583975, "time_profile/ref_logps": 35.74056311137974, "time_profile/reward": 1.1472970116883516, "train/gen_step": 13704.0, "unified/clip_ratio/high_mean": 0.03075463885033969, "unified/clip_ratio/low_mean": 0.01234049254344427, "unified/clip_ratio/region_mean": 0.0430951316957362, "unified/kl": 0.09634337952593341, "unified/loss": 0.004022476720820123 }, { "epoch": 2.755804643714972, "grad_norm": 0.0032422547228634357, "learning_rate": 1e-05, "loss": 0.0034, "step": 217, "time_profile/curr_logprobs_and_update": 0.2900120613339823, "train/gen_step": 13768.0, "unified/clip_ratio/high_mean": 0.022472380485851318, "unified/clip_ratio/low_mean": 0.0018062998951791087, "unified/clip_ratio/region_mean": 0.02427868053837301, "unified/kl": 0.0843336817342788, "unified/loss": 0.0032579440839981544 }, { "epoch": 2.768614891913531, "gen/advantages_abs_mean": 0.027143746614456177, "gen/advantages_max": 0.18130779266357422, "gen/advantages_mean": -2.7939677238464355e-09, "gen/advantages_min": -0.14696919918060303, "gen/advantages_std": 0.04213613271713257, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.171875, "gen/prompt/max_length": 44.0, "gen/prompt/mean_length": 34.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.3068550229072571, "gen/reward_std": 0.301911324262619, "grad_norm": 0.002202271483838558, "learning_rate": 1e-05, "loss": 0.0041, "rewards/perceptual_score_reward_func/mean": 0.30286717414855957, "rewards/perceptual_score_reward_func/std": 0.29476356506347656, "step": 218, "time_profile/curr_logprobs_and_update": 0.2916179225721862, "time_profile/image_rollout": 77.22199551947415, "time_profile/old_logps": 35.64645759947598, "time_profile/ref_logps": 35.69035775959492, "time_profile/reward": 1.2508190497756004, "train/gen_step": 13832.0, "unified/clip_ratio/high_mean": 0.06337775956217229, "unified/clip_ratio/low_mean": 0.008298227623527055, "unified/clip_ratio/region_mean": 0.07167598631258443, "unified/kl": 0.09170893608825281, "unified/loss": 0.003902100499090011 }, { "epoch": 2.7814251401120895, "grad_norm": 0.0019570006988942623, "learning_rate": 1e-05, "loss": 0.0026, "step": 219, "time_profile/curr_logprobs_and_update": 0.29028787108836696, "train/gen_step": 13896.0, "unified/clip_ratio/high_mean": 0.00017125081467384007, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.00017125081467384007, "unified/kl": 0.06491467054001987, "unified/loss": 0.0025595305351089337 }, { "epoch": 2.7942353883106485, "gen/advantages_abs_mean": 0.01623566634953022, "gen/advantages_max": 0.047403693199157715, "gen/advantages_mean": -2.7939677238464355e-09, "gen/advantages_min": -0.09490931034088135, "gen/advantages_std": 0.021146854385733604, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 36.0, "gen/prompt/mean_length": 35.0, "gen/prompt/min_length": 34.0, "gen/reward": 0.8461313247680664, "gen/reward_std": 0.03506263345479965, "grad_norm": 0.002078034682199359, "learning_rate": 1e-05, "loss": 0.0025, "rewards/perceptual_score_reward_func/mean": 0.8457783460617065, "rewards/perceptual_score_reward_func/std": 0.03492708504199982, "step": 220, "time_profile/curr_logprobs_and_update": 0.29303739650640637, "time_profile/image_rollout": 85.6809338927269, "time_profile/old_logps": 35.65229227952659, "time_profile/ref_logps": 35.679596012458205, "time_profile/reward": 1.6915065199136734, "train/gen_step": 13960.0, "unified/clip_ratio/high_mean": 0.0017831098166425363, "unified/clip_ratio/low_mean": 0.00016667497675371123, "unified/clip_ratio/region_mean": 0.0019497847970342264, "unified/kl": 0.06186116026947275, "unified/loss": 0.002318331704088905 }, { "epoch": 2.8070456365092076, "grad_norm": 0.0016549167921766639, "learning_rate": 1e-05, "loss": 0.0022, "step": 221, "time_profile/curr_logprobs_and_update": 0.28271615999983624, "train/gen_step": 14024.0, "unified/clip_ratio/high_mean": 0.0004973574132236536, "unified/clip_ratio/low_mean": 3.728380397660658e-05, "unified/clip_ratio/region_mean": 0.0005346412172002601, "unified/kl": 0.055540885601658374, "unified/loss": 0.0021841253794718796 }, { "epoch": 2.819855884707766, "gen/advantages_abs_mean": 0.015649918466806412, "gen/advantages_max": 0.10429143905639648, "gen/advantages_mean": 1.3969838619232178e-09, "gen/advantages_min": -0.1413099765777588, "gen/advantages_std": 0.024658359587192535, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.15625, "gen/prompt/max_length": 33.0, "gen/prompt/mean_length": 32.0, "gen/prompt/min_length": 31.0, "gen/reward": 0.4220384359359741, "gen/reward_std": 0.416412353515625, "grad_norm": 0.0014546598540619016, "learning_rate": 1e-05, "loss": 0.0022, "rewards/perceptual_score_reward_func/mean": 0.4332512319087982, "rewards/perceptual_score_reward_func/std": 0.4173754155635834, "step": 222, "time_profile/curr_logprobs_and_update": 0.2969915381690953, "time_profile/image_rollout": 90.88989748992026, "time_profile/old_logps": 35.64420824497938, "time_profile/ref_logps": 35.67107901349664, "time_profile/reward": 2.101542688906193, "train/gen_step": 14088.0, "unified/clip_ratio/high_mean": 0.0011120515982838697, "unified/clip_ratio/low_mean": 0.0002531600803195033, "unified/clip_ratio/region_mean": 0.0013652116831508465, "unified/kl": 0.055687586311250925, "unified/loss": 0.0022361490337061696 }, { "epoch": 2.832666132906325, "grad_norm": 0.0014627320924773812, "learning_rate": 1e-05, "loss": 0.0024, "step": 223, "time_profile/curr_logprobs_and_update": 0.2890927131229546, "train/gen_step": 14152.0, "unified/clip_ratio/high_mean": 6.403710813174257e-05, "unified/clip_ratio/low_mean": 2.539202614570968e-05, "unified/clip_ratio/region_mean": 8.942913427745225e-05, "unified/kl": 0.060441506502684206, "unified/loss": 0.0024136415540851885 }, { "epoch": 2.8454763811048838, "gen/advantages_abs_mean": 0.06136321276426315, "gen/advantages_max": 0.2737995982170105, "gen/advantages_mean": -2.3283064365386963e-10, "gen/advantages_min": -0.27382445335388184, "gen/advantages_std": 0.079731285572052, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 30.0, "gen/prompt/mean_length": 27.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.2018337845802307, "gen/reward_std": 0.14443330466747284, "grad_norm": 0.0031836305279284716, "learning_rate": 1e-05, "loss": 0.0022, "rewards/perceptual_score_reward_func/mean": 0.19993999600410461, "rewards/perceptual_score_reward_func/std": 0.15581290423870087, "step": 224, "time_profile/curr_logprobs_and_update": 0.2822554874292109, "time_profile/image_rollout": 86.69328206963837, "time_profile/old_logps": 35.651397259905934, "time_profile/ref_logps": 35.67883262783289, "time_profile/reward": 1.7083067148923874, "train/gen_step": 14216.0, "unified/clip_ratio/high_mean": 0.00011875155269081006, "unified/clip_ratio/low_mean": 0.0006717665119140293, "unified/clip_ratio/region_mean": 0.0007905180646048393, "unified/kl": 0.05397293172427453, "unified/loss": 0.002211625182098942 }, { "epoch": 2.858286629303443, "grad_norm": 0.0036669722758233547, "learning_rate": 1e-05, "loss": 0.0012, "step": 225, "time_profile/curr_logprobs_and_update": 0.2889335306826979, "train/gen_step": 14280.0, "unified/clip_ratio/high_mean": 3.7436996535689104e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 3.7436996535689104e-05, "unified/kl": 0.03027197896153666, "unified/loss": 0.0011347782128723338 }, { "epoch": 2.8710968775020014, "gen/advantages_abs_mean": 0.060056887567043304, "gen/advantages_max": 0.2160344123840332, "gen/advantages_mean": 1.6298145055770874e-09, "gen/advantages_min": -0.29413020610809326, "gen/advantages_std": 0.07890523970127106, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 185.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.6171910762786865, "gen/reward_std": 0.10012081265449524, "grad_norm": 0.002574487356469035, "learning_rate": 1e-05, "loss": 0.0016, "rewards/perceptual_score_reward_func/mean": 0.6302154064178467, "rewards/perceptual_score_reward_func/std": 0.09106422960758209, "step": 226, "time_profile/curr_logprobs_and_update": 0.28898188957828097, "time_profile/image_rollout": 94.76224956661463, "time_profile/old_logps": 35.704933416098356, "time_profile/ref_logps": 35.78546800278127, "time_profile/reward": 1.1764553282409906, "train/gen_step": 14344.0, "unified/clip_ratio/high_mean": 0.00030724572025064845, "unified/clip_ratio/low_mean": 4.166991584497737e-05, "unified/clip_ratio/region_mean": 0.0003489156360956258, "unified/kl": 0.04056441711145453, "unified/loss": 0.0017030270500981715 }, { "epoch": 2.8839071257005604, "grad_norm": 0.005676161963492632, "learning_rate": 1e-05, "loss": 0.0046, "step": 227, "time_profile/curr_logprobs_and_update": 0.2964037763304077, "train/gen_step": 14408.0, "unified/clip_ratio/high_mean": 0.0003664525029307697, "unified/clip_ratio/low_mean": 8.569373949285364e-05, "unified/clip_ratio/region_mean": 0.000452146234238171, "unified/kl": 0.11509677162393928, "unified/loss": 0.004453327323972189 }, { "epoch": 2.8967173738991194, "gen/advantages_abs_mean": 0.033830348402261734, "gen/advantages_max": 0.14224708080291748, "gen/advantages_mean": -1.3969838619232178e-09, "gen/advantages_min": -0.2486705780029297, "gen/advantages_std": 0.04694213718175888, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 87.0, "gen/prompt/min_length": 31.0, "gen/reward": 0.7471082210540771, "gen/reward_std": 0.09803838282823563, "grad_norm": 0.005515103228390217, "learning_rate": 1e-05, "loss": 0.0044, "rewards/perceptual_score_reward_func/mean": 0.7399195432662964, "rewards/perceptual_score_reward_func/std": 0.10945288091897964, "step": 228, "time_profile/curr_logprobs_and_update": 0.29186375689459965, "time_profile/image_rollout": 89.08404904603958, "time_profile/old_logps": 35.664834290742874, "time_profile/ref_logps": 35.68366582505405, "time_profile/reward": 1.2496778517961502, "train/gen_step": 14472.0, "unified/clip_ratio/high_mean": 0.013442613232655276, "unified/clip_ratio/low_mean": 0.0009425432981515769, "unified/clip_ratio/region_mean": 0.014385156513526454, "unified/kl": 0.1082348006311804, "unified/loss": 0.00420809791364718 }, { "epoch": 2.909527622097678, "grad_norm": 0.0044300127774477005, "learning_rate": 1e-05, "loss": 0.0042, "step": 229, "time_profile/curr_logprobs_and_update": 0.29350452186190523, "train/gen_step": 14536.0, "unified/clip_ratio/high_mean": 0.000650564178613422, "unified/clip_ratio/low_mean": 0.005874285083336872, "unified/clip_ratio/region_mean": 0.0065248492301179795, "unified/kl": 0.10480844264384359, "unified/loss": 0.004268346909498177 }, { "epoch": 2.922337870296237, "gen/advantages_abs_mean": 0.08245858550071716, "gen/advantages_max": 0.2647748589515686, "gen/advantages_mean": -3.7834979593753815e-10, "gen/advantages_min": -0.3321816921234131, "gen/advantages_std": 0.10184665769338608, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 42.0, "gen/prompt/mean_length": 33.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.36596447229385376, "gen/reward_std": 0.22325140237808228, "grad_norm": 0.003050280502066016, "learning_rate": 1e-05, "loss": 0.0036, "rewards/perceptual_score_reward_func/mean": 0.35242119431495667, "rewards/perceptual_score_reward_func/std": 0.22620998322963715, "step": 230, "time_profile/curr_logprobs_and_update": 0.29015435941983014, "time_profile/image_rollout": 98.16377575136721, "time_profile/old_logps": 35.6339815557003, "time_profile/ref_logps": 35.689124369993806, "time_profile/reward": 1.4899360593408346, "train/gen_step": 14600.0, "unified/clip_ratio/high_mean": 0.003210875910554023, "unified/clip_ratio/low_mean": 0.017312906777988246, "unified/clip_ratio/region_mean": 0.02052378270400368, "unified/kl": 0.08625791693339124, "unified/loss": 0.0036507347490442044 }, { "epoch": 2.935148118494796, "grad_norm": 0.0037899394519627094, "learning_rate": 1e-05, "loss": 0.0017, "step": 231, "time_profile/curr_logprobs_and_update": 0.2809451754146721, "train/gen_step": 14664.0, "unified/clip_ratio/high_mean": 0.0003451885913818842, "unified/clip_ratio/low_mean": 0.00013730012778978562, "unified/clip_ratio/region_mean": 0.0004824887219001539, "unified/kl": 0.0409627566114068, "unified/loss": 0.0017391420788044343 }, { "epoch": 2.9479583666933546, "gen/advantages_abs_mean": 0.05695874243974686, "gen/advantages_max": 0.2941986918449402, "gen/advantages_mean": 2.0954757928848267e-09, "gen/advantages_min": -0.2176368236541748, "gen/advantages_std": 0.07234527170658112, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 80.5, "gen/prompt/min_length": 24.0, "gen/reward": 0.4000024199485779, "gen/reward_std": 0.31324514746665955, "grad_norm": 0.0029120896942913532, "learning_rate": 1e-05, "loss": 0.0017, "rewards/perceptual_score_reward_func/mean": 0.39365696907043457, "rewards/perceptual_score_reward_func/std": 0.31685662269592285, "step": 232, "time_profile/curr_logprobs_and_update": 0.2896963453677017, "time_profile/image_rollout": 90.54285294935107, "time_profile/old_logps": 35.659948740154505, "time_profile/ref_logps": 35.70540129579604, "time_profile/reward": 1.3401405308395624, "train/gen_step": 14728.0, "unified/clip_ratio/high_mean": 0.004589359151395911, "unified/clip_ratio/low_mean": 0.0004914687769996817, "unified/clip_ratio/region_mean": 0.0050808279165721615, "unified/kl": 0.041754993551876396, "unified/loss": 0.0016714632110961247 }, { "epoch": 2.9607686148919137, "grad_norm": 0.003255711402744055, "learning_rate": 1e-05, "loss": 0.002, "step": 233, "time_profile/curr_logprobs_and_update": 0.28853817415074445, "train/gen_step": 14792.0, "unified/clip_ratio/high_mean": 0.00022434214497479843, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.00022434214497479843, "unified/kl": 0.04886005027219653, "unified/loss": 0.0021349032186321892 }, { "epoch": 2.9735788630904723, "gen/advantages_abs_mean": 0.06837429106235504, "gen/advantages_max": 0.20085787773132324, "gen/advantages_mean": -1.3969838619232178e-09, "gen/advantages_min": -0.24849843978881836, "gen/advantages_std": 0.08356224000453949, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 215.5, "gen/prompt/min_length": 137.0, "gen/reward": 0.5199116468429565, "gen/reward_std": 0.08993534743785858, "grad_norm": 0.005596943199634552, "learning_rate": 1e-05, "loss": 0.002, "rewards/perceptual_score_reward_func/mean": 0.5330313444137573, "rewards/perceptual_score_reward_func/std": 0.08884021639823914, "step": 234, "time_profile/curr_logprobs_and_update": 0.28999189706519246, "time_profile/image_rollout": 90.35698483511806, "time_profile/old_logps": 35.76694936491549, "time_profile/ref_logps": 35.79822871275246, "time_profile/reward": 1.2922454122453928, "train/gen_step": 14856.0, "unified/clip_ratio/high_mean": 0.002169189732740051, "unified/clip_ratio/low_mean": 7.525764704041649e-05, "unified/clip_ratio/region_mean": 0.0022444473797804676, "unified/kl": 0.05060361948562786, "unified/loss": 0.0021131058651917556 }, { "epoch": 2.9863891112890313, "grad_norm": 0.00613767746835947, "learning_rate": 1e-05, "loss": 0.003, "step": 235, "time_profile/curr_logprobs_and_update": 0.2892706178536173, "train/gen_step": 14920.0, "unified/clip_ratio/high_mean": 0.00022474777415482095, "unified/clip_ratio/low_mean": 1.6009220416890457e-05, "unified/clip_ratio/region_mean": 0.0002407569945717114, "unified/kl": 0.0750474848318845, "unified/loss": 0.0028697888183160103 }, { "epoch": 2.99919935948759, "gen/advantages_abs_mean": 0.05524168536067009, "gen/advantages_max": 0.19999942183494568, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.35143446922302246, "gen/advantages_std": 0.07161935418844223, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.576038658618927, "gen/reward_std": 0.09756524860858917, "grad_norm": 0.004759375471621752, "learning_rate": 1e-05, "loss": 0.0028, "rewards/perceptual_score_reward_func/mean": 0.5787273645401001, "rewards/perceptual_score_reward_func/std": 0.08526910096406937, "step": 236, "time_profile/curr_logprobs_and_update": 0.2881124487903435, "time_profile/image_rollout": 91.76546191610396, "time_profile/old_logps": 35.70581114850938, "time_profile/ref_logps": 35.819912476465106, "time_profile/reward": 1.0791106317192316, "train/gen_step": 14984.0, "unified/clip_ratio/high_mean": 0.0028773223521056934, "unified/clip_ratio/low_mean": 0.0034019033937511267, "unified/clip_ratio/region_mean": 0.006279225777689135, "unified/kl": 0.07088860735530034, "unified/loss": 0.0028101922980567906 }, { "epoch": 3.0, "grad_norm": 0.005377649795264006, "learning_rate": 1e-05, "loss": 0.0006, "step": 237, "time_profile/curr_logprobs_and_update": 0.3263985142111778, "train/gen_step": 14988.0, "unified/clip_ratio/high_mean": 0.00037564977537840605, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.00037564977537840605, "unified/kl": 0.23306765407323837, "unified/loss": -0.02903024887200445 }, { "epoch": 3.012810248198559, "grad_norm": 0.0040442608296871185, "learning_rate": 1e-05, "loss": 0.0054, "step": 238, "time_profile/curr_logprobs_and_update": 0.2922834643977694, "train/gen_step": 15052.0, "unified/clip_ratio/high_mean": 0.007114615003956715, "unified/clip_ratio/low_mean": 0.005583924483289593, "unified/clip_ratio/region_mean": 0.01269853939083987, "unified/kl": 0.13325618638191372, "unified/loss": 0.005477978451381205 }, { "epoch": 3.0256204963971176, "gen/advantages_abs_mean": 0.06815211474895477, "gen/advantages_max": 0.28162434697151184, "gen/advantages_mean": -4.656612873077393e-10, "gen/advantages_min": -0.36226972937583923, "gen/advantages_std": 0.0867423266172409, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 126.0, "gen/prompt/min_length": 24.0, "gen/reward": 0.48455798625946045, "gen/reward_std": 0.14753340184688568, "grad_norm": 0.004220765084028244, "learning_rate": 1e-05, "loss": 0.0047, "rewards/perceptual_score_reward_func/mean": 0.4674367308616638, "rewards/perceptual_score_reward_func/std": 0.15108731389045715, "step": 239, "time_profile/curr_logprobs_and_update": 0.2890873631986324, "time_profile/image_rollout": 85.50802209228277, "time_profile/old_logps": 35.64921068586409, "time_profile/ref_logps": 35.71976723894477, "time_profile/reward": 1.38679701089859, "train/gen_step": 15116.0, "unified/clip_ratio/high_mean": 0.016038957779528573, "unified/clip_ratio/low_mean": 0.021020568940002704, "unified/clip_ratio/region_mean": 0.03705952680320479, "unified/kl": 0.11398179235402495, "unified/loss": 0.006121751257524011 }, { "epoch": 3.0384307445956766, "grad_norm": 0.004147673025727272, "learning_rate": 1e-05, "loss": 0.0025, "step": 240, "time_profile/curr_logprobs_and_update": 0.29534867490292527, "train/gen_step": 15180.0, "unified/clip_ratio/high_mean": 0.00034446546123945154, "unified/clip_ratio/low_mean": 0.0001585743866598932, "unified/clip_ratio/region_mean": 0.0005030398488088395, "unified/kl": 0.062158423592336476, "unified/loss": 0.0025324274738522945 }, { "epoch": 3.051240992794235, "gen/advantages_abs_mean": 0.05185795575380325, "gen/advantages_max": 0.19245928525924683, "gen/advantages_mean": 4.656612873077393e-10, "gen/advantages_min": -0.19449368119239807, "gen/advantages_std": 0.06433796137571335, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 80.5, "gen/prompt/min_length": 24.0, "gen/reward": 0.18533486127853394, "gen/reward_std": 0.12151897698640823, "grad_norm": 0.0035060045775026083, "learning_rate": 1e-05, "loss": 0.0037, "rewards/perceptual_score_reward_func/mean": 0.18132202327251434, "rewards/perceptual_score_reward_func/std": 0.12059197574853897, "step": 241, "time_profile/curr_logprobs_and_update": 0.2936995446216315, "time_profile/image_rollout": 92.46782938763499, "time_profile/old_logps": 35.647363713011146, "time_profile/ref_logps": 35.7009810525924, "time_profile/reward": 1.3794072847813368, "train/gen_step": 15244.0, "unified/clip_ratio/high_mean": 0.0025567706343281316, "unified/clip_ratio/low_mean": 0.0014044004792594933, "unified/clip_ratio/region_mean": 0.0039611710963072255, "unified/kl": 0.05547615705290809, "unified/loss": 0.004834193030546885 }, { "epoch": 3.0640512409927942, "grad_norm": 0.002918737241998315, "learning_rate": 1e-05, "loss": 0.0013, "step": 242, "time_profile/curr_logprobs_and_update": 0.28326690808171406, "train/gen_step": 15308.0, "unified/clip_ratio/high_mean": 0.0, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0, "unified/kl": 0.03205126043758355, "unified/loss": 0.0012558446269395063 }, { "epoch": 3.0768614891913533, "gen/advantages_abs_mean": 0.04403460770845413, "gen/advantages_max": 0.2058252990245819, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.245433509349823, "gen/advantages_std": 0.06192699819803238, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 35.0, "gen/prompt/mean_length": 29.5, "gen/prompt/min_length": 24.0, "gen/reward": 0.6110314726829529, "gen/reward_std": 0.24258162081241608, "grad_norm": 0.0026578244287520647, "learning_rate": 1e-05, "loss": 0.0001, "rewards/perceptual_score_reward_func/mean": 0.6051622629165649, "rewards/perceptual_score_reward_func/std": 0.24469920992851257, "step": 243, "time_profile/curr_logprobs_and_update": 0.2965333420142997, "time_profile/image_rollout": 88.86197500675917, "time_profile/old_logps": 35.613538667559624, "time_profile/ref_logps": 35.66835015080869, "time_profile/reward": 1.5575009007006884, "train/gen_step": 15372.0, "unified/clip_ratio/high_mean": 5.28329064763966e-05, "unified/clip_ratio/low_mean": 3.773496882786276e-05, "unified/clip_ratio/region_mean": 9.056787530425936e-05, "unified/kl": 0.03381443230318837, "unified/loss": 7.984058447618736e-05 }, { "epoch": 3.089671737389912, "grad_norm": 0.0016529714921489358, "learning_rate": 1e-05, "loss": 0.0018, "step": 244, "time_profile/curr_logprobs_and_update": 0.28808122701593675, "train/gen_step": 15436.0, "unified/clip_ratio/high_mean": 0.0001275367403650307, "unified/clip_ratio/low_mean": 1.1909298336831853e-05, "unified/clip_ratio/region_mean": 0.00013944603870186256, "unified/kl": 0.04395691870013252, "unified/loss": 0.001838372672409605 }, { "epoch": 3.102481985588471, "gen/advantages_abs_mean": 0.05394706502556801, "gen/advantages_max": 0.24188965559005737, "gen/advantages_mean": 6.984919309616089e-10, "gen/advantages_min": -0.29084014892578125, "gen/advantages_std": 0.0771428570151329, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 133.0, "gen/prompt/min_length": 38.0, "gen/reward": 0.5607863068580627, "gen/reward_std": 0.30254271626472473, "grad_norm": 0.0015357909724116325, "learning_rate": 1e-05, "loss": 0.0016, "rewards/perceptual_score_reward_func/mean": 0.5605348348617554, "rewards/perceptual_score_reward_func/std": 0.30506521463394165, "step": 245, "time_profile/curr_logprobs_and_update": 0.2840186757093761, "time_profile/image_rollout": 86.64872058853507, "time_profile/old_logps": 35.670260759070516, "time_profile/ref_logps": 35.72483950294554, "time_profile/reward": 2.2138646077364683, "train/gen_step": 15500.0, "unified/clip_ratio/high_mean": 0.000323626689350931, "unified/clip_ratio/low_mean": 0.0001830254377637175, "unified/clip_ratio/region_mean": 0.0005066521271146485, "unified/kl": 0.0435222196392715, "unified/loss": 0.0005058303959231125 }, { "epoch": 3.1152922337870295, "grad_norm": 0.0013075098395347595, "learning_rate": 1e-05, "loss": 0.0017, "step": 246, "time_profile/curr_logprobs_and_update": 0.29552398252417333, "train/gen_step": 15564.0, "unified/clip_ratio/high_mean": 0.0008637319715489866, "unified/clip_ratio/low_mean": 5.0364660637569614e-05, "unified/clip_ratio/region_mean": 0.0009140966330960509, "unified/kl": 0.04288260970497504, "unified/loss": 0.0017171224171761423 }, { "epoch": 3.1281024819855885, "gen/advantages_abs_mean": 0.037114981561899185, "gen/advantages_max": 0.16461652517318726, "gen/advantages_mean": 4.6566128730773926e-09, "gen/advantages_min": -0.19582903385162354, "gen/advantages_std": 0.05011643096804619, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 143.0, "gen/prompt/min_length": 143.0, "gen/reward": 0.6422978043556213, "gen/reward_std": 0.09519995748996735, "grad_norm": 0.003914408851414919, "learning_rate": 1e-05, "loss": 0.0036, "rewards/perceptual_score_reward_func/mean": 0.6484772562980652, "rewards/perceptual_score_reward_func/std": 0.09529213607311249, "step": 247, "time_profile/curr_logprobs_and_update": 0.2900813828164246, "time_profile/image_rollout": 93.27366822212934, "time_profile/old_logps": 35.687854655086994, "time_profile/ref_logps": 35.74452555552125, "time_profile/reward": 0.8686411567032337, "train/gen_step": 15628.0, "unified/clip_ratio/high_mean": 0.004472575677027635, "unified/clip_ratio/low_mean": 0.0002732068396653631, "unified/clip_ratio/region_mean": 0.004745782523059461, "unified/kl": 0.09377857428626157, "unified/loss": 0.00661650193069363 }, { "epoch": 3.1409127301841475, "grad_norm": 0.004214221611618996, "learning_rate": 1e-05, "loss": 0.0095, "step": 248, "time_profile/curr_logprobs_and_update": 0.2855500411824323, "train/gen_step": 15692.0, "unified/clip_ratio/high_mean": 0.002388650888860866, "unified/clip_ratio/low_mean": 0.00043833946892846143, "unified/clip_ratio/region_mean": 0.002826990362336801, "unified/kl": 0.23597962281201035, "unified/loss": 0.008077786382727936 }, { "epoch": 3.153722978382706, "gen/advantages_abs_mean": 0.057654447853565216, "gen/advantages_max": 0.2809915542602539, "gen/advantages_mean": -1.1641532182693481e-09, "gen/advantages_min": -0.29573673009872437, "gen/advantages_std": 0.0747082307934761, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 185.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.5992565155029297, "gen/reward_std": 0.17862142622470856, "grad_norm": 0.004478012677282095, "learning_rate": 1e-05, "loss": 0.0098, "rewards/perceptual_score_reward_func/mean": 0.6018717288970947, "rewards/perceptual_score_reward_func/std": 0.18333376944065094, "step": 249, "time_profile/curr_logprobs_and_update": 0.29048692525248043, "time_profile/image_rollout": 98.79939872026443, "time_profile/old_logps": 35.707147816196084, "time_profile/ref_logps": 35.76749726943672, "time_profile/reward": 1.223227709531784, "train/gen_step": 15756.0, "unified/clip_ratio/high_mean": 0.004625103229045635, "unified/clip_ratio/low_mean": 0.004679095056417282, "unified/clip_ratio/region_mean": 0.00930419833457563, "unified/kl": 0.2325795415090397, "unified/loss": 0.006798942915338557 }, { "epoch": 3.166533226581265, "grad_norm": 0.005828510504215956, "learning_rate": 1e-05, "loss": 0.0053, "step": 250, "time_profile/curr_logprobs_and_update": 0.295324581413297, "train/gen_step": 15820.0, "unified/clip_ratio/high_mean": 0.0033738677548171836, "unified/clip_ratio/low_mean": 0.00015024944696051534, "unified/clip_ratio/region_mean": 0.003524117216329614, "unified/kl": 0.1311271166196093, "unified/loss": 0.00534750197766698 }, { "epoch": 3.1793434747798237, "gen/advantages_abs_mean": 0.051578670740127563, "gen/advantages_max": 0.2560831308364868, "gen/advantages_mean": 4.889443516731262e-09, "gen/advantages_min": -0.24882763624191284, "gen/advantages_std": 0.07479658722877502, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 130.0, "gen/prompt/min_length": 32.0, "gen/reward": 0.6277204751968384, "gen/reward_std": 0.1594085395336151, "grad_norm": 0.005873980466276407, "learning_rate": 1e-05, "loss": 0.0032, "rewards/perceptual_score_reward_func/mean": 0.6243882775306702, "rewards/perceptual_score_reward_func/std": 0.15960080921649933, "step": 251, "time_profile/curr_logprobs_and_update": 0.28632479067891836, "time_profile/image_rollout": 93.90890072844923, "time_profile/old_logps": 35.66965501010418, "time_profile/ref_logps": 35.7002028003335, "time_profile/reward": 1.3597954772412777, "train/gen_step": 15884.0, "unified/clip_ratio/high_mean": 0.024379982000027667, "unified/clip_ratio/low_mean": 0.001214746066580119, "unified/clip_ratio/region_mean": 0.02559472790289874, "unified/kl": 0.10064856286044233, "unified/loss": 0.002321239304365008 }, { "epoch": 3.1921537229783827, "grad_norm": 0.0062650516629219055, "learning_rate": 1e-05, "loss": 0.0021, "step": 252, "time_profile/curr_logprobs_and_update": 0.28965292443172075, "train/gen_step": 15948.0, "unified/clip_ratio/high_mean": 0.0006192159962665755, "unified/clip_ratio/low_mean": 3.767355974559905e-05, "unified/clip_ratio/region_mean": 0.000656889557831164, "unified/kl": 0.051964796730317175, "unified/loss": 0.0021718055440942408 }, { "epoch": 3.2049639711769418, "gen/advantages_abs_mean": 0.05800483375787735, "gen/advantages_max": 0.22309106588363647, "gen/advantages_mean": 1.3969838619232178e-09, "gen/advantages_min": -0.30087709426879883, "gen/advantages_std": 0.07652728259563446, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 134.5, "gen/prompt/min_length": 41.0, "gen/reward": 0.703700065612793, "gen/reward_std": 0.1433587670326233, "grad_norm": 0.003542720340192318, "learning_rate": 1e-05, "loss": 0.0029, "rewards/perceptual_score_reward_func/mean": 0.699893593788147, "rewards/perceptual_score_reward_func/std": 0.14788110554218292, "step": 253, "time_profile/curr_logprobs_and_update": 0.2873108710628003, "time_profile/image_rollout": 78.67467383109033, "time_profile/old_logps": 35.68670476414263, "time_profile/ref_logps": 35.72687746025622, "time_profile/reward": 1.352814106270671, "train/gen_step": 16012.0, "unified/clip_ratio/high_mean": 0.00668658099584718, "unified/clip_ratio/low_mean": 0.0005819909283673042, "unified/clip_ratio/region_mean": 0.007268571912391053, "unified/kl": 0.054736606223741546, "unified/loss": 0.0021627833293678123 }, { "epoch": 3.2177742193755003, "grad_norm": 0.0020388448610901833, "learning_rate": 1e-05, "loss": 0.0024, "step": 254, "time_profile/curr_logprobs_and_update": 0.2826185535523109, "train/gen_step": 16076.0, "unified/clip_ratio/high_mean": 0.0021658961159118917, "unified/clip_ratio/low_mean": 4.8730007620179094e-05, "unified/clip_ratio/region_mean": 0.0022146261235320708, "unified/kl": 0.06000680977012962, "unified/loss": 0.0024088529658570224 }, { "epoch": 3.2305844675740594, "gen/advantages_abs_mean": 0.03484244644641876, "gen/advantages_max": 0.17698565125465393, "gen/advantages_mean": 1.1641532182693481e-09, "gen/advantages_min": -0.13506032526493073, "gen/advantages_std": 0.0459853820502758, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 38.0, "gen/prompt/mean_length": 31.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.5287748575210571, "gen/reward_std": 0.29880011081695557, "grad_norm": 0.0023810556158423424, "learning_rate": 1e-05, "loss": 0.0025, "rewards/perceptual_score_reward_func/mean": 0.5372272729873657, "rewards/perceptual_score_reward_func/std": 0.2943369746208191, "step": 255, "time_profile/curr_logprobs_and_update": 0.2931378345238045, "time_profile/image_rollout": 93.7112015336752, "time_profile/old_logps": 35.630799155682325, "time_profile/ref_logps": 35.65894949436188, "time_profile/reward": 1.3760113697499037, "train/gen_step": 16140.0, "unified/clip_ratio/high_mean": 0.009280130810111586, "unified/clip_ratio/low_mean": 0.000359229604327993, "unified/clip_ratio/region_mean": 0.009639360386245244, "unified/kl": 0.05741891771322116, "unified/loss": 0.005403364793892251 }, { "epoch": 3.243394715772618, "grad_norm": 0.002519667847082019, "learning_rate": 1e-05, "loss": 0.0017, "step": 256, "time_profile/curr_logprobs_and_update": 0.28939395473571494, "train/gen_step": 16204.0, "unified/clip_ratio/high_mean": 0.00035258051229902776, "unified/clip_ratio/low_mean": 5.343413431546651e-05, "unified/clip_ratio/region_mean": 0.00040601464388601016, "unified/kl": 0.04141872317995876, "unified/loss": 0.0016114869595185155 }, { "epoch": 3.256204963971177, "gen/advantages_abs_mean": 0.02622368559241295, "gen/advantages_max": 0.08395981788635254, "gen/advantages_mean": -2.0954757928848267e-09, "gen/advantages_min": -0.14057499170303345, "gen/advantages_std": 0.03329704329371452, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 42.0, "gen/prompt/mean_length": 39.5, "gen/prompt/min_length": 37.0, "gen/reward": 0.7857663631439209, "gen/reward_std": 0.04608968645334244, "grad_norm": 0.0017559886910021305, "learning_rate": 1e-05, "loss": 0.0011, "rewards/perceptual_score_reward_func/mean": 0.7845939993858337, "rewards/perceptual_score_reward_func/std": 0.049433447420597076, "step": 257, "time_profile/curr_logprobs_and_update": 0.285079111228697, "time_profile/image_rollout": 89.50349676422775, "time_profile/old_logps": 35.64108816161752, "time_profile/ref_logps": 35.680126287043095, "time_profile/reward": 1.5028599482029676, "train/gen_step": 16268.0, "unified/clip_ratio/high_mean": 0.0007780354662827449, "unified/clip_ratio/low_mean": 0.0001060587646861677, "unified/clip_ratio/region_mean": 0.0008840942300594179, "unified/kl": 0.042232642124872655, "unified/loss": -0.0017025310626195278 }, { "epoch": 3.2690152121697356, "grad_norm": 0.0020468831062316895, "learning_rate": 1e-05, "loss": 0.0019, "step": 258, "time_profile/curr_logprobs_and_update": 0.29102804412832484, "train/gen_step": 16332.0, "unified/clip_ratio/high_mean": 0.00019496802542562364, "unified/clip_ratio/low_mean": 3.838280917989323e-05, "unified/clip_ratio/region_mean": 0.00023335083460551687, "unified/kl": 0.049175292369909585, "unified/loss": 0.0019437705554992135 }, { "epoch": 3.2818254603682946, "gen/advantages_abs_mean": 0.05668795853853226, "gen/advantages_max": 0.22122755646705627, "gen/advantages_mean": 4.656612873077393e-10, "gen/advantages_min": -0.30950745940208435, "gen/advantages_std": 0.07221390306949615, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 126.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.3420964181423187, "gen/reward_std": 0.18341200053691864, "grad_norm": 0.0035164812579751015, "learning_rate": 1e-05, "loss": 0.0016, "rewards/perceptual_score_reward_func/mean": 0.332736998796463, "rewards/perceptual_score_reward_func/std": 0.1799333691596985, "step": 259, "time_profile/curr_logprobs_and_update": 0.28926581400446594, "time_profile/image_rollout": 84.9461916051805, "time_profile/old_logps": 35.632869094610214, "time_profile/ref_logps": 35.70344665646553, "time_profile/reward": 1.4849092904478312, "train/gen_step": 16396.0, "unified/clip_ratio/high_mean": 0.000548151053408219, "unified/clip_ratio/low_mean": 0.0005735490922234021, "unified/clip_ratio/region_mean": 0.001121700142903137, "unified/kl": 0.0501270120148547, "unified/loss": 0.002472478828394742 }, { "epoch": 3.2946357085668536, "grad_norm": 0.00313062802888453, "learning_rate": 1e-05, "loss": 0.0022, "step": 260, "time_profile/curr_logprobs_and_update": 0.2928492010105401, "train/gen_step": 16460.0, "unified/clip_ratio/high_mean": 1.1160714166180696e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 1.1160714166180696e-05, "unified/kl": 0.0554106883937493, "unified/loss": 0.0023214585526147857 }, { "epoch": 3.307445956765412, "gen/advantages_abs_mean": 0.050924405455589294, "gen/advantages_max": 0.20732958614826202, "gen/advantages_mean": -1.3969838619232178e-09, "gen/advantages_min": -0.24469858407974243, "gen/advantages_std": 0.06935644894838333, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 132.5, "gen/prompt/min_length": 37.0, "gen/reward": 0.5703046321868896, "gen/reward_std": 0.27719688415527344, "grad_norm": 0.0019420040771365166, "learning_rate": 1e-05, "loss": 0.0031, "rewards/perceptual_score_reward_func/mean": 0.5662599802017212, "rewards/perceptual_score_reward_func/std": 0.27992069721221924, "step": 261, "time_profile/curr_logprobs_and_update": 0.29200352300540544, "time_profile/image_rollout": 88.96158868446946, "time_profile/old_logps": 35.90520252101123, "time_profile/ref_logps": 35.727664368227124, "time_profile/reward": 1.2210984416306019, "train/gen_step": 16524.0, "unified/clip_ratio/high_mean": 0.00047545267625537235, "unified/clip_ratio/low_mean": 1.2765523024427239e-05, "unified/clip_ratio/region_mean": 0.0004882181992797996, "unified/kl": 0.058303156634792686, "unified/loss": 0.00208740836387733 }, { "epoch": 3.3202562049639712, "grad_norm": 0.002041127998381853, "learning_rate": 1e-05, "loss": 0.0025, "step": 262, "time_profile/curr_logprobs_and_update": 0.28587006652378477, "train/gen_step": 16588.0, "unified/clip_ratio/high_mean": 8.956663259596098e-05, "unified/clip_ratio/low_mean": 5.191157106310129e-05, "unified/clip_ratio/region_mean": 0.00014147820365906227, "unified/kl": 0.06144321715692058, "unified/loss": 0.0024497655213053804 }, { "epoch": 3.3330664531625303, "gen/advantages_abs_mean": 0.04744440317153931, "gen/advantages_max": 0.1995270550251007, "gen/advantages_mean": 9.313225746154785e-10, "gen/advantages_min": -0.21491345763206482, "gen/advantages_std": 0.06199962645769119, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 140.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.5491054654121399, "gen/reward_std": 0.08672457933425903, "grad_norm": 0.002755101304501295, "learning_rate": 1e-05, "loss": 0.003, "rewards/perceptual_score_reward_func/mean": 0.5549399852752686, "rewards/perceptual_score_reward_func/std": 0.08706492930650711, "step": 263, "time_profile/curr_logprobs_and_update": 0.2881781238829717, "time_profile/image_rollout": 91.32398653961718, "time_profile/old_logps": 35.87527330592275, "time_profile/ref_logps": 35.74318656511605, "time_profile/reward": 1.2014333847910166, "train/gen_step": 16652.0, "unified/clip_ratio/high_mean": 0.0002988719397762907, "unified/clip_ratio/low_mean": 0.0005312867833708879, "unified/clip_ratio/region_mean": 0.0008301587204186944, "unified/kl": 0.07159855647478253, "unified/loss": 0.002919944567111088 }, { "epoch": 3.345876701361089, "grad_norm": 0.004094029776751995, "learning_rate": 1e-05, "loss": 0.0047, "step": 264, "time_profile/curr_logprobs_and_update": 0.28873762642615475, "train/gen_step": 16716.0, "unified/clip_ratio/high_mean": 0.00018977822855958948, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.00018977822855958948, "unified/kl": 0.11650378745980561, "unified/loss": 0.00521194833345362 }, { "epoch": 3.358686949559648, "gen/advantages_abs_mean": 0.05893790349364281, "gen/advantages_max": 0.2842107117176056, "gen/advantages_mean": -2.3283064365386963e-09, "gen/advantages_min": -0.2532750964164734, "gen/advantages_std": 0.08093169331550598, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 185.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.6093672513961792, "gen/reward_std": 0.1476283073425293, "grad_norm": 0.010794364847242832, "learning_rate": 1e-05, "loss": 0.0044, "rewards/perceptual_score_reward_func/mean": 0.62360680103302, "rewards/perceptual_score_reward_func/std": 0.13393056392669678, "step": 265, "time_profile/curr_logprobs_and_update": 0.2906798219191842, "time_profile/image_rollout": 91.62773103825748, "time_profile/old_logps": 35.756982708349824, "time_profile/ref_logps": 35.8212514705956, "time_profile/reward": 1.3091953620314598, "train/gen_step": 16780.0, "unified/clip_ratio/high_mean": 0.0007829357255104696, "unified/clip_ratio/low_mean": 0.00030623077418567846, "unified/clip_ratio/region_mean": 0.0010891665015151375, "unified/kl": 0.11171365738846362, "unified/loss": 0.005526022511730844 }, { "epoch": 3.3714971977582064, "grad_norm": 0.0054925777949392796, "learning_rate": 1e-05, "loss": 0.0058, "step": 266, "time_profile/curr_logprobs_and_update": 0.28407792330835946, "train/gen_step": 16844.0, "unified/clip_ratio/high_mean": 0.006384057131981535, "unified/clip_ratio/low_mean": 0.00062740799603489, "unified/clip_ratio/region_mean": 0.007011465103460068, "unified/kl": 0.1416146681876853, "unified/loss": 0.005470271369631519 }, { "epoch": 3.3843074459567655, "gen/advantages_abs_mean": 0.05981067568063736, "gen/advantages_max": 0.239865243434906, "gen/advantages_mean": -1.1641532182693481e-09, "gen/advantages_min": -0.2284674048423767, "gen/advantages_std": 0.07678545266389847, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.5440489649772644, "gen/reward_std": 0.11048609018325806, "grad_norm": 0.003351012710481882, "learning_rate": 1e-05, "loss": 0.0043, "rewards/perceptual_score_reward_func/mean": 0.5470424294471741, "rewards/perceptual_score_reward_func/std": 0.11494408547878265, "step": 267, "time_profile/curr_logprobs_and_update": 0.29731816417188384, "time_profile/image_rollout": 90.62626018188894, "time_profile/old_logps": 35.7683735601604, "time_profile/ref_logps": 35.837917340919375, "time_profile/reward": 1.1168938986957073, "train/gen_step": 16908.0, "unified/clip_ratio/high_mean": 0.0213104724916775, "unified/clip_ratio/low_mean": 0.0021109240315126954, "unified/clip_ratio/region_mean": 0.023421396685080254, "unified/kl": 0.1302080180030316, "unified/loss": 0.0008558859917684458 }, { "epoch": 3.397117694155324, "grad_norm": 0.004734479822218418, "learning_rate": 1e-05, "loss": 0.0051, "step": 268, "time_profile/curr_logprobs_and_update": 0.2901776253420394, "train/gen_step": 16972.0, "unified/clip_ratio/high_mean": 0.0021853026237295126, "unified/clip_ratio/low_mean": 8.935081768868258e-05, "unified/clip_ratio/region_mean": 0.0022746534550606157, "unified/kl": 0.12629019923042506, "unified/loss": 0.008493884124163742 }, { "epoch": 3.409927942353883, "gen/advantages_abs_mean": 0.07885642349720001, "gen/advantages_max": 0.27771949768066406, "gen/advantages_mean": 0.0, "gen/advantages_min": -0.27976107597351074, "gen/advantages_std": 0.0983208417892456, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 28.0, "gen/prompt/mean_length": 26.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.31731441617012024, "gen/reward_std": 0.1525343656539917, "grad_norm": 0.005082281306385994, "learning_rate": 1e-05, "loss": 0.0065, "rewards/perceptual_score_reward_func/mean": 0.3259761929512024, "rewards/perceptual_score_reward_func/std": 0.14577125012874603, "step": 269, "time_profile/curr_logprobs_and_update": 0.28629041809472255, "time_profile/image_rollout": 95.50635491497815, "time_profile/old_logps": 35.63145359046757, "time_profile/ref_logps": 35.680125527083874, "time_profile/reward": 1.40938363596797, "train/gen_step": 17036.0, "unified/clip_ratio/high_mean": 0.006612370569200721, "unified/clip_ratio/low_mean": 0.0005135899291417445, "unified/clip_ratio/region_mean": 0.0071259605610975996, "unified/kl": 0.10105247830506414, "unified/loss": 0.009375499214911542 }, { "epoch": 3.422738190552442, "grad_norm": 0.004122909624129534, "learning_rate": 1e-05, "loss": 0.0012, "step": 270, "time_profile/curr_logprobs_and_update": 0.2959313581814058, "train/gen_step": 17100.0, "unified/clip_ratio/high_mean": 1.1556952813407406e-05, "unified/clip_ratio/low_mean": 1.2056327250320464e-05, "unified/clip_ratio/region_mean": 2.361328006372787e-05, "unified/kl": 0.02998922864207998, "unified/loss": 0.0011360755161149427 }, { "epoch": 3.4355484387510007, "gen/advantages_abs_mean": 0.05167783796787262, "gen/advantages_max": 0.2315201759338379, "gen/advantages_mean": 4.656612873077393e-10, "gen/advantages_min": -0.3078737258911133, "gen/advantages_std": 0.06637614965438843, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 140.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.6377387046813965, "gen/reward_std": 0.07187502831220627, "grad_norm": 0.0038060385268181562, "learning_rate": 1e-05, "loss": 0.0005, "rewards/perceptual_score_reward_func/mean": 0.6316916346549988, "rewards/perceptual_score_reward_func/std": 0.06951053440570831, "step": 271, "time_profile/curr_logprobs_and_update": 0.2952291261171922, "time_profile/image_rollout": 92.62281246297061, "time_profile/old_logps": 35.6961055342108, "time_profile/ref_logps": 35.77339258044958, "time_profile/reward": 1.0998948644846678, "train/gen_step": 17164.0, "unified/clip_ratio/high_mean": 7.730922334303614e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 7.730922334303614e-05, "unified/kl": 0.04771494321175851, "unified/loss": 0.0025305816452600993 }, { "epoch": 3.4483586869495597, "grad_norm": 0.0035483732353895903, "learning_rate": 1e-05, "loss": 0.0044, "step": 272, "time_profile/curr_logprobs_and_update": 0.2881234941887669, "train/gen_step": 17228.0, "unified/clip_ratio/high_mean": 0.0009083378363357042, "unified/clip_ratio/low_mean": 2.7453177608549595e-05, "unified/clip_ratio/region_mean": 0.0009357910148537485, "unified/kl": 0.11065650603268296, "unified/loss": 0.004713995716883801 }, { "epoch": 3.4611689351481187, "gen/advantages_abs_mean": 0.04610411450266838, "gen/advantages_max": 0.15632766485214233, "gen/advantages_mean": 2.3283064365386963e-10, "gen/advantages_min": -0.20781588554382324, "gen/advantages_std": 0.058578308671712875, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 32.0, "gen/prompt/mean_length": 31.0, "gen/prompt/min_length": 30.0, "gen/reward": 0.54497230052948, "gen/reward_std": 0.23054379224777222, "grad_norm": 0.004041146952658892, "learning_rate": 1e-05, "loss": 0.0039, "rewards/perceptual_score_reward_func/mean": 0.5383381843566895, "rewards/perceptual_score_reward_func/std": 0.24360765516757965, "step": 273, "time_profile/curr_logprobs_and_update": 0.289339731563814, "time_profile/image_rollout": 90.77092687599361, "time_profile/old_logps": 35.637864058837295, "time_profile/ref_logps": 35.69791112653911, "time_profile/reward": 1.9225932322442532, "train/gen_step": 17292.0, "unified/clip_ratio/high_mean": 0.003151820457787835, "unified/clip_ratio/low_mean": 0.00025966245357267326, "unified/clip_ratio/region_mean": 0.0034114828986275825, "unified/kl": 0.10243399511091411, "unified/loss": 0.004216766497847857 }, { "epoch": 3.4739791833466773, "grad_norm": 0.0035277053248137236, "learning_rate": 1e-05, "loss": 0.0019, "step": 274, "time_profile/curr_logprobs_and_update": 0.2899961236398667, "train/gen_step": 17356.0, "unified/clip_ratio/high_mean": 0.00041052952474274207, "unified/clip_ratio/low_mean": 0.0019760358327403083, "unified/clip_ratio/region_mean": 0.002386565354754566, "unified/kl": 0.04827476281207055, "unified/loss": 0.0019140739777867566 }, { "epoch": 3.4867894315452364, "gen/advantages_abs_mean": 0.04911169782280922, "gen/advantages_max": 0.26427870988845825, "gen/advantages_mean": -4.656612873077393e-10, "gen/advantages_min": -0.2291024774312973, "gen/advantages_std": 0.0661192312836647, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 36.0, "gen/prompt/mean_length": 31.5, "gen/prompt/min_length": 27.0, "gen/reward": 0.5072143077850342, "gen/reward_std": 0.29288727045059204, "grad_norm": 0.0041518728248775005, "learning_rate": 1e-05, "loss": -0.0001, "rewards/perceptual_score_reward_func/mean": 0.5160332918167114, "rewards/perceptual_score_reward_func/std": 0.27702122926712036, "step": 275, "time_profile/curr_logprobs_and_update": 0.2842450351163279, "time_profile/image_rollout": 88.49953482858837, "time_profile/old_logps": 35.63493550941348, "time_profile/ref_logps": 35.67222382687032, "time_profile/reward": 1.4600203577429056, "train/gen_step": 17420.0, "unified/clip_ratio/high_mean": 0.0034367842599749565, "unified/clip_ratio/low_mean": 0.018341819089073397, "unified/clip_ratio/region_mean": 0.021778603123493667, "unified/kl": 0.04001146586961113, "unified/loss": -3.612683156006824e-05 }, { "epoch": 3.499599679743795, "grad_norm": 0.004384258762001991, "learning_rate": 1e-05, "loss": 0.0013, "step": 276, "time_profile/curr_logprobs_and_update": 0.2950630830600858, "train/gen_step": 17484.0, "unified/clip_ratio/high_mean": 0.00029130261464160867, "unified/clip_ratio/low_mean": 5.080641403765185e-05, "unified/clip_ratio/region_mean": 0.0003421090295887552, "unified/kl": 0.0315111142990645, "unified/loss": 0.001208443153814187 }, { "epoch": 3.512409927942354, "gen/advantages_abs_mean": 0.05698845535516739, "gen/advantages_max": 0.16889894008636475, "gen/advantages_mean": 2.561137080192566e-09, "gen/advantages_min": -0.3212686777114868, "gen/advantages_std": 0.07203952968120575, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 185.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.5802289247512817, "gen/reward_std": 0.09428770840167999, "grad_norm": 0.0037689167074859142, "learning_rate": 1e-05, "loss": 0.0034, "rewards/perceptual_score_reward_func/mean": 0.5970678925514221, "rewards/perceptual_score_reward_func/std": 0.08215753734111786, "step": 277, "time_profile/curr_logprobs_and_update": 0.2907980792224407, "time_profile/image_rollout": 95.3401379995048, "time_profile/old_logps": 35.72558932006359, "time_profile/ref_logps": 35.828392228111625, "time_profile/reward": 3.191845502704382, "train/gen_step": 17548.0, "unified/clip_ratio/high_mean": 0.0012386310318106553, "unified/clip_ratio/low_mean": 0.0001734837287585833, "unified/clip_ratio/region_mean": 0.0014121147605692386, "unified/kl": 0.05011636400013231, "unified/loss": 0.0025757047515071463 }, { "epoch": 3.5252201761409125, "grad_norm": 0.0033455120865255594, "learning_rate": 1e-05, "loss": 0.0046, "step": 278, "time_profile/curr_logprobs_and_update": 0.28396442910889164, "train/gen_step": 17612.0, "unified/clip_ratio/high_mean": 0.00014397111044672783, "unified/clip_ratio/low_mean": 3.5590467632573564e-05, "unified/clip_ratio/region_mean": 0.0001795615780793014, "unified/kl": 0.11658623901894316, "unified/loss": 0.004760882653272347 }, { "epoch": 3.5380304243394716, "gen/advantages_abs_mean": 0.06220916286110878, "gen/advantages_max": 0.19965755939483643, "gen/advantages_mean": 3.958120942115784e-09, "gen/advantages_min": -0.2763780355453491, "gen/advantages_std": 0.07711361348628998, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 215.5, "gen/prompt/min_length": 137.0, "gen/reward": 0.5751262903213501, "gen/reward_std": 0.08173548430204391, "grad_norm": 0.0033656784798949957, "learning_rate": 1e-05, "loss": 0.0045, "rewards/perceptual_score_reward_func/mean": 0.5695576667785645, "rewards/perceptual_score_reward_func/std": 0.07363401353359222, "step": 279, "time_profile/curr_logprobs_and_update": 0.2877699746750295, "time_profile/image_rollout": 77.98259479552507, "time_profile/old_logps": 35.70114668272436, "time_profile/ref_logps": 35.775142496451735, "time_profile/reward": 1.3990372698754072, "train/gen_step": 17676.0, "unified/clip_ratio/high_mean": 0.0005160919226909755, "unified/clip_ratio/low_mean": 0.00017728943566908129, "unified/clip_ratio/region_mean": 0.0006933813556315727, "unified/kl": 0.10497432359261438, "unified/loss": 0.002874232982343017 }, { "epoch": 3.5508406725380306, "grad_norm": 0.0034624349791556597, "learning_rate": 1e-05, "loss": 0.0026, "step": 280, "time_profile/curr_logprobs_and_update": 0.2857127044117078, "train/gen_step": 17740.0, "unified/clip_ratio/high_mean": 3.8896130718057975e-05, "unified/clip_ratio/low_mean": 2.6701456590672024e-05, "unified/clip_ratio/region_mean": 6.559758730873e-05, "unified/kl": 0.06620051473146304, "unified/loss": 0.0025284356404426944 }, { "epoch": 3.563650920736589, "gen/advantages_abs_mean": 0.07148271054029465, "gen/advantages_max": 0.29644376039505005, "gen/advantages_mean": 1.3387762010097504e-09, "gen/advantages_min": -0.2896309494972229, "gen/advantages_std": 0.0903797522187233, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 83.5, "gen/prompt/min_length": 24.0, "gen/reward": 0.4678086042404175, "gen/reward_std": 0.24638544023036957, "grad_norm": 0.0029037499334663153, "learning_rate": 1e-05, "loss": 0.0024, "rewards/perceptual_score_reward_func/mean": 0.4608888030052185, "rewards/perceptual_score_reward_func/std": 0.261143296957016, "step": 281, "time_profile/curr_logprobs_and_update": 0.2909189436759334, "time_profile/image_rollout": 95.60725989565253, "time_profile/old_logps": 35.645109789445996, "time_profile/ref_logps": 35.74024346843362, "time_profile/reward": 1.1167160719633102, "train/gen_step": 17804.0, "unified/clip_ratio/high_mean": 7.616222410433693e-05, "unified/clip_ratio/low_mean": 3.8152402339619584e-05, "unified/clip_ratio/region_mean": 0.00011431462644395651, "unified/kl": 0.06412994969286956, "unified/loss": 0.006115963831689442 }, { "epoch": 3.576461168935148, "grad_norm": 0.0027928680647164583, "learning_rate": 1e-05, "loss": 0.0033, "step": 282, "time_profile/curr_logprobs_and_update": 0.2950462262961082, "train/gen_step": 17868.0, "unified/clip_ratio/high_mean": 0.00020154348021605983, "unified/clip_ratio/low_mean": 2.6584201805235352e-05, "unified/clip_ratio/region_mean": 0.00022812768202129519, "unified/kl": 0.08310258729034103, "unified/loss": 0.0033516893672640435 }, { "epoch": 3.5892714171337072, "gen/advantages_abs_mean": 0.07729902863502502, "gen/advantages_max": 0.3134341835975647, "gen/advantages_mean": 2.0372681319713593e-10, "gen/advantages_min": -0.24613593518733978, "gen/advantages_std": 0.09904753416776657, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 31.0, "gen/prompt/mean_length": 28.5, "gen/prompt/min_length": 26.0, "gen/reward": 0.20991452038288116, "gen/reward_std": 0.1033468246459961, "grad_norm": 0.004990903660655022, "learning_rate": 1e-05, "loss": 0.0029, "rewards/perceptual_score_reward_func/mean": 0.19766443967819214, "rewards/perceptual_score_reward_func/std": 0.10781577229499817, "step": 283, "time_profile/curr_logprobs_and_update": 0.28558273872477, "time_profile/image_rollout": 84.52166864462197, "time_profile/old_logps": 35.65723523311317, "time_profile/ref_logps": 35.67234238050878, "time_profile/reward": 1.3341108299791813, "train/gen_step": 17932.0, "unified/clip_ratio/high_mean": 0.0004901750253338832, "unified/clip_ratio/low_mean": 0.0002691970212254091, "unified/clip_ratio/region_mean": 0.0007593720411023241, "unified/kl": 0.07404309092089534, "unified/loss": -0.005812747418531217 }, { "epoch": 3.602081665332266, "grad_norm": 0.007926159538328648, "learning_rate": 1e-05, "loss": 0.0012, "step": 284, "time_profile/curr_logprobs_and_update": 0.2852334416238591, "train/gen_step": 17996.0, "unified/clip_ratio/high_mean": 1.2283804608159699e-05, "unified/clip_ratio/low_mean": 1.4051259313418996e-05, "unified/clip_ratio/region_mean": 2.6335063921578694e-05, "unified/kl": 0.02951611264143139, "unified/loss": 0.0011699605111061828 }, { "epoch": 3.6148919135308244, "gen/advantages_abs_mean": 0.04824373871088028, "gen/advantages_max": 0.16505038738250732, "gen/advantages_mean": -3.4924596548080444e-09, "gen/advantages_min": -0.2584998607635498, "gen/advantages_std": 0.06254779547452927, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 88.5, "gen/prompt/min_length": 34.0, "gen/reward": 0.7271822690963745, "gen/reward_std": 0.0663105696439743, "grad_norm": 0.006434672512114048, "learning_rate": 1e-05, "loss": 0.0023, "rewards/perceptual_score_reward_func/mean": 0.7265818119049072, "rewards/perceptual_score_reward_func/std": 0.06613854318857193, "step": 285, "time_profile/curr_logprobs_and_update": 0.2933608774037566, "time_profile/image_rollout": 88.68789251707494, "time_profile/old_logps": 35.656789338216186, "time_profile/ref_logps": 35.682189056649804, "time_profile/reward": 1.2701308466494083, "train/gen_step": 18060.0, "unified/clip_ratio/high_mean": 0.00017991956792684505, "unified/clip_ratio/low_mean": 1.3469827536027879e-05, "unified/clip_ratio/region_mean": 0.00019338939546287293, "unified/kl": 0.047669891180703416, "unified/loss": 0.011713605775639735 }, { "epoch": 3.6277021617293834, "grad_norm": 0.0027151338290423155, "learning_rate": 1e-05, "loss": 0.0044, "step": 286, "time_profile/curr_logprobs_and_update": 0.2846964886994101, "train/gen_step": 18124.0, "unified/clip_ratio/high_mean": 0.000592122969464981, "unified/clip_ratio/low_mean": 0.0002473404074407881, "unified/clip_ratio/region_mean": 0.0008394633778152638, "unified/kl": 0.1111522851861082, "unified/loss": 0.004183867895335425 }, { "epoch": 3.6405124099279424, "gen/advantages_abs_mean": 0.0678078830242157, "gen/advantages_max": 0.21553650498390198, "gen/advantages_mean": -4.656612873077393e-10, "gen/advantages_min": -0.2846578359603882, "gen/advantages_std": 0.0855150818824768, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 218.5, "gen/prompt/min_length": 143.0, "gen/reward": 0.39473026990890503, "gen/reward_std": 0.24421042203903198, "grad_norm": 0.0041110278107225895, "learning_rate": 1e-05, "loss": 0.0054, "rewards/perceptual_score_reward_func/mean": 0.4020701050758362, "rewards/perceptual_score_reward_func/std": 0.24086683988571167, "step": 287, "time_profile/curr_logprobs_and_update": 0.28402337976149283, "time_profile/image_rollout": 79.04989433661103, "time_profile/old_logps": 35.72478462010622, "time_profile/ref_logps": 35.819537691771984, "time_profile/reward": 0.9356822613626719, "train/gen_step": 18188.0, "unified/clip_ratio/high_mean": 0.0011436262520874152, "unified/clip_ratio/low_mean": 0.00047107959471759386, "unified/clip_ratio/region_mean": 0.0016147058449860197, "unified/kl": 0.12687812180956826, "unified/loss": 0.00387902719558042 }, { "epoch": 3.653322658126501, "grad_norm": 0.004605242982506752, "learning_rate": 1e-05, "loss": 0.0046, "step": 288, "time_profile/curr_logprobs_and_update": 0.2842605091573205, "train/gen_step": 18252.0, "unified/clip_ratio/high_mean": 0.0002273049494760926, "unified/clip_ratio/low_mean": 7.214290326373884e-05, "unified/clip_ratio/region_mean": 0.00029944785183033673, "unified/kl": 0.1161597307655029, "unified/loss": 0.004344568809756311 }, { "epoch": 3.66613290632506, "gen/advantages_abs_mean": 0.06362708657979965, "gen/advantages_max": 0.24109137058258057, "gen/advantages_mean": 1.6298145055770874e-09, "gen/advantages_min": -0.26409316062927246, "gen/advantages_std": 0.07972931861877441, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 137.0, "gen/prompt/mean_length": 137.0, "gen/prompt/min_length": 137.0, "gen/reward": 0.36953791975975037, "gen/reward_std": 0.18224549293518066, "grad_norm": 0.005123942159116268, "learning_rate": 1e-05, "loss": 0.0054, "rewards/perceptual_score_reward_func/mean": 0.35553839802742004, "rewards/perceptual_score_reward_func/std": 0.17481675744056702, "step": 289, "time_profile/curr_logprobs_and_update": 0.29193573028896935, "time_profile/image_rollout": 85.58525138907135, "time_profile/old_logps": 35.699933636933565, "time_profile/ref_logps": 35.74023015238345, "time_profile/reward": 1.5181463621556759, "train/gen_step": 18316.0, "unified/clip_ratio/high_mean": 0.01066688062655885, "unified/clip_ratio/low_mean": 0.0003187119891663315, "unified/clip_ratio/region_mean": 0.010985592674842337, "unified/kl": 0.10665562597569078, "unified/loss": 0.005077178055216791 }, { "epoch": 3.678943154523619, "grad_norm": 0.004347388166934252, "learning_rate": 1e-05, "loss": 0.0027, "step": 290, "time_profile/curr_logprobs_and_update": 0.2874660540255718, "train/gen_step": 18380.0, "unified/clip_ratio/high_mean": 0.00016701010554243112, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.00016701010554243112, "unified/kl": 0.06843785307137296, "unified/loss": 0.0026681148301577196 }, { "epoch": 3.6917534027221777, "gen/advantages_abs_mean": 0.023959465324878693, "gen/advantages_max": 0.1302698850631714, "gen/advantages_mean": -6.984919309616089e-10, "gen/advantages_min": -0.21591639518737793, "gen/advantages_std": 0.042961977422237396, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.453125, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 90.5, "gen/prompt/min_length": 38.0, "gen/reward": 0.3222031593322754, "gen/reward_std": 0.3252832293510437, "grad_norm": 0.0031189958099275827, "learning_rate": 1e-05, "loss": 0.0017, "rewards/perceptual_score_reward_func/mean": 0.3289386034011841, "rewards/perceptual_score_reward_func/std": 0.3337442874908447, "step": 291, "time_profile/curr_logprobs_and_update": 0.28930792823666707, "time_profile/image_rollout": 89.22134350053966, "time_profile/old_logps": 35.68188642151654, "time_profile/ref_logps": 35.70580958202481, "time_profile/reward": 1.0724818594753742, "train/gen_step": 18444.0, "unified/clip_ratio/high_mean": 0.006382234801094455, "unified/clip_ratio/low_mean": 0.0002502836205167114, "unified/clip_ratio/region_mean": 0.006632518420701672, "unified/kl": 0.07385592849459499, "unified/loss": 0.00274933292712376 }, { "epoch": 3.7045636509207367, "grad_norm": 0.0009837422985583544, "learning_rate": 1e-05, "loss": 0.0035, "step": 292, "time_profile/curr_logprobs_and_update": 0.2856500959896948, "train/gen_step": 18508.0, "unified/clip_ratio/high_mean": 0.0016065689296738128, "unified/clip_ratio/low_mean": 5.288676311465679e-05, "unified/clip_ratio/region_mean": 0.001659455691878975, "unified/kl": 0.0873052715905942, "unified/loss": 0.0035271293800178682 }, { "epoch": 3.7173738991192953, "gen/advantages_abs_mean": 0.032585710287094116, "gen/advantages_max": 0.1677337884902954, "gen/advantages_mean": -2.3283064365386963e-09, "gen/advantages_min": -0.19779980182647705, "gen/advantages_std": 0.052031490951776505, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.21875, "gen/prompt/max_length": 228.0, "gen/prompt/mean_length": 126.5, "gen/prompt/min_length": 25.0, "gen/reward": 0.3170202672481537, "gen/reward_std": 0.3172413110733032, "grad_norm": 0.0009276257478632033, "learning_rate": 1e-05, "loss": 0.0036, "rewards/perceptual_score_reward_func/mean": 0.33117929100990295, "rewards/perceptual_score_reward_func/std": 0.32941934466362, "step": 293, "time_profile/curr_logprobs_and_update": 0.2861288280109875, "time_profile/image_rollout": 88.20444161072373, "time_profile/old_logps": 35.684086905792356, "time_profile/ref_logps": 35.73149266280234, "time_profile/reward": 1.3472717106342316, "train/gen_step": 18572.0, "unified/clip_ratio/high_mean": 0.00624829082062206, "unified/clip_ratio/low_mean": 0.00016511899048055056, "unified/clip_ratio/region_mean": 0.0064134098020076635, "unified/kl": 0.07742297963704914, "unified/loss": 0.003311360438601696 }, { "epoch": 3.7301841473178543, "grad_norm": 0.0010312836384400725, "learning_rate": 1e-05, "loss": 0.002, "step": 294, "time_profile/curr_logprobs_and_update": 0.28893115191021934, "train/gen_step": 18636.0, "unified/clip_ratio/high_mean": 0.00042864468014158774, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.00042864468014158774, "unified/kl": 0.051507175172446296, "unified/loss": 0.0020400361800057 }, { "epoch": 3.742994395516413, "gen/advantages_abs_mean": 0.06168805807828903, "gen/advantages_max": 0.2514798939228058, "gen/advantages_mean": -1.6298145055770874e-09, "gen/advantages_min": -0.31687089800834656, "gen/advantages_std": 0.08253538608551025, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 294.0, "gen/prompt/mean_length": 165.5, "gen/prompt/min_length": 37.0, "gen/reward": 0.6195996403694153, "gen/reward_std": 0.17864473164081573, "grad_norm": 0.001671893522143364, "learning_rate": 1e-05, "loss": 0.0016, "rewards/perceptual_score_reward_func/mean": 0.6047117114067078, "rewards/perceptual_score_reward_func/std": 0.18849465250968933, "step": 295, "time_profile/curr_logprobs_and_update": 0.2885678320308216, "time_profile/image_rollout": 80.15901150368154, "time_profile/old_logps": 35.669525284320116, "time_profile/ref_logps": 35.711911994963884, "time_profile/reward": 1.5623396970331669, "train/gen_step": 18700.0, "unified/clip_ratio/high_mean": 0.0023336565727731795, "unified/clip_ratio/low_mean": 7.46693549444899e-05, "unified/clip_ratio/region_mean": 0.002408325938631606, "unified/kl": 0.05192991314106621, "unified/loss": 0.00448986226720649 }, { "epoch": 3.755804643714972, "grad_norm": 0.0020732299890369177, "learning_rate": 1e-05, "loss": 0.0021, "step": 296, "time_profile/curr_logprobs_and_update": 0.2906850943691097, "train/gen_step": 18764.0, "unified/clip_ratio/high_mean": 0.0004890980990239768, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 0.0004890980990239768, "unified/kl": 0.05222447955748066, "unified/loss": 0.002113723692673375 }, { "epoch": 3.768614891913531, "gen/advantages_abs_mean": 0.05659784376621246, "gen/advantages_max": 0.30854809284210205, "gen/advantages_mean": 2.9103830456733704e-11, "gen/advantages_min": -0.14766725897789001, "gen/advantages_std": 0.0727003961801529, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 31.0, "gen/prompt/mean_length": 30.0, "gen/prompt/min_length": 29.0, "gen/reward": 0.0630592554807663, "gen/reward_std": 0.07766459882259369, "grad_norm": 0.002387274755164981, "learning_rate": 1e-05, "loss": 0.0021, "rewards/perceptual_score_reward_func/mean": 0.07399304956197739, "rewards/perceptual_score_reward_func/std": 0.07755821943283081, "step": 297, "time_profile/curr_logprobs_and_update": 0.28434587141964585, "time_profile/image_rollout": 93.55125546082854, "time_profile/old_logps": 35.63593592122197, "time_profile/ref_logps": 35.681266540661454, "time_profile/reward": 1.2651427760720253, "train/gen_step": 18828.0, "unified/clip_ratio/high_mean": 0.002428019828585093, "unified/clip_ratio/low_mean": 2.390239114902215e-05, "unified/clip_ratio/region_mean": 0.0024519222188246204, "unified/kl": 0.04881385323824361, "unified/loss": -0.0021698455511796055 }, { "epoch": 3.7814251401120895, "grad_norm": 0.0029673695098608732, "learning_rate": 1e-05, "loss": 0.0016, "step": 298, "time_profile/curr_logprobs_and_update": 0.2842209104564972, "train/gen_step": 18892.0, "unified/clip_ratio/high_mean": 2.6307398002245463e-05, "unified/clip_ratio/low_mean": 0.0, "unified/clip_ratio/region_mean": 2.6307398002245463e-05, "unified/kl": 0.041427473712246865, "unified/loss": 0.0016878378992259968 }, { "epoch": 3.7942353883106485, "gen/advantages_abs_mean": 0.039250317960977554, "gen/advantages_max": 0.1470203995704651, "gen/advantages_mean": 4.656612873077393e-10, "gen/advantages_min": -0.23213261365890503, "gen/advantages_std": 0.05306250602006912, "gen/completion/max_length": 1024.0, "gen/completion/mean_length": 1024.0, "gen/completion/min_length": 1024.0, "gen/frac_reward_zero_std": 0.0, "gen/prompt/max_length": 143.0, "gen/prompt/mean_length": 89.0, "gen/prompt/min_length": 35.0, "gen/reward": 0.7142139673233032, "gen/reward_std": 0.13610295951366425, "grad_norm": 0.0023270256351679564, "learning_rate": 1e-05, "loss": 0.0026, "rewards/perceptual_score_reward_func/mean": 0.7127842903137207, "rewards/perceptual_score_reward_func/std": 0.14060144126415253, "step": 299, "time_profile/curr_logprobs_and_update": 0.28757855866570026, "time_profile/image_rollout": 88.57772912271321, "time_profile/old_logps": 35.677918802946806, "time_profile/ref_logps": 35.72353080287576, "time_profile/reward": 1.1193790771067142, "train/gen_step": 18956.0, "unified/clip_ratio/high_mean": 6.198136179591529e-05, "unified/clip_ratio/low_mean": 1.169535971712321e-05, "unified/clip_ratio/region_mean": 7.36767215130385e-05, "unified/kl": 0.05895602604141459, "unified/loss": 0.0031285112600016873 }, { "epoch": 3.8070456365092076, "grad_norm": 0.001265124068595469, "learning_rate": 1e-05, "loss": 0.0047, "step": 300, "time_profile/curr_logprobs_and_update": 0.2926370550703723, "train/gen_step": 19020.0, "unified/clip_ratio/high_mean": 0.00042617201506800484, "unified/clip_ratio/low_mean": 2.537912587285973e-05, "unified/clip_ratio/region_mean": 0.00045155114094086457, "unified/kl": 0.11658903432544321, "unified/loss": 0.004803504161827732 } ], "logging_steps": 1.0, "max_steps": 790, "num_input_tokens_seen": 0, "num_train_epochs": 10, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 1, "trial_name": null, "trial_params": null }