File size: 77,940 Bytes
f67fae8 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204 1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422 1423 1424 1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681 1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 | {
"best_global_step": null,
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.021367521367521368,
"eval_steps": 500,
"global_step": 50,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.5555555820465088,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2837.0,
"completions/mean_length": 2626.97216796875,
"completions/mean_terminated_length": 2160.6875,
"completions/min_length": 1285.0,
"completions/min_terminated_length": 1285.0,
"entropy": 0.17718915144602457,
"epoch": 0.00042735042735042735,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07994551956653595,
"learning_rate": 0.0,
"loss": 0.0468,
"num_tokens": 102485.0,
"reward": -1.153435230255127,
"reward_std": 1.0195810794830322,
"rewards/hint_following/mean": -0.2777777910232544,
"rewards/hint_following/std": 0.8819171190261841,
"rewards/length_penalty/mean": -0.8756573796272278,
"rewards/length_penalty/std": 0.17644810676574707,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9937517642974854,
"sampling/importance_sampling_ratio/min": 0.16920268535614014,
"sampling/sampling_logp_difference/max": 1.7766579389572144,
"sampling/sampling_logp_difference/mean": 0.02078823931515217,
"step": 1,
"step_time": 67.38753714214545
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.25,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2961.0,
"completions/mean_length": 1809.4444580078125,
"completions/mean_terminated_length": 1412.5926513671875,
"completions/min_length": 858.0,
"completions/min_terminated_length": 858.0,
"entropy": 0.1950028215845426,
"epoch": 0.0008547008547008547,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07918090373277664,
"learning_rate": 3.846153846153847e-06,
"loss": 0.022,
"num_tokens": 177279.0,
"reward": -0.4642592668533325,
"reward_std": 1.043834924697876,
"rewards/hint_following/mean": 0.1388888955116272,
"rewards/hint_following/std": 0.798311710357666,
"rewards/length_penalty/mean": -0.6031482219696045,
"rewards/length_penalty/std": 0.28368300199508667,
"sampling/importance_sampling_ratio/max": 2.0890045166015625,
"sampling/importance_sampling_ratio/mean": 0.9933527708053589,
"sampling/importance_sampling_ratio/min": 0.29880645871162415,
"sampling/sampling_logp_difference/max": 1.2079591751098633,
"sampling/sampling_logp_difference/mean": 0.021868562325835228,
"step": 2,
"step_time": 59.25448451703414
},
{
"clip_ratio/high_max": 0.007054112308348219,
"clip_ratio/high_mean": 0.007054112308348219,
"clip_ratio/low_mean": 0.0024281112127937376,
"clip_ratio/low_min": 0.0024281112127937376,
"clip_ratio/region_mean": 0.009482223695764938,
"completions/clipped_ratio": 0.1666666716337204,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2411.0,
"completions/mean_length": 1750.8055419921875,
"completions/mean_terminated_length": 1500.966796875,
"completions/min_length": 939.0,
"completions/min_terminated_length": 939.0,
"entropy": 0.1983068659901619,
"epoch": 0.001282051282051282,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06695634126663208,
"learning_rate": 7.692307692307694e-06,
"loss": 0.0035,
"num_tokens": 245288.0,
"reward": -0.25026851892471313,
"reward_std": 0.9555131196975708,
"rewards/hint_following/mean": 0.3333333432674408,
"rewards/hint_following/std": 0.7559289932250977,
"rewards/length_penalty/mean": -0.5836018323898315,
"rewards/length_penalty/std": 0.22866536676883698,
"sampling/importance_sampling_ratio/max": 2.208913803100586,
"sampling/importance_sampling_ratio/mean": 0.9931869506835938,
"sampling/importance_sampling_ratio/min": 0.30834895372390747,
"sampling/sampling_logp_difference/max": 1.176523208618164,
"sampling/sampling_logp_difference/mean": 0.022443262860178947,
"step": 3,
"step_time": 55.47524960897863
},
{
"clip_ratio/high_max": 0.0025329499427850046,
"clip_ratio/high_mean": 0.0025329499427850046,
"clip_ratio/low_mean": 0.0070530143566429615,
"clip_ratio/low_min": 0.0070530143566429615,
"clip_ratio/region_mean": 0.009585964183012644,
"completions/clipped_ratio": 0.472222238779068,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2857.0,
"completions/mean_length": 2469.27783203125,
"completions/mean_terminated_length": 1994.4210205078125,
"completions/min_length": 627.0,
"completions/min_terminated_length": 627.0,
"entropy": 0.1949674834807714,
"epoch": 0.0017094017094017094,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.08172055333852768,
"learning_rate": 1.153846153846154e-05,
"loss": 0.0355,
"num_tokens": 339768.0,
"reward": -1.0453147888183594,
"reward_std": 0.8787956237792969,
"rewards/hint_following/mean": -0.2222222238779068,
"rewards/hint_following/std": 0.7601169347763062,
"rewards/length_penalty/mean": -0.8230926394462585,
"rewards/length_penalty/std": 0.2654260993003845,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.993114173412323,
"sampling/importance_sampling_ratio/min": 0.19437530636787415,
"sampling/sampling_logp_difference/max": 1.6379644870758057,
"sampling/sampling_logp_difference/mean": 0.022510718554258347,
"step": 4,
"step_time": 62.77226335310843
},
{
"clip_ratio/high_max": 0.001851275311006854,
"clip_ratio/high_mean": 0.001851275311006854,
"clip_ratio/low_mean": 0.00705820966201524,
"clip_ratio/low_min": 0.00705820966201524,
"clip_ratio/region_mean": 0.008909484914814433,
"completions/clipped_ratio": 0.4166666567325592,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2915.0,
"completions/mean_length": 2362.8056640625,
"completions/mean_terminated_length": 1907.666748046875,
"completions/min_length": 1109.0,
"completions/min_terminated_length": 1109.0,
"entropy": 0.18256587783495584,
"epoch": 0.002136752136752137,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.072978176176548,
"learning_rate": 1.5384615384615387e-05,
"loss": 0.1055,
"num_tokens": 431687.0,
"reward": -0.9264907836914062,
"reward_std": 1.0278923511505127,
"rewards/hint_following/mean": -0.1388888955116272,
"rewards/hint_following/std": 0.8333333730697632,
"rewards/length_penalty/mean": -0.787601888179779,
"rewards/length_penalty/std": 0.2262839674949646,
"sampling/importance_sampling_ratio/max": 2.5575737953186035,
"sampling/importance_sampling_ratio/mean": 0.9932507872581482,
"sampling/importance_sampling_ratio/min": 0.2952674627304077,
"sampling/sampling_logp_difference/max": 1.2198736667633057,
"sampling/sampling_logp_difference/mean": 0.020803121849894524,
"step": 5,
"step_time": 61.06088067602832
},
{
"clip_ratio/high_max": 0.0030960943549871445,
"clip_ratio/high_mean": 0.0030960943549871445,
"clip_ratio/low_mean": 0.006137795591106017,
"clip_ratio/low_min": 0.006137795591106017,
"clip_ratio/region_mean": 0.009233890101313591,
"completions/clipped_ratio": 0.4166666567325592,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2929.0,
"completions/mean_length": 2417.444580078125,
"completions/mean_terminated_length": 2001.3333740234375,
"completions/min_length": 1434.0,
"completions/min_terminated_length": 1434.0,
"entropy": 0.1919809157649676,
"epoch": 0.002564102564102564,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07703683525323868,
"learning_rate": 1.923076923076923e-05,
"loss": 0.1019,
"num_tokens": 525555.0,
"reward": -0.7224815487861633,
"reward_std": 1.1388682126998901,
"rewards/hint_following/mean": 0.0833333358168602,
"rewards/hint_following/std": 0.9673232436180115,
"rewards/length_penalty/mean": -0.8058148622512817,
"rewards/length_penalty/std": 0.18899519741535187,
"sampling/importance_sampling_ratio/max": 2.9181816577911377,
"sampling/importance_sampling_ratio/mean": 0.9935172200202942,
"sampling/importance_sampling_ratio/min": 0.14165793359279633,
"sampling/sampling_logp_difference/max": 1.9543399810791016,
"sampling/sampling_logp_difference/mean": 0.021658122539520264,
"step": 6,
"step_time": 64.93633897381369
},
{
"clip_ratio/high_max": 0.004218692930104832,
"clip_ratio/high_mean": 0.004218692930104832,
"clip_ratio/low_mean": 0.0048820926652600365,
"clip_ratio/low_min": 0.0048820926652600365,
"clip_ratio/region_mean": 0.009100785556559762,
"completions/clipped_ratio": 0.2222222238779068,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2933.0,
"completions/mean_length": 2029.0833740234375,
"completions/mean_terminated_length": 1751.6785888671875,
"completions/min_length": 1009.0,
"completions/min_terminated_length": 1009.0,
"entropy": 0.20896865924199423,
"epoch": 0.0029914529914529917,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07187572866678238,
"learning_rate": 2.307692307692308e-05,
"loss": 0.0798,
"num_tokens": 606186.0,
"reward": -0.4819166362285614,
"reward_std": 0.9970327019691467,
"rewards/hint_following/mean": 0.1944444477558136,
"rewards/hint_following/std": 0.7862913012504578,
"rewards/length_penalty/mean": -0.676361083984375,
"rewards/length_penalty/std": 0.2366827428340912,
"sampling/importance_sampling_ratio/max": 2.6964023113250732,
"sampling/importance_sampling_ratio/mean": 0.9927431344985962,
"sampling/importance_sampling_ratio/min": 0.21346372365951538,
"sampling/sampling_logp_difference/max": 1.5442883968353271,
"sampling/sampling_logp_difference/mean": 0.022996528074145317,
"step": 7,
"step_time": 59.43198294797912
},
{
"clip_ratio/high_max": 0.007654782539854447,
"clip_ratio/high_mean": 0.007654782539854447,
"clip_ratio/low_mean": 0.002519401100774606,
"clip_ratio/low_min": 0.002519401100774606,
"clip_ratio/region_mean": 0.010174183640629053,
"completions/clipped_ratio": 0.1666666716337204,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2957.0,
"completions/mean_length": 2227.97216796875,
"completions/mean_terminated_length": 2073.56689453125,
"completions/min_length": 1197.0,
"completions/min_terminated_length": 1197.0,
"entropy": 0.19751630226771036,
"epoch": 0.003418803418803419,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.08499564230442047,
"learning_rate": 2.6923076923076923e-05,
"loss": 0.0572,
"num_tokens": 692279.0,
"reward": -0.4093240797519684,
"reward_std": 0.8091796040534973,
"rewards/hint_following/mean": 0.3333333432674408,
"rewards/hint_following/std": 0.7171371579170227,
"rewards/length_penalty/mean": -0.7426574230194092,
"rewards/length_penalty/std": 0.19112928211688995,
"sampling/importance_sampling_ratio/max": 2.491806745529175,
"sampling/importance_sampling_ratio/mean": 0.9929631352424622,
"sampling/importance_sampling_ratio/min": 0.3287244141101837,
"sampling/sampling_logp_difference/max": 1.1125354766845703,
"sampling/sampling_logp_difference/mean": 0.021952621638774872,
"step": 8,
"step_time": 60.087568536167964
},
{
"clip_ratio/high_max": 0.007754642438764374,
"clip_ratio/high_mean": 0.007754642438764374,
"clip_ratio/low_mean": 0.0032427439582534134,
"clip_ratio/low_min": 0.0032427439582534134,
"clip_ratio/region_mean": 0.010997386649250984,
"completions/clipped_ratio": 0.1111111119389534,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2948.0,
"completions/mean_length": 2034.2222900390625,
"completions/mean_terminated_length": 1913.5,
"completions/min_length": 778.0,
"completions/min_terminated_length": 778.0,
"entropy": 0.21248381833235422,
"epoch": 0.0038461538461538464,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07896988093852997,
"learning_rate": 3.0769230769230774e-05,
"loss": 0.0952,
"num_tokens": 771241.0,
"reward": -0.28918519616127014,
"reward_std": 0.7873722314834595,
"rewards/hint_following/mean": 0.3888888955116272,
"rewards/hint_following/std": 0.6877615451812744,
"rewards/length_penalty/mean": -0.6780741214752197,
"rewards/length_penalty/std": 0.22194787859916687,
"sampling/importance_sampling_ratio/max": 2.757654905319214,
"sampling/importance_sampling_ratio/mean": 0.9926807880401611,
"sampling/importance_sampling_ratio/min": 0.19047939777374268,
"sampling/sampling_logp_difference/max": 1.6582112312316895,
"sampling/sampling_logp_difference/mean": 0.02367428131401539,
"step": 9,
"step_time": 58.2687593010487
},
{
"clip_ratio/high_max": 0.0022279491822700948,
"clip_ratio/high_mean": 0.0022279491822700948,
"clip_ratio/low_mean": 0.0019324645400047302,
"clip_ratio/low_min": 0.0019324645400047302,
"clip_ratio/region_mean": 0.004160413751378655,
"completions/clipped_ratio": 0.3611111044883728,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2960.0,
"completions/mean_length": 2266.111083984375,
"completions/mean_terminated_length": 1851.3043212890625,
"completions/min_length": 784.0,
"completions/min_terminated_length": 784.0,
"entropy": 0.20578551292419434,
"epoch": 0.004273504273504274,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06615690886974335,
"learning_rate": 3.461538461538462e-05,
"loss": 0.0543,
"num_tokens": 858521.0,
"reward": -0.5331481099128723,
"reward_std": 1.1243869066238403,
"rewards/hint_following/mean": 0.2222222238779068,
"rewards/hint_following/std": 0.929242730140686,
"rewards/length_penalty/mean": -0.7553703784942627,
"rewards/length_penalty/std": 0.23448802530765533,
"sampling/importance_sampling_ratio/max": 2.0015952587127686,
"sampling/importance_sampling_ratio/mean": 0.9925298094749451,
"sampling/importance_sampling_ratio/min": 0.30108708143234253,
"sampling/sampling_logp_difference/max": 1.2003557682037354,
"sampling/sampling_logp_difference/mean": 0.021938424557447433,
"step": 10,
"step_time": 60.113012868212536
},
{
"clip_ratio/high_max": 0.002628828883947184,
"clip_ratio/high_mean": 0.002628828883947184,
"clip_ratio/low_mean": 0.0007913654941755036,
"clip_ratio/low_min": 0.0007913654941755036,
"clip_ratio/region_mean": 0.0034201944169277945,
"completions/clipped_ratio": 0.3888888955116272,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2856.0,
"completions/mean_length": 2308.111083984375,
"completions/mean_terminated_length": 1867.8182373046875,
"completions/min_length": 853.0,
"completions/min_terminated_length": 853.0,
"entropy": 0.1780667081475258,
"epoch": 0.004700854700854701,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06938029080629349,
"learning_rate": 3.846153846153846e-05,
"loss": 0.0506,
"num_tokens": 948171.0,
"reward": -1.0471482276916504,
"reward_std": 0.824184238910675,
"rewards/hint_following/mean": -0.2777777910232544,
"rewards/hint_following/std": 0.6594851613044739,
"rewards/length_penalty/mean": -0.7693703770637512,
"rewards/length_penalty/std": 0.24463628232479095,
"sampling/importance_sampling_ratio/max": 2.5170931816101074,
"sampling/importance_sampling_ratio/mean": 0.9934613108634949,
"sampling/importance_sampling_ratio/min": 0.29526859521865845,
"sampling/sampling_logp_difference/max": 1.21986985206604,
"sampling/sampling_logp_difference/mean": 0.021038096398115158,
"step": 11,
"step_time": 61.20216279185843
},
{
"clip_ratio/high_max": 0.009717889750997225,
"clip_ratio/high_mean": 0.009717889750997225,
"clip_ratio/low_mean": 0.001428690991209199,
"clip_ratio/low_min": 0.001428690991209199,
"clip_ratio/region_mean": 0.011146580955634514,
"completions/clipped_ratio": 0.2777777910232544,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2997.0,
"completions/mean_length": 2494.944580078125,
"completions/mean_terminated_length": 2300.6923828125,
"completions/min_length": 1195.0,
"completions/min_terminated_length": 1195.0,
"entropy": 0.21741948276758194,
"epoch": 0.005128205128205128,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.08027122914791107,
"learning_rate": 4.230769230769231e-05,
"loss": 0.0271,
"num_tokens": 1045165.0,
"reward": -0.6094259023666382,
"reward_std": 0.9179477691650391,
"rewards/hint_following/mean": 0.2222222238779068,
"rewards/hint_following/std": 0.8655670881271362,
"rewards/length_penalty/mean": -0.8316481113433838,
"rewards/length_penalty/std": 0.1910414844751358,
"sampling/importance_sampling_ratio/max": 2.7227704524993896,
"sampling/importance_sampling_ratio/mean": 0.9923471808433533,
"sampling/importance_sampling_ratio/min": 0.309160441160202,
"sampling/sampling_logp_difference/max": 1.1738948822021484,
"sampling/sampling_logp_difference/mean": 0.02328832633793354,
"step": 12,
"step_time": 63.202891704044305
},
{
"clip_ratio/high_max": 0.0020594243348265686,
"clip_ratio/high_mean": 0.0020594243348265686,
"clip_ratio/low_mean": 0.0059360921538124485,
"clip_ratio/low_min": 0.0059360921538124485,
"clip_ratio/region_mean": 0.007995516527444124,
"completions/clipped_ratio": 0.2222222238779068,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2981.0,
"completions/mean_length": 2085.638916015625,
"completions/mean_terminated_length": 1824.3929443359375,
"completions/min_length": 956.0,
"completions/min_terminated_length": 956.0,
"entropy": 0.18983352681001028,
"epoch": 0.005555555555555556,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.2806412875652313,
"learning_rate": 4.615384615384616e-05,
"loss": 0.0294,
"num_tokens": 1128414.0,
"reward": -0.6674352288246155,
"reward_std": 0.8722909092903137,
"rewards/hint_following/mean": 0.02777777798473835,
"rewards/hint_following/std": 0.6963624954223633,
"rewards/length_penalty/mean": -0.6952130198478699,
"rewards/length_penalty/std": 0.23719921708106995,
"sampling/importance_sampling_ratio/max": 2.7140581607818604,
"sampling/importance_sampling_ratio/mean": 0.9935481548309326,
"sampling/importance_sampling_ratio/min": 0.19567115604877472,
"sampling/sampling_logp_difference/max": 1.6313197612762451,
"sampling/sampling_logp_difference/mean": 0.02119005285203457,
"step": 13,
"step_time": 61.55426321423147
},
{
"clip_ratio/high_max": 0.0014092368267786999,
"clip_ratio/high_mean": 0.0014092368267786999,
"clip_ratio/low_mean": 0.007215868448838592,
"clip_ratio/low_min": 0.007215868448838592,
"clip_ratio/region_mean": 0.008625105411435166,
"completions/clipped_ratio": 0.1944444477558136,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2941.0,
"completions/mean_length": 1680.3055419921875,
"completions/mean_terminated_length": 1361.7586669921875,
"completions/min_length": 398.0,
"completions/min_terminated_length": 398.0,
"entropy": 0.20419775694608688,
"epoch": 0.005982905982905983,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.08586689084768295,
"learning_rate": 5e-05,
"loss": 0.0681,
"num_tokens": 1193759.0,
"reward": -0.5878795981407166,
"reward_std": 0.8454972505569458,
"rewards/hint_following/mean": -0.02777777798473835,
"rewards/hint_following/std": 0.6087979674339294,
"rewards/length_penalty/mean": -0.5601018071174622,
"rewards/length_penalty/std": 0.31150683760643005,
"sampling/importance_sampling_ratio/max": 2.1082680225372314,
"sampling/importance_sampling_ratio/mean": 0.992834746837616,
"sampling/importance_sampling_ratio/min": 0.3020838499069214,
"sampling/sampling_logp_difference/max": 1.1970505714416504,
"sampling/sampling_logp_difference/mean": 0.02261131815612316,
"step": 14,
"step_time": 53.72449496493209
},
{
"clip_ratio/high_max": 0.007441268069669604,
"clip_ratio/high_mean": 0.007441268069669604,
"clip_ratio/low_mean": 0.002903422418360909,
"clip_ratio/low_min": 0.002903422418360909,
"clip_ratio/region_mean": 0.01034469079847137,
"completions/clipped_ratio": 0.25,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2906.0,
"completions/mean_length": 2191.111083984375,
"completions/mean_terminated_length": 1921.4814453125,
"completions/min_length": 397.0,
"completions/min_terminated_length": 397.0,
"entropy": 0.22065167874097824,
"epoch": 0.00641025641025641,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07726756483316422,
"learning_rate": 5e-05,
"loss": 0.0177,
"num_tokens": 1279125.0,
"reward": -0.7859259247779846,
"reward_std": 0.7896111607551575,
"rewards/hint_following/mean": -0.0555555559694767,
"rewards/hint_following/std": 0.6737716794013977,
"rewards/length_penalty/mean": -0.7303703427314758,
"rewards/length_penalty/std": 0.2772618532180786,
"sampling/importance_sampling_ratio/max": 2.1340627670288086,
"sampling/importance_sampling_ratio/mean": 0.9917824268341064,
"sampling/importance_sampling_ratio/min": 0.07570598274469376,
"sampling/sampling_logp_difference/max": 2.5808980464935303,
"sampling/sampling_logp_difference/mean": 0.023916445672512054,
"step": 15,
"step_time": 59.183635055902414
},
{
"clip_ratio/high_max": 0.007959824210653702,
"clip_ratio/high_mean": 0.007959824210653702,
"clip_ratio/low_mean": 0.0026605394668877125,
"clip_ratio/low_min": 0.0026605394668877125,
"clip_ratio/region_mean": 0.010620363832761845,
"completions/clipped_ratio": 0.1388888955116272,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2836.0,
"completions/mean_length": 1801.9722900390625,
"completions/mean_terminated_length": 1612.0,
"completions/min_length": 660.0,
"completions/min_terminated_length": 660.0,
"entropy": 0.2382892519235611,
"epoch": 0.006837606837606838,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07441020756959915,
"learning_rate": 5e-05,
"loss": 0.1035,
"num_tokens": 1351580.0,
"reward": -0.2951018214225769,
"reward_std": 0.9211878180503845,
"rewards/hint_following/mean": 0.3055555522441864,
"rewards/hint_following/std": 0.7099072337150574,
"rewards/length_penalty/mean": -0.6006573438644409,
"rewards/length_penalty/std": 0.2538762390613556,
"sampling/importance_sampling_ratio/max": 2.141491413116455,
"sampling/importance_sampling_ratio/mean": 0.9912582039833069,
"sampling/importance_sampling_ratio/min": 0.3961084187030792,
"sampling/sampling_logp_difference/max": 0.9260673522949219,
"sampling/sampling_logp_difference/mean": 0.024969136342406273,
"step": 16,
"step_time": 58.414970112848096
},
{
"clip_ratio/high_max": 0.00782025900358955,
"clip_ratio/high_mean": 0.00782025900358955,
"clip_ratio/low_mean": 0.0016357484661663573,
"clip_ratio/low_min": 0.0016357484661663573,
"clip_ratio/region_mean": 0.00945600758617123,
"completions/clipped_ratio": 0.0833333358168602,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2959.0,
"completions/mean_length": 1858.02783203125,
"completions/mean_terminated_length": 1754.212158203125,
"completions/min_length": 523.0,
"completions/min_terminated_length": 523.0,
"entropy": 0.2335476577281952,
"epoch": 0.007264957264957265,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06890398263931274,
"learning_rate": 5e-05,
"loss": 0.0923,
"num_tokens": 1425669.0,
"reward": 0.10287962108850479,
"reward_std": 0.7695233225822449,
"rewards/hint_following/mean": 0.7222222089767456,
"rewards/hint_following/std": 0.6146363019943237,
"rewards/length_penalty/mean": -0.619342565536499,
"rewards/length_penalty/std": 0.2540966868400574,
"sampling/importance_sampling_ratio/max": 2.8489930629730225,
"sampling/importance_sampling_ratio/mean": 0.9912996888160706,
"sampling/importance_sampling_ratio/min": 0.3177354037761688,
"sampling/sampling_logp_difference/max": 1.1465363502502441,
"sampling/sampling_logp_difference/mean": 0.0239370446652174,
"step": 17,
"step_time": 56.41279660095461
},
{
"clip_ratio/high_max": 0.008962325053289533,
"clip_ratio/high_mean": 0.008962325053289533,
"clip_ratio/low_mean": 0.001638414493451516,
"clip_ratio/low_min": 0.001638414493451516,
"clip_ratio/region_mean": 0.010600739469130835,
"completions/clipped_ratio": 0.2222222238779068,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2681.0,
"completions/mean_length": 1895.4722900390625,
"completions/mean_terminated_length": 1579.8929443359375,
"completions/min_length": 409.0,
"completions/min_terminated_length": 409.0,
"entropy": 0.2237540160616239,
"epoch": 0.007692307692307693,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07782972604036331,
"learning_rate": 5e-05,
"loss": 0.0547,
"num_tokens": 1501340.0,
"reward": -0.3818241059780121,
"reward_std": 0.9488492608070374,
"rewards/hint_following/mean": 0.25,
"rewards/hint_following/std": 0.8062257766723633,
"rewards/length_penalty/mean": -0.6318240761756897,
"rewards/length_penalty/std": 0.2734360098838806,
"sampling/importance_sampling_ratio/max": 2.2928225994110107,
"sampling/importance_sampling_ratio/mean": 0.9916783571243286,
"sampling/importance_sampling_ratio/min": 0.3001212775707245,
"sampling/sampling_logp_difference/max": 1.203568696975708,
"sampling/sampling_logp_difference/mean": 0.023737527430057526,
"step": 18,
"step_time": 58.57279848307371
},
{
"clip_ratio/high_max": 0.0038812818626562753,
"clip_ratio/high_mean": 0.0038812818626562753,
"clip_ratio/low_mean": 0.005094039059864978,
"clip_ratio/low_min": 0.005094039059864978,
"clip_ratio/region_mean": 0.00897532096132636,
"completions/clipped_ratio": 0.1666666716337204,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2937.0,
"completions/mean_length": 2020.138916015625,
"completions/mean_terminated_length": 1824.166748046875,
"completions/min_length": 765.0,
"completions/min_terminated_length": 765.0,
"entropy": 0.2477496862411499,
"epoch": 0.00811965811965812,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07387188822031021,
"learning_rate": 5e-05,
"loss": 0.0524,
"num_tokens": 1581145.0,
"reward": -0.39560189843177795,
"reward_std": 0.9154050350189209,
"rewards/hint_following/mean": 0.2777777910232544,
"rewards/hint_following/std": 0.7410845756530762,
"rewards/length_penalty/mean": -0.67337965965271,
"rewards/length_penalty/std": 0.24809792637825012,
"sampling/importance_sampling_ratio/max": 2.1414597034454346,
"sampling/importance_sampling_ratio/mean": 0.9909990429878235,
"sampling/importance_sampling_ratio/min": 0.250805139541626,
"sampling/sampling_logp_difference/max": 1.383078932762146,
"sampling/sampling_logp_difference/mean": 0.025834236294031143,
"step": 19,
"step_time": 58.03654656501021
},
{
"clip_ratio/high_max": 0.0034701917708540955,
"clip_ratio/high_mean": 0.0034701917708540955,
"clip_ratio/low_mean": 0.004899940686300397,
"clip_ratio/low_min": 0.004899940686300397,
"clip_ratio/region_mean": 0.00837013234073917,
"completions/clipped_ratio": 0.2777777910232544,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2645.0,
"completions/mean_length": 1926.1666259765625,
"completions/mean_terminated_length": 1513.1539306640625,
"completions/min_length": 1111.0,
"completions/min_terminated_length": 1111.0,
"entropy": 0.2643888195355733,
"epoch": 0.008547008547008548,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07248850166797638,
"learning_rate": 5e-05,
"loss": 0.0748,
"num_tokens": 1658131.0,
"reward": -0.25316667556762695,
"reward_std": 1.1320863962173462,
"rewards/hint_following/mean": 0.3888888955116272,
"rewards/hint_following/std": 0.903256893157959,
"rewards/length_penalty/mean": -0.6420555114746094,
"rewards/length_penalty/std": 0.2418719381093979,
"sampling/importance_sampling_ratio/max": 2.639763593673706,
"sampling/importance_sampling_ratio/mean": 0.990202009677887,
"sampling/importance_sampling_ratio/min": 0.3388563394546509,
"sampling/sampling_logp_difference/max": 1.082179069519043,
"sampling/sampling_logp_difference/mean": 0.026784788817167282,
"step": 20,
"step_time": 59.42642685002647
},
{
"clip_ratio/high_max": 0.0018190126866102219,
"clip_ratio/high_mean": 0.0018190126866102219,
"clip_ratio/low_mean": 0.005009224289096892,
"clip_ratio/low_min": 0.005009224289096892,
"clip_ratio/region_mean": 0.006828237092122436,
"completions/clipped_ratio": 0.3611111044883728,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2893.0,
"completions/mean_length": 2087.0556640625,
"completions/mean_terminated_length": 1571.04345703125,
"completions/min_length": 621.0,
"completions/min_terminated_length": 621.0,
"entropy": 0.23874077697594961,
"epoch": 0.008974358974358974,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07083278149366379,
"learning_rate": 5e-05,
"loss": 0.0432,
"num_tokens": 1741971.0,
"reward": -0.7512407302856445,
"reward_std": 1.0566543340682983,
"rewards/hint_following/mean": -0.0555555559694767,
"rewards/hint_following/std": 0.826159656047821,
"rewards/length_penalty/mean": -0.6956851482391357,
"rewards/length_penalty/std": 0.29789718985557556,
"sampling/importance_sampling_ratio/max": 2.2389557361602783,
"sampling/importance_sampling_ratio/mean": 0.9909273386001587,
"sampling/importance_sampling_ratio/min": 0.1903061866760254,
"sampling/sampling_logp_difference/max": 1.659121036529541,
"sampling/sampling_logp_difference/mean": 0.024532614275813103,
"step": 21,
"step_time": 62.53779820702039
},
{
"clip_ratio/high_max": 0.002690806732668231,
"clip_ratio/high_mean": 0.002690806732668231,
"clip_ratio/low_mean": 0.0009356096270494163,
"clip_ratio/low_min": 0.0009356096270494163,
"clip_ratio/region_mean": 0.003626416379120201,
"completions/clipped_ratio": 0.1111111119389534,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2897.0,
"completions/mean_length": 1785.02783203125,
"completions/mean_terminated_length": 1633.15625,
"completions/min_length": 687.0,
"completions/min_terminated_length": 687.0,
"entropy": 0.2271458531419436,
"epoch": 0.009401709401709401,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06376982480287552,
"learning_rate": 5e-05,
"loss": 0.0558,
"num_tokens": 1812208.0,
"reward": -0.12278702855110168,
"reward_std": 0.8545697927474976,
"rewards/hint_following/mean": 0.4722222089767456,
"rewards/hint_following/std": 0.6963624358177185,
"rewards/length_penalty/mean": -0.5950092673301697,
"rewards/length_penalty/std": 0.2556186616420746,
"sampling/importance_sampling_ratio/max": 2.0840954780578613,
"sampling/importance_sampling_ratio/mean": 0.9917669892311096,
"sampling/importance_sampling_ratio/min": 0.4361201226711273,
"sampling/sampling_logp_difference/max": 0.8298375606536865,
"sampling/sampling_logp_difference/mean": 0.02324368618428707,
"step": 22,
"step_time": 54.28889209416229
},
{
"clip_ratio/high_max": 0.00034443634406973916,
"clip_ratio/high_mean": 0.00034443634406973916,
"clip_ratio/low_mean": 0.001613353689511617,
"clip_ratio/low_min": 0.001613353689511617,
"clip_ratio/region_mean": 0.001957789994776249,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2755.0,
"completions/max_terminated_length": 2755.0,
"completions/mean_length": 1515.5555419921875,
"completions/mean_terminated_length": 1515.5555419921875,
"completions/min_length": 761.0,
"completions/min_terminated_length": 761.0,
"entropy": 0.26160697638988495,
"epoch": 0.009829059829059829,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.05521637201309204,
"learning_rate": 5e-05,
"loss": 0.0467,
"num_tokens": 1873452.0,
"reward": 0.18925924599170685,
"reward_std": 0.6213830709457397,
"rewards/hint_following/mean": 0.6944444179534912,
"rewards/hint_following/std": 0.467176616191864,
"rewards/length_penalty/mean": -0.5051851868629456,
"rewards/length_penalty/std": 0.1804182529449463,
"sampling/importance_sampling_ratio/max": 2.6960017681121826,
"sampling/importance_sampling_ratio/mean": 0.9903501272201538,
"sampling/importance_sampling_ratio/min": 0.2622704803943634,
"sampling/sampling_logp_difference/max": 1.33837890625,
"sampling/sampling_logp_difference/mean": 0.026966720819473267,
"step": 23,
"step_time": 49.74234241596423
},
{
"clip_ratio/high_max": 0.009169099697222313,
"clip_ratio/high_mean": 0.009169099697222313,
"clip_ratio/low_mean": 0.0011019935482181609,
"clip_ratio/low_min": 0.0011019935482181609,
"clip_ratio/region_mean": 0.010271093342453241,
"completions/clipped_ratio": 0.02777777798473835,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2438.0,
"completions/mean_length": 1354.77783203125,
"completions/mean_terminated_length": 1307.771484375,
"completions/min_length": 358.0,
"completions/min_terminated_length": 358.0,
"entropy": 0.28231214980284375,
"epoch": 0.010256410256410256,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.061877232044935226,
"learning_rate": 5e-05,
"loss": 0.0513,
"num_tokens": 1928578.0,
"reward": 0.0761851817369461,
"reward_std": 0.6364034414291382,
"rewards/hint_following/mean": 0.5277777910232544,
"rewards/hint_following/std": 0.559903621673584,
"rewards/length_penalty/mean": -0.4515925645828247,
"rewards/length_penalty/std": 0.2236867994070053,
"sampling/importance_sampling_ratio/max": 2.3720901012420654,
"sampling/importance_sampling_ratio/mean": 0.9896314144134521,
"sampling/importance_sampling_ratio/min": 0.3866640329360962,
"sampling/sampling_logp_difference/max": 0.9501991271972656,
"sampling/sampling_logp_difference/mean": 0.028320252895355225,
"step": 24,
"step_time": 51.51962701487355
},
{
"clip_ratio/high_max": 0.007340530709673961,
"clip_ratio/high_mean": 0.007340530709673961,
"clip_ratio/low_mean": 0.002632848802022636,
"clip_ratio/low_min": 0.002632848802022636,
"clip_ratio/region_mean": 0.00997337931767106,
"completions/clipped_ratio": 0.0555555559694767,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2833.0,
"completions/mean_length": 1369.52783203125,
"completions/mean_terminated_length": 1273.61767578125,
"completions/min_length": 489.0,
"completions/min_terminated_length": 489.0,
"entropy": 0.24688356121381125,
"epoch": 0.010683760683760684,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0653451606631279,
"learning_rate": 5e-05,
"loss": 0.0562,
"num_tokens": 1983563.0,
"reward": -0.15095369517803192,
"reward_std": 0.692315936088562,
"rewards/hint_following/mean": 0.3055555522441864,
"rewards/hint_following/std": 0.576662540435791,
"rewards/length_penalty/mean": -0.4565092623233795,
"rewards/length_penalty/std": 0.22569186985492706,
"sampling/importance_sampling_ratio/max": 2.21685791015625,
"sampling/importance_sampling_ratio/mean": 0.9907597899436951,
"sampling/importance_sampling_ratio/min": 0.26015329360961914,
"sampling/sampling_logp_difference/max": 1.3464841842651367,
"sampling/sampling_logp_difference/mean": 0.02600344829261303,
"step": 25,
"step_time": 53.30105395393912
},
{
"clip_ratio/high_max": 0.005985865951515734,
"clip_ratio/high_mean": 0.005985865951515734,
"clip_ratio/low_mean": 0.0026564535413247845,
"clip_ratio/low_min": 0.0026564535413247845,
"clip_ratio/region_mean": 0.008642319512243072,
"completions/clipped_ratio": 0.0833333358168602,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2935.0,
"completions/mean_length": 1874.6944580078125,
"completions/mean_terminated_length": 1772.39404296875,
"completions/min_length": 921.0,
"completions/min_terminated_length": 921.0,
"entropy": 0.24719314028819403,
"epoch": 0.011111111111111112,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07119674980640411,
"learning_rate": 5e-05,
"loss": 0.0874,
"num_tokens": 2058072.0,
"reward": -0.06934259831905365,
"reward_std": 0.8091042637825012,
"rewards/hint_following/mean": 0.5555555820465088,
"rewards/hint_following/std": 0.6522245407104492,
"rewards/length_penalty/mean": -0.6248981952667236,
"rewards/length_penalty/std": 0.21120469272136688,
"sampling/importance_sampling_ratio/max": 2.3882269859313965,
"sampling/importance_sampling_ratio/mean": 0.9909408688545227,
"sampling/importance_sampling_ratio/min": 0.3095887005329132,
"sampling/sampling_logp_difference/max": 1.1725106239318848,
"sampling/sampling_logp_difference/mean": 0.024691704660654068,
"step": 26,
"step_time": 55.06593968113884
},
{
"clip_ratio/high_max": 0.006906990543939173,
"clip_ratio/high_mean": 0.006906990543939173,
"clip_ratio/low_mean": 0.002326829261922588,
"clip_ratio/low_min": 0.002326829261922588,
"clip_ratio/region_mean": 0.009233820019289851,
"completions/clipped_ratio": 0.1388888955116272,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2980.0,
"completions/mean_length": 1767.5833740234375,
"completions/mean_terminated_length": 1568.806396484375,
"completions/min_length": 614.0,
"completions/min_terminated_length": 614.0,
"entropy": 0.25413808474938077,
"epoch": 0.011538461538461539,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06020393222570419,
"learning_rate": 5e-05,
"loss": 0.0678,
"num_tokens": 2126523.0,
"reward": -0.3114166855812073,
"reward_std": 0.8995095491409302,
"rewards/hint_following/mean": 0.2777777910232544,
"rewards/hint_following/std": 0.7014724016189575,
"rewards/length_penalty/mean": -0.5891944169998169,
"rewards/length_penalty/std": 0.24767503142356873,
"sampling/importance_sampling_ratio/max": 2.9181816577911377,
"sampling/importance_sampling_ratio/mean": 0.9907821416854858,
"sampling/importance_sampling_ratio/min": 0.28627336025238037,
"sampling/sampling_logp_difference/max": 1.2508081197738647,
"sampling/sampling_logp_difference/mean": 0.025369901210069656,
"step": 27,
"step_time": 54.32182147912681
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.3333333432674408,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 1529.0,
"completions/mean_length": 1610.5833740234375,
"completions/mean_terminated_length": 915.875,
"completions/min_length": 478.0,
"completions/min_terminated_length": 478.0,
"entropy": 0.19960021724303564,
"epoch": 0.011965811965811967,
"frac_reward_zero_std": 1.0,
"grad_norm": 0.0,
"learning_rate": 5e-05,
"loss": 0.0,
"num_tokens": 2192742.0,
"reward": -0.8701944351196289,
"reward_std": 0.8150903582572937,
"rewards/hint_following/mean": -0.3333333432674408,
"rewards/hint_following/std": 0.47809144854545593,
"rewards/length_penalty/mean": -0.5368611216545105,
"rewards/length_penalty/std": 0.3438311517238617,
"sampling/importance_sampling_ratio/max": 2.0817480087280273,
"sampling/importance_sampling_ratio/mean": 0.9929863214492798,
"sampling/importance_sampling_ratio/min": 0.09024836868047714,
"sampling/sampling_logp_difference/max": 2.4051897525787354,
"sampling/sampling_logp_difference/mean": 0.02168331854045391,
"step": 28,
"step_time": 56.78273802890908
},
{
"clip_ratio/high_max": 0.001870962364288668,
"clip_ratio/high_mean": 0.001870962364288668,
"clip_ratio/low_mean": 0.004373128215471904,
"clip_ratio/low_min": 0.004373128215471904,
"clip_ratio/region_mean": 0.006244090540955464,
"completions/clipped_ratio": 0.1944444477558136,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2978.0,
"completions/mean_length": 1729.77783203125,
"completions/mean_terminated_length": 1423.17236328125,
"completions/min_length": 529.0,
"completions/min_terminated_length": 529.0,
"entropy": 0.28223206599553424,
"epoch": 0.012393162393162393,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07922516763210297,
"learning_rate": 5e-05,
"loss": 0.0378,
"num_tokens": 2262334.0,
"reward": -0.3265925943851471,
"reward_std": 0.9465053081512451,
"rewards/hint_following/mean": 0.25,
"rewards/hint_following/std": 0.7699722051620483,
"rewards/length_penalty/mean": -0.5765926241874695,
"rewards/length_penalty/std": 0.28062164783477783,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9894134402275085,
"sampling/importance_sampling_ratio/min": 0.29406988620758057,
"sampling/sampling_logp_difference/max": 1.760861873626709,
"sampling/sampling_logp_difference/mean": 0.026971129700541496,
"step": 29,
"step_time": 55.79818298702594
},
{
"clip_ratio/high_max": 0.001682481961324811,
"clip_ratio/high_mean": 0.001682481961324811,
"clip_ratio/low_mean": 0.00047758713481016457,
"clip_ratio/low_min": 0.00047758713481016457,
"clip_ratio/region_mean": 0.002160069086433699,
"completions/clipped_ratio": 0.1666666716337204,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2747.0,
"completions/mean_length": 1684.638916015625,
"completions/mean_terminated_length": 1421.5667724609375,
"completions/min_length": 805.0,
"completions/min_terminated_length": 805.0,
"entropy": 0.2508200804392497,
"epoch": 0.01282051282051282,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06276588886976242,
"learning_rate": 5e-05,
"loss": 0.0312,
"num_tokens": 2331153.0,
"reward": 0.02178703434765339,
"reward_std": 0.9737120270729065,
"rewards/hint_following/mean": 0.5833333134651184,
"rewards/hint_following/std": 0.7699722051620483,
"rewards/length_penalty/mean": -0.5615463256835938,
"rewards/length_penalty/std": 0.233141228556633,
"sampling/importance_sampling_ratio/max": 2.9634652137756348,
"sampling/importance_sampling_ratio/mean": 0.9905918836593628,
"sampling/importance_sampling_ratio/min": 0.2952629625797272,
"sampling/sampling_logp_difference/max": 1.2198889255523682,
"sampling/sampling_logp_difference/mean": 0.024356598034501076,
"step": 30,
"step_time": 56.55290222284384
},
{
"clip_ratio/high_max": 0.008109931523601214,
"clip_ratio/high_mean": 0.008109931523601214,
"clip_ratio/low_mean": 0.0004286583377203594,
"clip_ratio/low_min": 0.0004286583377203594,
"clip_ratio/region_mean": 0.008538589735204974,
"completions/clipped_ratio": 0.0833333358168602,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2567.0,
"completions/mean_length": 1786.2222900390625,
"completions/mean_terminated_length": 1675.8787841796875,
"completions/min_length": 468.0,
"completions/min_terminated_length": 468.0,
"entropy": 0.2908742278814316,
"epoch": 0.013247863247863248,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07696390151977539,
"learning_rate": 5e-05,
"loss": 0.0561,
"num_tokens": 2403953.0,
"reward": -0.0954074040055275,
"reward_std": 0.6877580285072327,
"rewards/hint_following/mean": 0.5,
"rewards/hint_following/std": 0.6546536684036255,
"rewards/length_penalty/mean": -0.5954073667526245,
"rewards/length_penalty/std": 0.22547654807567596,
"sampling/importance_sampling_ratio/max": 2.491806745529175,
"sampling/importance_sampling_ratio/mean": 0.9886121153831482,
"sampling/importance_sampling_ratio/min": 0.16667605936527252,
"sampling/sampling_logp_difference/max": 1.7917031049728394,
"sampling/sampling_logp_difference/mean": 0.02771041914820671,
"step": 31,
"step_time": 58.1784061481012
},
{
"clip_ratio/high_max": 0.0027225150261074305,
"clip_ratio/high_mean": 0.0027225150261074305,
"clip_ratio/low_mean": 0.003243642975576222,
"clip_ratio/low_min": 0.003243642975576222,
"clip_ratio/region_mean": 0.00596615804048876,
"completions/clipped_ratio": 0.1666666716337204,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2917.0,
"completions/mean_length": 2085.916748046875,
"completions/mean_terminated_length": 1903.10009765625,
"completions/min_length": 1014.0,
"completions/min_terminated_length": 1014.0,
"entropy": 0.278283953666687,
"epoch": 0.013675213675213675,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06229299306869507,
"learning_rate": 5e-05,
"loss": 0.0363,
"num_tokens": 2486342.0,
"reward": -0.36197224259376526,
"reward_std": 0.8716595768928528,
"rewards/hint_following/mean": 0.3333333432674408,
"rewards/hint_following/std": 0.7559289932250977,
"rewards/length_penalty/mean": -0.695305585861206,
"rewards/length_penalty/std": 0.1993320733308792,
"sampling/importance_sampling_ratio/max": 2.49179744720459,
"sampling/importance_sampling_ratio/mean": 0.9892889261245728,
"sampling/importance_sampling_ratio/min": 0.19078373908996582,
"sampling/sampling_logp_difference/max": 1.6566147804260254,
"sampling/sampling_logp_difference/mean": 0.02677111327648163,
"step": 32,
"step_time": 59.783586613833904
},
{
"clip_ratio/high_max": 0.0016072407306637615,
"clip_ratio/high_mean": 0.0016072407306637615,
"clip_ratio/low_mean": 0.0016927721832568448,
"clip_ratio/low_min": 0.0016927721832568448,
"clip_ratio/region_mean": 0.003300012855712945,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2629.0,
"completions/max_terminated_length": 2629.0,
"completions/mean_length": 1464.5833740234375,
"completions/mean_terminated_length": 1464.5833740234375,
"completions/min_length": 577.0,
"completions/min_terminated_length": 577.0,
"entropy": 0.25622066855430603,
"epoch": 0.014102564102564103,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.05397763103246689,
"learning_rate": 5e-05,
"loss": 0.0059,
"num_tokens": 2544023.0,
"reward": -0.29375001788139343,
"reward_std": 0.45689767599105835,
"rewards/hint_following/mean": 0.1944444477558136,
"rewards/hint_following/std": 0.4013865292072296,
"rewards/length_penalty/mean": -0.48819446563720703,
"rewards/length_penalty/std": 0.14601026475429535,
"sampling/importance_sampling_ratio/max": 2.081763982772827,
"sampling/importance_sampling_ratio/mean": 0.9904037117958069,
"sampling/importance_sampling_ratio/min": 0.339209645986557,
"sampling/sampling_logp_difference/max": 1.08113694190979,
"sampling/sampling_logp_difference/mean": 0.02619028650224209,
"step": 33,
"step_time": 49.13755757326726
},
{
"clip_ratio/high_max": 0.001959889098846664,
"clip_ratio/high_mean": 0.001959889098846664,
"clip_ratio/low_mean": 0.0017016992593804996,
"clip_ratio/low_min": 0.0017016992593804996,
"clip_ratio/region_mean": 0.0036615883776297173,
"completions/clipped_ratio": 0.1666666716337204,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2814.0,
"completions/mean_length": 1581.1666259765625,
"completions/mean_terminated_length": 1297.4000244140625,
"completions/min_length": 447.0,
"completions/min_terminated_length": 447.0,
"entropy": 0.25040922313928604,
"epoch": 0.01452991452991453,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06385543942451477,
"learning_rate": 5e-05,
"loss": 0.046,
"num_tokens": 2606177.0,
"reward": -0.0826110988855362,
"reward_std": 0.9639437198638916,
"rewards/hint_following/mean": 0.4444444477558136,
"rewards/hint_following/std": 0.7725447416305542,
"rewards/length_penalty/mean": -0.5270555019378662,
"rewards/length_penalty/std": 0.2994069457054138,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9912077784538269,
"sampling/importance_sampling_ratio/min": 0.31699931621551514,
"sampling/sampling_logp_difference/max": 1.2634482383728027,
"sampling/sampling_logp_difference/mean": 0.02445373497903347,
"step": 34,
"step_time": 54.58746225386858
},
{
"clip_ratio/high_max": 0.0011685320253794391,
"clip_ratio/high_mean": 0.0011685320253794391,
"clip_ratio/low_mean": 0.0014686054394890864,
"clip_ratio/low_min": 0.0014686054394890864,
"clip_ratio/region_mean": 0.0026371375230761864,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2743.0,
"completions/max_terminated_length": 2743.0,
"completions/mean_length": 1370.4722900390625,
"completions/mean_terminated_length": 1370.4722900390625,
"completions/min_length": 650.0,
"completions/min_terminated_length": 650.0,
"entropy": 0.26418235152959824,
"epoch": 0.014957264957264958,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.057452693581581116,
"learning_rate": 5e-05,
"loss": 0.01,
"num_tokens": 2661580.0,
"reward": 0.15428705513477325,
"reward_std": 0.5879056453704834,
"rewards/hint_following/mean": 0.6111111044883728,
"rewards/hint_following/std": 0.49441322684288025,
"rewards/length_penalty/mean": -0.45682409405708313,
"rewards/length_penalty/std": 0.1867014318704605,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9900287985801697,
"sampling/importance_sampling_ratio/min": 0.15399949252605438,
"sampling/sampling_logp_difference/max": 1.8708059787750244,
"sampling/sampling_logp_difference/mean": 0.027402520179748535,
"step": 35,
"step_time": 47.69401362503413
},
{
"clip_ratio/high_max": 0.0057743463742857175,
"clip_ratio/high_mean": 0.0057743463742857175,
"clip_ratio/low_mean": 0.0029491251916624606,
"clip_ratio/low_min": 0.0029491251916624606,
"clip_ratio/region_mean": 0.00872347146893541,
"completions/clipped_ratio": 0.1111111119389534,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2813.0,
"completions/mean_length": 1503.6944580078125,
"completions/mean_terminated_length": 1316.65625,
"completions/min_length": 456.0,
"completions/min_terminated_length": 456.0,
"entropy": 0.2601601282755534,
"epoch": 0.015384615384615385,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06052026152610779,
"learning_rate": 5e-05,
"loss": 0.0217,
"num_tokens": 2721131.0,
"reward": -0.44567596912384033,
"reward_std": 0.6641168594360352,
"rewards/hint_following/mean": 0.0555555559694767,
"rewards/hint_following/std": 0.5315446853637695,
"rewards/length_penalty/mean": -0.5012314915657043,
"rewards/length_penalty/std": 0.2637265622615814,
"sampling/importance_sampling_ratio/max": 2.678779125213623,
"sampling/importance_sampling_ratio/mean": 0.98993980884552,
"sampling/importance_sampling_ratio/min": 0.3602158725261688,
"sampling/sampling_logp_difference/max": 1.0210518836975098,
"sampling/sampling_logp_difference/mean": 0.02683359384536743,
"step": 36,
"step_time": 52.9094661710551
},
{
"clip_ratio/high_max": 0.002038249202693502,
"clip_ratio/high_mean": 0.002038249202693502,
"clip_ratio/low_mean": 0.00521218675809602,
"clip_ratio/low_min": 0.00521218675809602,
"clip_ratio/region_mean": 0.007250435883179307,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2391.0,
"completions/max_terminated_length": 2391.0,
"completions/mean_length": 1279.638916015625,
"completions/mean_terminated_length": 1279.638916015625,
"completions/min_length": 508.0,
"completions/min_terminated_length": 508.0,
"entropy": 0.26387400925159454,
"epoch": 0.01581196581196581,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0660337284207344,
"learning_rate": 5e-05,
"loss": 0.0072,
"num_tokens": 2774056.0,
"reward": 0.12900924682617188,
"reward_std": 0.5431578159332275,
"rewards/hint_following/mean": 0.5555555820465088,
"rewards/hint_following/std": 0.5039526224136353,
"rewards/length_penalty/mean": -0.4265463054180145,
"rewards/length_penalty/std": 0.12839393317699432,
"sampling/importance_sampling_ratio/max": 2.7058398723602295,
"sampling/importance_sampling_ratio/mean": 0.9903796911239624,
"sampling/importance_sampling_ratio/min": 0.1684948354959488,
"sampling/sampling_logp_difference/max": 1.7808502912521362,
"sampling/sampling_logp_difference/mean": 0.027670899406075478,
"step": 37,
"step_time": 42.42564612603746
},
{
"clip_ratio/high_max": 0.005249147206389655,
"clip_ratio/high_mean": 0.005249147206389655,
"clip_ratio/low_mean": 0.0019288610492367297,
"clip_ratio/low_min": 0.0019288610492367297,
"clip_ratio/region_mean": 0.007178008245925109,
"completions/clipped_ratio": 0.1388888955116272,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2773.0,
"completions/mean_length": 1562.9166259765625,
"completions/mean_terminated_length": 1331.1290283203125,
"completions/min_length": 508.0,
"completions/min_terminated_length": 508.0,
"entropy": 0.2739727521936099,
"epoch": 0.01623931623931624,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.05931999906897545,
"learning_rate": 5e-05,
"loss": 0.0111,
"num_tokens": 2836603.0,
"reward": 0.034583330154418945,
"reward_std": 0.9446364045143127,
"rewards/hint_following/mean": 0.5555555820465088,
"rewards/hint_following/std": 0.7346308827400208,
"rewards/length_penalty/mean": -0.5209722518920898,
"rewards/length_penalty/std": 0.2565736174583435,
"sampling/importance_sampling_ratio/max": 2.9026551246643066,
"sampling/importance_sampling_ratio/mean": 0.9896891117095947,
"sampling/importance_sampling_ratio/min": 0.1531003713607788,
"sampling/sampling_logp_difference/max": 1.8766615390777588,
"sampling/sampling_logp_difference/mean": 0.02661670744419098,
"step": 38,
"step_time": 54.65668713499326
},
{
"clip_ratio/high_max": 0.0073317300993949175,
"clip_ratio/high_mean": 0.0073317300993949175,
"clip_ratio/low_mean": 0.0012036147963954136,
"clip_ratio/low_min": 0.0012036147963954136,
"clip_ratio/region_mean": 0.008535344852134585,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2373.0,
"completions/max_terminated_length": 2373.0,
"completions/mean_length": 1587.7222900390625,
"completions/mean_terminated_length": 1587.7222900390625,
"completions/min_length": 887.0,
"completions/min_terminated_length": 887.0,
"entropy": 0.2784932255744934,
"epoch": 0.016666666666666666,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.05303090810775757,
"learning_rate": 5e-05,
"loss": -0.0225,
"num_tokens": 2900313.0,
"reward": 0.22075927257537842,
"reward_std": 0.3945108652114868,
"rewards/hint_following/mean": 0.75,
"rewards/hint_following/std": 0.43915504217147827,
"rewards/length_penalty/mean": -0.5292407274246216,
"rewards/length_penalty/std": 0.10378893464803696,
"sampling/importance_sampling_ratio/max": 2.234417200088501,
"sampling/importance_sampling_ratio/mean": 0.9896702170372009,
"sampling/importance_sampling_ratio/min": 0.22993366420269012,
"sampling/sampling_logp_difference/max": 1.4699643850326538,
"sampling/sampling_logp_difference/mean": 0.027140147984027863,
"step": 39,
"step_time": 45.66687847697176
},
{
"clip_ratio/high_max": 0.008284316320593158,
"clip_ratio/high_mean": 0.008284316320593158,
"clip_ratio/low_mean": 0.0006285767303779721,
"clip_ratio/low_min": 0.0006285767303779721,
"clip_ratio/region_mean": 0.008912893012166023,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2490.0,
"completions/max_terminated_length": 2490.0,
"completions/mean_length": 1358.638916015625,
"completions/mean_terminated_length": 1358.638916015625,
"completions/min_length": 668.0,
"completions/min_terminated_length": 668.0,
"entropy": 0.2635483493407567,
"epoch": 0.017094017094017096,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.061241693794727325,
"learning_rate": 5e-05,
"loss": -0.0001,
"num_tokens": 2955062.0,
"reward": 0.2693426012992859,
"reward_std": 0.4795752167701721,
"rewards/hint_following/mean": 0.7222222089767456,
"rewards/hint_following/std": 0.45425674319267273,
"rewards/length_penalty/mean": -0.4528796374797821,
"rewards/length_penalty/std": 0.17493696510791779,
"sampling/importance_sampling_ratio/max": 2.239511728286743,
"sampling/importance_sampling_ratio/mean": 0.9903267621994019,
"sampling/importance_sampling_ratio/min": 0.11606565117835999,
"sampling/sampling_logp_difference/max": 2.153599262237549,
"sampling/sampling_logp_difference/mean": 0.02778884768486023,
"step": 40,
"step_time": 44.78665240900591
},
{
"clip_ratio/high_max": 0.0024162703969826302,
"clip_ratio/high_mean": 0.0024162703969826302,
"clip_ratio/low_mean": 0.004380864285243054,
"clip_ratio/low_min": 0.004380864285243054,
"clip_ratio/region_mean": 0.006797134643420577,
"completions/clipped_ratio": 0.02777777798473835,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2931.0,
"completions/mean_length": 1570.75,
"completions/mean_terminated_length": 1529.914306640625,
"completions/min_length": 437.0,
"completions/min_terminated_length": 437.0,
"entropy": 0.2851356367270152,
"epoch": 0.01752136752136752,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06564202159643173,
"learning_rate": 5e-05,
"loss": 0.0083,
"num_tokens": 3020207.0,
"reward": 0.1708611100912094,
"reward_std": 0.6461936831474304,
"rewards/hint_following/mean": 0.6944444179534912,
"rewards/hint_following/std": 0.524782657623291,
"rewards/length_penalty/mean": -0.5235832929611206,
"rewards/length_penalty/std": 0.23531726002693176,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9896803498268127,
"sampling/importance_sampling_ratio/min": 0.2426270991563797,
"sampling/sampling_logp_difference/max": 1.4162296056747437,
"sampling/sampling_logp_difference/mean": 0.02950497530400753,
"step": 41,
"step_time": 55.908626700984314
},
{
"clip_ratio/high_max": 0.0035200511144163706,
"clip_ratio/high_mean": 0.0035200511144163706,
"clip_ratio/low_mean": 0.0037633493387450776,
"clip_ratio/low_min": 0.0037633493387450776,
"clip_ratio/region_mean": 0.007283400433758895,
"completions/clipped_ratio": 0.1944444477558136,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2916.0,
"completions/mean_length": 1708.861083984375,
"completions/mean_terminated_length": 1397.2069091796875,
"completions/min_length": 601.0,
"completions/min_terminated_length": 601.0,
"entropy": 0.297603373726209,
"epoch": 0.017948717948717947,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06374821066856384,
"learning_rate": 5e-05,
"loss": 0.0258,
"num_tokens": 3087486.0,
"reward": -0.40295371413230896,
"reward_std": 0.9737837910652161,
"rewards/hint_following/mean": 0.1666666716337204,
"rewards/hint_following/std": 0.7367883920669556,
"rewards/length_penalty/mean": -0.5696203708648682,
"rewards/length_penalty/std": 0.28024521470069885,
"sampling/importance_sampling_ratio/max": 2.632629871368408,
"sampling/importance_sampling_ratio/mean": 0.9891336560249329,
"sampling/importance_sampling_ratio/min": 0.16689430177211761,
"sampling/sampling_logp_difference/max": 1.79039466381073,
"sampling/sampling_logp_difference/mean": 0.029687926173210144,
"step": 42,
"step_time": 55.97696550388355
},
{
"clip_ratio/high_max": 0.005934966184819738,
"clip_ratio/high_mean": 0.005934966184819738,
"clip_ratio/low_mean": 0.0023504976027955613,
"clip_ratio/low_min": 0.0023504976027955613,
"clip_ratio/region_mean": 0.008285463865225514,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2141.0,
"completions/max_terminated_length": 2141.0,
"completions/mean_length": 1201.638916015625,
"completions/mean_terminated_length": 1201.638916015625,
"completions/min_length": 517.0,
"completions/min_terminated_length": 517.0,
"entropy": 0.29462653398513794,
"epoch": 0.018376068376068377,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.05552098527550697,
"learning_rate": 5e-05,
"loss": 0.0036,
"num_tokens": 3135263.0,
"reward": 0.12723149359226227,
"reward_std": 0.5270769596099854,
"rewards/hint_following/mean": 0.5277777910232544,
"rewards/hint_following/std": 0.506309449672699,
"rewards/length_penalty/mean": -0.40054628252983093,
"rewards/length_penalty/std": 0.14752288162708282,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9892792701721191,
"sampling/importance_sampling_ratio/min": 0.3035065531730652,
"sampling/sampling_logp_difference/max": 1.192352056503296,
"sampling/sampling_logp_difference/mean": 0.030870094895362854,
"step": 43,
"step_time": 37.3642987072235
},
{
"clip_ratio/high_max": 0.0028009160111347833,
"clip_ratio/high_mean": 0.0028009160111347833,
"clip_ratio/low_mean": 0.003960246841112773,
"clip_ratio/low_min": 0.003960246841112773,
"clip_ratio/region_mean": 0.0067611627746373415,
"completions/clipped_ratio": 0.2777777910232544,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2767.0,
"completions/mean_length": 1834.611083984375,
"completions/mean_terminated_length": 1386.3846435546875,
"completions/min_length": 439.0,
"completions/min_terminated_length": 439.0,
"entropy": 0.2577643742163976,
"epoch": 0.018803418803418803,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06315090507268906,
"learning_rate": 5e-05,
"loss": 0.0313,
"num_tokens": 3208011.0,
"reward": -0.22264817357063293,
"reward_std": 1.158618688583374,
"rewards/hint_following/mean": 0.3888888955116272,
"rewards/hint_following/std": 0.903256893157959,
"rewards/length_penalty/mean": -0.6115370988845825,
"rewards/length_penalty/std": 0.3052543103694916,
"sampling/importance_sampling_ratio/max": 2.770780324935913,
"sampling/importance_sampling_ratio/mean": 0.9908032417297363,
"sampling/importance_sampling_ratio/min": 0.14089906215667725,
"sampling/sampling_logp_difference/max": 1.9597115516662598,
"sampling/sampling_logp_difference/mean": 0.026477616280317307,
"step": 44,
"step_time": 57.508703485131264
},
{
"clip_ratio/high_max": 0.0022431810890945294,
"clip_ratio/high_mean": 0.0022431810890945294,
"clip_ratio/low_mean": 0.0008466135962711027,
"clip_ratio/low_min": 0.0008466135962711027,
"clip_ratio/region_mean": 0.003089794685365632,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2567.0,
"completions/max_terminated_length": 2567.0,
"completions/mean_length": 1403.611083984375,
"completions/mean_terminated_length": 1403.611083984375,
"completions/min_length": 791.0,
"completions/min_terminated_length": 791.0,
"entropy": 0.2975164105494817,
"epoch": 0.019230769230769232,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.0667228028178215,
"learning_rate": 5e-05,
"loss": 0.0231,
"num_tokens": 3265633.0,
"reward": 0.39324072003364563,
"reward_std": 0.3923882842063904,
"rewards/hint_following/mean": 0.8611111044883728,
"rewards/hint_following/std": 0.35073620080947876,
"rewards/length_penalty/mean": -0.4678703546524048,
"rewards/length_penalty/std": 0.15880942344665527,
"sampling/importance_sampling_ratio/max": 2.800001382827759,
"sampling/importance_sampling_ratio/mean": 0.9887976050376892,
"sampling/importance_sampling_ratio/min": 0.2656780183315277,
"sampling/sampling_logp_difference/max": 1.325470209121704,
"sampling/sampling_logp_difference/mean": 0.030811548233032227,
"step": 45,
"step_time": 45.27160808304325
},
{
"clip_ratio/high_max": 0.007887479305888215,
"clip_ratio/high_mean": 0.007887479305888215,
"clip_ratio/low_mean": 0.000994703887651364,
"clip_ratio/low_min": 0.000994703887651364,
"clip_ratio/region_mean": 0.008882183115929365,
"completions/clipped_ratio": 0.0,
"completions/max_length": 2359.0,
"completions/max_terminated_length": 2359.0,
"completions/mean_length": 1282.52783203125,
"completions/mean_terminated_length": 1282.52783203125,
"completions/min_length": 561.0,
"completions/min_terminated_length": 561.0,
"entropy": 0.2848661094903946,
"epoch": 0.019658119658119658,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.051431454718112946,
"learning_rate": 5e-05,
"loss": -0.0024,
"num_tokens": 3318716.0,
"reward": 0.37804630398750305,
"reward_std": 0.3656369149684906,
"rewards/hint_following/mean": 0.8055555820465088,
"rewards/hint_following/std": 0.4013865292072296,
"rewards/length_penalty/mean": -0.42750924825668335,
"rewards/length_penalty/std": 0.16107404232025146,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9896759986877441,
"sampling/importance_sampling_ratio/min": 0.09941968321800232,
"sampling/sampling_logp_difference/max": 2.3084051609039307,
"sampling/sampling_logp_difference/mean": 0.029690390452742577,
"step": 46,
"step_time": 41.876618986018
},
{
"clip_ratio/high_max": 0.003261159232351929,
"clip_ratio/high_mean": 0.003261159232351929,
"clip_ratio/low_mean": 0.0030720558910009763,
"clip_ratio/low_min": 0.0030720558910009763,
"clip_ratio/region_mean": 0.0063332150457426906,
"completions/clipped_ratio": 0.0833333358168602,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2990.0,
"completions/mean_length": 1868.77783203125,
"completions/mean_terminated_length": 1765.939453125,
"completions/min_length": 947.0,
"completions/min_terminated_length": 947.0,
"entropy": 0.3000795841217041,
"epoch": 0.020085470085470087,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.07183089107275009,
"learning_rate": 5e-05,
"loss": 0.0154,
"num_tokens": 3395160.0,
"reward": -0.12292594462633133,
"reward_std": 0.785350501537323,
"rewards/hint_following/mean": 0.5,
"rewards/hint_following/std": 0.6546536684036255,
"rewards/length_penalty/mean": -0.6229259371757507,
"rewards/length_penalty/std": 0.2120964676141739,
"sampling/importance_sampling_ratio/max": 2.9097461700439453,
"sampling/importance_sampling_ratio/mean": 0.988696277141571,
"sampling/importance_sampling_ratio/min": 0.10777360945940018,
"sampling/sampling_logp_difference/max": 2.227722406387329,
"sampling/sampling_logp_difference/mean": 0.030243437737226486,
"step": 47,
"step_time": 58.67247669992503
},
{
"clip_ratio/high_max": 0.008134961283455292,
"clip_ratio/high_mean": 0.008134961283455292,
"clip_ratio/low_mean": 0.0004609357565641403,
"clip_ratio/low_min": 0.0004609357565641403,
"clip_ratio/region_mean": 0.008595897040019432,
"completions/clipped_ratio": 0.2222222238779068,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2543.0,
"completions/mean_length": 1645.8333740234375,
"completions/mean_terminated_length": 1258.9285888671875,
"completions/min_length": 729.0,
"completions/min_terminated_length": 729.0,
"entropy": 0.24671275417009988,
"epoch": 0.020512820512820513,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06476566940546036,
"learning_rate": 5e-05,
"loss": 0.0513,
"num_tokens": 3461970.0,
"reward": 0.006944431457668543,
"reward_std": 1.0957543849945068,
"rewards/hint_following/mean": 0.5555555820465088,
"rewards/hint_following/std": 0.8432740569114685,
"rewards/length_penalty/mean": -0.5486111044883728,
"rewards/length_penalty/std": 0.2772509455680847,
"sampling/importance_sampling_ratio/max": 2.0642147064208984,
"sampling/importance_sampling_ratio/mean": 0.9907264113426208,
"sampling/importance_sampling_ratio/min": 0.04710150137543678,
"sampling/sampling_logp_difference/max": 3.055450439453125,
"sampling/sampling_logp_difference/mean": 0.025603190064430237,
"step": 48,
"step_time": 55.27118606586009
},
{
"clip_ratio/high_max": 0.0011131278927981232,
"clip_ratio/high_mean": 0.0011131278927981232,
"clip_ratio/low_mean": 0.006329516724993785,
"clip_ratio/low_min": 0.006329516724993785,
"clip_ratio/region_mean": 0.007442644564434886,
"completions/clipped_ratio": 0.0,
"completions/max_length": 1867.0,
"completions/max_terminated_length": 1867.0,
"completions/mean_length": 1157.6666259765625,
"completions/mean_terminated_length": 1157.6666259765625,
"completions/min_length": 722.0,
"completions/min_terminated_length": 722.0,
"entropy": 0.2628251537680626,
"epoch": 0.02094017094017094,
"frac_reward_zero_std": 0.0,
"grad_norm": 0.06659235060214996,
"learning_rate": 5e-05,
"loss": 0.0356,
"num_tokens": 3510510.0,
"reward": 0.5863333344459534,
"reward_std": 0.22435788810253143,
"rewards/hint_following/mean": 0.9722222089767456,
"rewards/hint_following/std": 0.1666666716337204,
"rewards/length_penalty/mean": -0.3858889043331146,
"rewards/length_penalty/std": 0.10318709164857864,
"sampling/importance_sampling_ratio/max": 2.3799936771392822,
"sampling/importance_sampling_ratio/mean": 0.9904940128326416,
"sampling/importance_sampling_ratio/min": 0.06988711655139923,
"sampling/sampling_logp_difference/max": 2.6608738899230957,
"sampling/sampling_logp_difference/mean": 0.030541572719812393,
"step": 49,
"step_time": 33.86789354507346
},
{
"clip_ratio/high_max": 0.0,
"clip_ratio/high_mean": 0.0,
"clip_ratio/low_mean": 0.0,
"clip_ratio/low_min": 0.0,
"clip_ratio/region_mean": 0.0,
"completions/clipped_ratio": 0.1666666716337204,
"completions/max_length": 3000.0,
"completions/max_terminated_length": 2940.0,
"completions/mean_length": 1683.0,
"completions/mean_terminated_length": 1419.60009765625,
"completions/min_length": 730.0,
"completions/min_terminated_length": 730.0,
"entropy": 0.24505582451820374,
"epoch": 0.021367521367521368,
"frac_reward_zero_std": 1.0,
"grad_norm": 0.0,
"learning_rate": 5e-05,
"loss": 0.0,
"num_tokens": 3577332.0,
"reward": 0.10566666722297668,
"reward_std": 0.9825097322463989,
"rewards/hint_following/mean": 0.6666666865348816,
"rewards/hint_following/std": 0.7559289932250977,
"rewards/length_penalty/mean": -0.5609999895095825,
"rewards/length_penalty/std": 0.30474284291267395,
"sampling/importance_sampling_ratio/max": 3.0,
"sampling/importance_sampling_ratio/mean": 0.9918250441551208,
"sampling/importance_sampling_ratio/min": 0.06838197261095047,
"sampling/sampling_logp_difference/max": 2.6826460361480713,
"sampling/sampling_logp_difference/mean": 0.028820084407925606,
"step": 50,
"step_time": 54.36680988012813
}
],
"logging_steps": 1,
"max_steps": 250,
"num_input_tokens_seen": 3577332,
"num_train_epochs": 1,
"save_steps": 50,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": false
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 6,
"trial_name": null,
"trial_params": null
}
|