File size: 41,389 Bytes
791eae6 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 |
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 1.0,
"eval_steps": 100,
"global_step": 48,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"debug/policy_chosen_logits": 32.32257843017578,
"debug/policy_chosen_logps": -428.3227844238281,
"debug/policy_rejected_logits": 30.82423210144043,
"debug/policy_rejected_logps": -426.3280029296875,
"debug/reference_chosen_logps": -428.3227844238281,
"debug/reference_rejected_logps": -426.3280029296875,
"epoch": 0.020833333333333332,
"grad_norm": 6.447932275019588,
"learning_rate": 1e-06,
"logits/chosen": 32.32257843017578,
"logits/rejected": 30.82423210144043,
"logps/chosen": -428.3227844238281,
"logps/rejected": -426.3280029296875,
"loss": 0.5,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"debug/policy_chosen_logits": 31.886442184448242,
"debug/policy_chosen_logps": -418.96539306640625,
"debug/policy_rejected_logits": 29.381662368774414,
"debug/policy_rejected_logps": -429.94775390625,
"debug/reference_chosen_logps": -418.5838623046875,
"debug/reference_rejected_logps": -429.845703125,
"epoch": 0.041666666666666664,
"grad_norm": 5.235639772033975,
"learning_rate": 1e-06,
"logits/chosen": 31.886442184448242,
"logits/rejected": 29.381662368774414,
"logps/chosen": -418.96539306640625,
"logps/rejected": -429.94775390625,
"loss": 0.4988,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.00381515477783978,
"rewards/margins": -0.0027948375791311264,
"rewards/rejected": -0.001020316849462688,
"step": 2
},
{
"debug/policy_chosen_logits": 31.19687843322754,
"debug/policy_chosen_logps": -417.73297119140625,
"debug/policy_rejected_logits": 29.822526931762695,
"debug/policy_rejected_logps": -424.8887939453125,
"debug/reference_chosen_logps": -416.591796875,
"debug/reference_rejected_logps": -424.60333251953125,
"epoch": 0.0625,
"grad_norm": 5.035431279495221,
"learning_rate": 1e-06,
"logits/chosen": 31.19687843322754,
"logits/rejected": 29.822526931762695,
"logps/chosen": -417.73297119140625,
"logps/rejected": -424.8887939453125,
"loss": 0.5013,
"rewards/accuracies": 0.125,
"rewards/chosen": -0.011411934159696102,
"rewards/margins": -0.008557281456887722,
"rewards/rejected": -0.002854652237147093,
"step": 3
},
{
"debug/policy_chosen_logits": 31.30547523498535,
"debug/policy_chosen_logps": -409.65521240234375,
"debug/policy_rejected_logits": 31.427326202392578,
"debug/policy_rejected_logps": -419.1351013183594,
"debug/reference_chosen_logps": -410.0574951171875,
"debug/reference_rejected_logps": -419.167724609375,
"epoch": 0.08333333333333333,
"grad_norm": 4.947716251775631,
"learning_rate": 1e-06,
"logits/chosen": 31.30547523498535,
"logits/rejected": 31.427326202392578,
"logps/chosen": -409.65521240234375,
"logps/rejected": -419.1351013183594,
"loss": 0.4997,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.0040228646248579025,
"rewards/margins": 0.0036964411847293377,
"rewards/rejected": 0.00032642344012856483,
"step": 4
},
{
"debug/policy_chosen_logits": 26.84766387939453,
"debug/policy_chosen_logps": -419.80255126953125,
"debug/policy_rejected_logits": 25.351024627685547,
"debug/policy_rejected_logps": -403.54278564453125,
"debug/reference_chosen_logps": -419.79803466796875,
"debug/reference_rejected_logps": -403.3204040527344,
"epoch": 0.10416666666666667,
"grad_norm": 5.437518779758296,
"learning_rate": 1e-06,
"logits/chosen": 26.84766387939453,
"logits/rejected": 25.351024627685547,
"logps/chosen": -419.80255126953125,
"logps/rejected": -403.54278564453125,
"loss": 0.5011,
"rewards/accuracies": 0.5,
"rewards/chosen": -4.535634070634842e-05,
"rewards/margins": 0.0021785348653793335,
"rewards/rejected": -0.0022238921374082565,
"step": 5
},
{
"debug/policy_chosen_logits": 28.140277862548828,
"debug/policy_chosen_logps": -435.39581298828125,
"debug/policy_rejected_logits": 29.35475730895996,
"debug/policy_rejected_logps": -438.25970458984375,
"debug/reference_chosen_logps": -435.1541748046875,
"debug/reference_rejected_logps": -436.6419677734375,
"epoch": 0.125,
"grad_norm": 5.3788783078874625,
"learning_rate": 1e-06,
"logits/chosen": 28.140277862548828,
"logits/rejected": 29.35475730895996,
"logps/chosen": -435.39581298828125,
"logps/rejected": -438.25970458984375,
"loss": 0.4935,
"rewards/accuracies": 0.875,
"rewards/chosen": -0.002416381612420082,
"rewards/margins": 0.013761366717517376,
"rewards/rejected": -0.016177749261260033,
"step": 6
},
{
"debug/policy_chosen_logits": 32.219444274902344,
"debug/policy_chosen_logps": -422.5959167480469,
"debug/policy_rejected_logits": 33.60882568359375,
"debug/policy_rejected_logps": -429.6781311035156,
"debug/reference_chosen_logps": -422.28350830078125,
"debug/reference_rejected_logps": -429.510009765625,
"epoch": 0.14583333333333334,
"grad_norm": 5.089230644656757,
"learning_rate": 1e-06,
"logits/chosen": 32.219444274902344,
"logits/rejected": 33.60882568359375,
"logps/chosen": -422.5959167480469,
"logps/rejected": -429.6781311035156,
"loss": 0.499,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.0031237031798809767,
"rewards/margins": -0.0014425661647692323,
"rewards/rejected": -0.0016811370151117444,
"step": 7
},
{
"debug/policy_chosen_logits": 34.4580192565918,
"debug/policy_chosen_logps": -425.48052978515625,
"debug/policy_rejected_logits": 34.77037048339844,
"debug/policy_rejected_logps": -458.2679138183594,
"debug/reference_chosen_logps": -425.6324462890625,
"debug/reference_rejected_logps": -455.59991455078125,
"epoch": 0.16666666666666666,
"grad_norm": 6.722332782296524,
"learning_rate": 1e-06,
"logits/chosen": 34.4580192565918,
"logits/rejected": 34.77037048339844,
"logps/chosen": -425.48052978515625,
"logps/rejected": -458.2679138183594,
"loss": 0.4938,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.0015194700099527836,
"rewards/margins": 0.02819930762052536,
"rewards/rejected": -0.026679838076233864,
"step": 8
},
{
"debug/policy_chosen_logits": 29.579063415527344,
"debug/policy_chosen_logps": -397.6627502441406,
"debug/policy_rejected_logits": 29.510604858398438,
"debug/policy_rejected_logps": -423.0687255859375,
"debug/reference_chosen_logps": -398.0777282714844,
"debug/reference_rejected_logps": -422.35186767578125,
"epoch": 0.1875,
"grad_norm": 5.150446773896013,
"learning_rate": 1e-06,
"logits/chosen": 29.579063415527344,
"logits/rejected": 29.510604858398438,
"logps/chosen": -397.6627502441406,
"logps/rejected": -423.0687255859375,
"loss": 0.4984,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.004149818792939186,
"rewards/margins": 0.011318511329591274,
"rewards/rejected": -0.007168693467974663,
"step": 9
},
{
"debug/policy_chosen_logits": 30.510459899902344,
"debug/policy_chosen_logps": -406.46258544921875,
"debug/policy_rejected_logits": 32.81418228149414,
"debug/policy_rejected_logps": -420.8358154296875,
"debug/reference_chosen_logps": -405.9990539550781,
"debug/reference_rejected_logps": -419.8515930175781,
"epoch": 0.20833333333333334,
"grad_norm": 5.292871534847833,
"learning_rate": 1e-06,
"logits/chosen": 30.510459899902344,
"logits/rejected": 32.81418228149414,
"logps/chosen": -406.46258544921875,
"logps/rejected": -420.8358154296875,
"loss": 0.4965,
"rewards/accuracies": 0.375,
"rewards/chosen": -0.004634971730411053,
"rewards/margins": 0.005207213573157787,
"rewards/rejected": -0.009842186234891415,
"step": 10
},
{
"debug/policy_chosen_logits": 29.758987426757812,
"debug/policy_chosen_logps": -421.1949462890625,
"debug/policy_rejected_logits": 30.991361618041992,
"debug/policy_rejected_logps": -440.055908203125,
"debug/reference_chosen_logps": -420.9813537597656,
"debug/reference_rejected_logps": -440.38330078125,
"epoch": 0.22916666666666666,
"grad_norm": 5.257332824569119,
"learning_rate": 1e-06,
"logits/chosen": 29.758987426757812,
"logits/rejected": 30.991361618041992,
"logps/chosen": -421.1949462890625,
"logps/rejected": -440.055908203125,
"loss": 0.4898,
"rewards/accuracies": 0.5,
"rewards/chosen": -0.002135887276381254,
"rewards/margins": -0.005409927107393742,
"rewards/rejected": 0.003274040063843131,
"step": 11
},
{
"debug/policy_chosen_logits": 32.33553695678711,
"debug/policy_chosen_logps": -442.6135559082031,
"debug/policy_rejected_logits": 32.11320114135742,
"debug/policy_rejected_logps": -444.33697509765625,
"debug/reference_chosen_logps": -441.39215087890625,
"debug/reference_rejected_logps": -443.31390380859375,
"epoch": 0.25,
"grad_norm": 6.359103183966066,
"learning_rate": 1e-06,
"logits/chosen": 32.33553695678711,
"logits/rejected": 32.11320114135742,
"logps/chosen": -442.6135559082031,
"logps/rejected": -444.33697509765625,
"loss": 0.4997,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.01221416424959898,
"rewards/margins": -0.001983833033591509,
"rewards/rejected": -0.010230331681668758,
"step": 12
},
{
"debug/policy_chosen_logits": 29.988950729370117,
"debug/policy_chosen_logps": -425.54913330078125,
"debug/policy_rejected_logits": 30.10309410095215,
"debug/policy_rejected_logps": -426.21905517578125,
"debug/reference_chosen_logps": -426.2005920410156,
"debug/reference_rejected_logps": -424.59246826171875,
"epoch": 0.2708333333333333,
"grad_norm": 5.288806638849553,
"learning_rate": 1e-06,
"logits/chosen": 29.988950729370117,
"logits/rejected": 30.10309410095215,
"logps/chosen": -425.54913330078125,
"logps/rejected": -426.21905517578125,
"loss": 0.4925,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.006514891982078552,
"rewards/margins": 0.022780990228056908,
"rewards/rejected": -0.016266098245978355,
"step": 13
},
{
"debug/policy_chosen_logits": 31.317047119140625,
"debug/policy_chosen_logps": -419.84283447265625,
"debug/policy_rejected_logits": 32.35562515258789,
"debug/policy_rejected_logps": -399.378173828125,
"debug/reference_chosen_logps": -421.95867919921875,
"debug/reference_rejected_logps": -399.3148193359375,
"epoch": 0.2916666666666667,
"grad_norm": 5.154675497648274,
"learning_rate": 1e-06,
"logits/chosen": 31.317047119140625,
"logits/rejected": 32.35562515258789,
"logps/chosen": -419.84283447265625,
"logps/rejected": -399.378173828125,
"loss": 0.4906,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.021158332005143166,
"rewards/margins": 0.021792029961943626,
"rewards/rejected": -0.0006336974911391735,
"step": 14
},
{
"debug/policy_chosen_logits": 29.34722328186035,
"debug/policy_chosen_logps": -394.68389892578125,
"debug/policy_rejected_logits": 27.714096069335938,
"debug/policy_rejected_logps": -414.51715087890625,
"debug/reference_chosen_logps": -395.9772033691406,
"debug/reference_rejected_logps": -414.28857421875,
"epoch": 0.3125,
"grad_norm": 5.191304228736527,
"learning_rate": 1e-06,
"logits/chosen": 29.34722328186035,
"logits/rejected": 27.714096069335938,
"logps/chosen": -394.68389892578125,
"logps/rejected": -414.51715087890625,
"loss": 0.4844,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.012933043763041496,
"rewards/margins": 0.015218695625662804,
"rewards/rejected": -0.0022856518626213074,
"step": 15
},
{
"debug/policy_chosen_logits": 29.499605178833008,
"debug/policy_chosen_logps": -416.2149658203125,
"debug/policy_rejected_logits": 31.734092712402344,
"debug/policy_rejected_logps": -420.2775573730469,
"debug/reference_chosen_logps": -417.6040954589844,
"debug/reference_rejected_logps": -421.9405822753906,
"epoch": 0.3333333333333333,
"grad_norm": 5.336647699178971,
"learning_rate": 1e-06,
"logits/chosen": 29.499605178833008,
"logits/rejected": 31.734092712402344,
"logps/chosen": -416.2149658203125,
"logps/rejected": -420.2775573730469,
"loss": 0.4909,
"rewards/accuracies": 0.25,
"rewards/chosen": 0.013891411013901234,
"rewards/margins": -0.002739105373620987,
"rewards/rejected": 0.016630517318844795,
"step": 16
},
{
"debug/policy_chosen_logits": 30.53592872619629,
"debug/policy_chosen_logps": -424.2174987792969,
"debug/policy_rejected_logits": 28.613372802734375,
"debug/policy_rejected_logps": -424.45184326171875,
"debug/reference_chosen_logps": -425.0408020019531,
"debug/reference_rejected_logps": -424.396728515625,
"epoch": 0.3541666666666667,
"grad_norm": 5.069961718294066,
"learning_rate": 1e-06,
"logits/chosen": 30.53592872619629,
"logits/rejected": 28.613372802734375,
"logps/chosen": -424.2174987792969,
"logps/rejected": -424.45184326171875,
"loss": 0.4845,
"rewards/accuracies": 0.375,
"rewards/chosen": 0.008232764899730682,
"rewards/margins": 0.00878402590751648,
"rewards/rejected": -0.000551263801753521,
"step": 17
},
{
"debug/policy_chosen_logits": 32.6837272644043,
"debug/policy_chosen_logps": -439.52508544921875,
"debug/policy_rejected_logits": 30.927047729492188,
"debug/policy_rejected_logps": -429.91094970703125,
"debug/reference_chosen_logps": -441.05828857421875,
"debug/reference_rejected_logps": -431.7398376464844,
"epoch": 0.375,
"grad_norm": 5.035721183912941,
"learning_rate": 1e-06,
"logits/chosen": 32.6837272644043,
"logits/rejected": 30.927047729492188,
"logps/chosen": -439.52508544921875,
"logps/rejected": -429.91094970703125,
"loss": 0.5027,
"rewards/accuracies": 0.375,
"rewards/chosen": 0.015331955626606941,
"rewards/margins": -0.0029569631442427635,
"rewards/rejected": 0.01828891783952713,
"step": 18
},
{
"debug/policy_chosen_logits": 30.297531127929688,
"debug/policy_chosen_logps": -402.0390319824219,
"debug/policy_rejected_logits": 33.57741165161133,
"debug/policy_rejected_logps": -410.6739501953125,
"debug/reference_chosen_logps": -401.3625793457031,
"debug/reference_rejected_logps": -410.6376037597656,
"epoch": 0.3958333333333333,
"grad_norm": 5.398638220710165,
"learning_rate": 1e-06,
"logits/chosen": 30.297531127929688,
"logits/rejected": 33.57741165161133,
"logps/chosen": -402.0390319824219,
"logps/rejected": -410.6739501953125,
"loss": 0.4776,
"rewards/accuracies": 0.25,
"rewards/chosen": -0.00676448829472065,
"rewards/margins": -0.006401100195944309,
"rewards/rejected": -0.0003633880987763405,
"step": 19
},
{
"debug/policy_chosen_logits": 31.950498580932617,
"debug/policy_chosen_logps": -414.8414306640625,
"debug/policy_rejected_logits": 33.85417938232422,
"debug/policy_rejected_logps": -425.31268310546875,
"debug/reference_chosen_logps": -415.87957763671875,
"debug/reference_rejected_logps": -425.8223876953125,
"epoch": 0.4166666666666667,
"grad_norm": 5.034205804652484,
"learning_rate": 1e-06,
"logits/chosen": 31.950498580932617,
"logits/rejected": 33.85417938232422,
"logps/chosen": -414.8414306640625,
"logps/rejected": -425.31268310546875,
"loss": 0.4901,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.010381774976849556,
"rewards/margins": 0.005284767597913742,
"rewards/rejected": 0.0050970083102583885,
"step": 20
},
{
"debug/policy_chosen_logits": 31.606367111206055,
"debug/policy_chosen_logps": -415.55767822265625,
"debug/policy_rejected_logits": 32.42195510864258,
"debug/policy_rejected_logps": -409.7369384765625,
"debug/reference_chosen_logps": -417.48565673828125,
"debug/reference_rejected_logps": -411.9869689941406,
"epoch": 0.4375,
"grad_norm": 4.815275729217639,
"learning_rate": 1e-06,
"logits/chosen": 31.606367111206055,
"logits/rejected": 32.42195510864258,
"logps/chosen": -415.55767822265625,
"logps/rejected": -409.7369384765625,
"loss": 0.4884,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.01928028091788292,
"rewards/margins": -0.0032197567634284496,
"rewards/rejected": 0.022500038146972656,
"step": 21
},
{
"debug/policy_chosen_logits": 30.70886993408203,
"debug/policy_chosen_logps": -433.744873046875,
"debug/policy_rejected_logits": 26.203298568725586,
"debug/policy_rejected_logps": -415.767822265625,
"debug/reference_chosen_logps": -431.77203369140625,
"debug/reference_rejected_logps": -414.41796875,
"epoch": 0.4583333333333333,
"grad_norm": 5.219226631225296,
"learning_rate": 1e-06,
"logits/chosen": 30.70886993408203,
"logits/rejected": 26.203298568725586,
"logps/chosen": -433.744873046875,
"logps/rejected": -415.767822265625,
"loss": 0.4748,
"rewards/accuracies": 0.25,
"rewards/chosen": -0.019728660583496094,
"rewards/margins": -0.006230468861758709,
"rewards/rejected": -0.01349819079041481,
"step": 22
},
{
"debug/policy_chosen_logits": 30.573122024536133,
"debug/policy_chosen_logps": -420.0906677246094,
"debug/policy_rejected_logits": 31.557247161865234,
"debug/policy_rejected_logps": -405.26837158203125,
"debug/reference_chosen_logps": -422.8019714355469,
"debug/reference_rejected_logps": -406.9261474609375,
"epoch": 0.4791666666666667,
"grad_norm": 4.90573818429262,
"learning_rate": 1e-06,
"logits/chosen": 30.573122024536133,
"logits/rejected": 31.557247161865234,
"logps/chosen": -420.0906677246094,
"logps/rejected": -405.26837158203125,
"loss": 0.4845,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.027113037183880806,
"rewards/margins": 0.010535049252212048,
"rewards/rejected": 0.016577987000346184,
"step": 23
},
{
"debug/policy_chosen_logits": 28.54072380065918,
"debug/policy_chosen_logps": -429.51544189453125,
"debug/policy_rejected_logits": 28.603498458862305,
"debug/policy_rejected_logps": -448.6463623046875,
"debug/reference_chosen_logps": -429.3318176269531,
"debug/reference_rejected_logps": -445.1684875488281,
"epoch": 0.5,
"grad_norm": 4.919240212142553,
"learning_rate": 1e-06,
"logits/chosen": 28.54072380065918,
"logits/rejected": 28.603498458862305,
"logps/chosen": -429.51544189453125,
"logps/rejected": -448.6463623046875,
"loss": 0.4835,
"rewards/accuracies": 0.75,
"rewards/chosen": -0.0018361280672252178,
"rewards/margins": 0.03294284641742706,
"rewards/rejected": -0.03477897495031357,
"step": 24
},
{
"debug/policy_chosen_logits": 32.81056594848633,
"debug/policy_chosen_logps": -443.39068603515625,
"debug/policy_rejected_logits": 30.637916564941406,
"debug/policy_rejected_logps": -425.4708251953125,
"debug/reference_chosen_logps": -439.9627685546875,
"debug/reference_rejected_logps": -420.35076904296875,
"epoch": 0.5208333333333334,
"grad_norm": 5.047798852836749,
"learning_rate": 1e-06,
"logits/chosen": 32.81056594848633,
"logits/rejected": 30.637916564941406,
"logps/chosen": -443.39068603515625,
"logps/rejected": -425.4708251953125,
"loss": 0.4832,
"rewards/accuracies": 0.625,
"rewards/chosen": -0.03427928686141968,
"rewards/margins": 0.016921542584896088,
"rewards/rejected": -0.051200829446315765,
"step": 25
},
{
"debug/policy_chosen_logits": 29.654489517211914,
"debug/policy_chosen_logps": -403.1080322265625,
"debug/policy_rejected_logits": 29.024642944335938,
"debug/policy_rejected_logps": -421.46270751953125,
"debug/reference_chosen_logps": -404.3522644042969,
"debug/reference_rejected_logps": -419.66693115234375,
"epoch": 0.5416666666666666,
"grad_norm": 4.941664000787305,
"learning_rate": 1e-06,
"logits/chosen": 29.654489517211914,
"logits/rejected": 29.024642944335938,
"logps/chosen": -403.1080322265625,
"logps/rejected": -421.46270751953125,
"loss": 0.4751,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.012442246079444885,
"rewards/margins": 0.030400237068533897,
"rewards/rejected": -0.01795799285173416,
"step": 26
},
{
"debug/policy_chosen_logits": 28.045677185058594,
"debug/policy_chosen_logps": -403.36614990234375,
"debug/policy_rejected_logits": 30.181791305541992,
"debug/policy_rejected_logps": -423.99591064453125,
"debug/reference_chosen_logps": -405.4366149902344,
"debug/reference_rejected_logps": -424.10650634765625,
"epoch": 0.5625,
"grad_norm": 5.109404536265639,
"learning_rate": 1e-06,
"logits/chosen": 28.045677185058594,
"logits/rejected": 30.181791305541992,
"logps/chosen": -403.36614990234375,
"logps/rejected": -423.99591064453125,
"loss": 0.4726,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.020704690366983414,
"rewards/margins": 0.019598999992012978,
"rewards/rejected": 0.0011056894436478615,
"step": 27
},
{
"debug/policy_chosen_logits": 30.849773406982422,
"debug/policy_chosen_logps": -421.11431884765625,
"debug/policy_rejected_logits": 30.653217315673828,
"debug/policy_rejected_logps": -440.32244873046875,
"debug/reference_chosen_logps": -418.60089111328125,
"debug/reference_rejected_logps": -439.1358642578125,
"epoch": 0.5833333333333334,
"grad_norm": 5.1018343354077205,
"learning_rate": 1e-06,
"logits/chosen": 30.849773406982422,
"logits/rejected": 30.653217315673828,
"logps/chosen": -421.11431884765625,
"logps/rejected": -440.32244873046875,
"loss": 0.4788,
"rewards/accuracies": 0.25,
"rewards/chosen": -0.025134162977337837,
"rewards/margins": -0.013268355280160904,
"rewards/rejected": -0.011865806765854359,
"step": 28
},
{
"debug/policy_chosen_logits": 32.95792770385742,
"debug/policy_chosen_logps": -438.89129638671875,
"debug/policy_rejected_logits": 32.24089813232422,
"debug/policy_rejected_logps": -445.365966796875,
"debug/reference_chosen_logps": -440.4767761230469,
"debug/reference_rejected_logps": -440.51837158203125,
"epoch": 0.6041666666666666,
"grad_norm": 4.632495681187026,
"learning_rate": 1e-06,
"logits/chosen": 32.95792770385742,
"logits/rejected": 32.24089813232422,
"logps/chosen": -438.89129638671875,
"logps/rejected": -445.365966796875,
"loss": 0.4649,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.015854567289352417,
"rewards/margins": 0.06433063000440598,
"rewards/rejected": -0.04847606271505356,
"step": 29
},
{
"debug/policy_chosen_logits": 31.374120712280273,
"debug/policy_chosen_logps": -412.0467529296875,
"debug/policy_rejected_logits": 30.670644760131836,
"debug/policy_rejected_logps": -437.80059814453125,
"debug/reference_chosen_logps": -414.59442138671875,
"debug/reference_rejected_logps": -433.7508544921875,
"epoch": 0.625,
"grad_norm": 5.135137765814023,
"learning_rate": 1e-06,
"logits/chosen": 31.374120712280273,
"logits/rejected": 30.670644760131836,
"logps/chosen": -412.0467529296875,
"logps/rejected": -437.80059814453125,
"loss": 0.4928,
"rewards/accuracies": 0.875,
"rewards/chosen": 0.025476723909378052,
"rewards/margins": 0.06597384810447693,
"rewards/rejected": -0.040497127920389175,
"step": 30
},
{
"debug/policy_chosen_logits": 28.232507705688477,
"debug/policy_chosen_logps": -414.2076721191406,
"debug/policy_rejected_logits": 27.216352462768555,
"debug/policy_rejected_logps": -437.1291809082031,
"debug/reference_chosen_logps": -417.13665771484375,
"debug/reference_rejected_logps": -431.5104064941406,
"epoch": 0.6458333333333334,
"grad_norm": 9.586730066675056,
"learning_rate": 1e-06,
"logits/chosen": 28.232507705688477,
"logits/rejected": 27.216352462768555,
"logps/chosen": -414.2076721191406,
"logps/rejected": -437.1291809082031,
"loss": 0.4718,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.029290199279785156,
"rewards/margins": 0.08547790348529816,
"rewards/rejected": -0.0561877004802227,
"step": 31
},
{
"debug/policy_chosen_logits": 31.265499114990234,
"debug/policy_chosen_logps": -415.2708740234375,
"debug/policy_rejected_logits": 30.586397171020508,
"debug/policy_rejected_logps": -420.286376953125,
"debug/reference_chosen_logps": -415.6477966308594,
"debug/reference_rejected_logps": -419.3028259277344,
"epoch": 0.6666666666666666,
"grad_norm": 5.088803834381108,
"learning_rate": 1e-06,
"logits/chosen": 31.265499114990234,
"logits/rejected": 30.586397171020508,
"logps/chosen": -415.2708740234375,
"logps/rejected": -420.286376953125,
"loss": 0.4781,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.003769032657146454,
"rewards/margins": 0.013604126870632172,
"rewards/rejected": -0.00983509048819542,
"step": 32
},
{
"debug/policy_chosen_logits": 28.746450424194336,
"debug/policy_chosen_logps": -418.0946350097656,
"debug/policy_rejected_logits": 30.699132919311523,
"debug/policy_rejected_logps": -422.5447998046875,
"debug/reference_chosen_logps": -421.5987243652344,
"debug/reference_rejected_logps": -423.47491455078125,
"epoch": 0.6875,
"grad_norm": 5.2883474953854925,
"learning_rate": 1e-06,
"logits/chosen": 28.746450424194336,
"logits/rejected": 30.699132919311523,
"logps/chosen": -418.0946350097656,
"logps/rejected": -422.5447998046875,
"loss": 0.4925,
"rewards/accuracies": 0.5,
"rewards/chosen": 0.03504092991352081,
"rewards/margins": 0.02574004977941513,
"rewards/rejected": 0.009300878271460533,
"step": 33
},
{
"debug/policy_chosen_logits": 32.71680450439453,
"debug/policy_chosen_logps": -405.2165222167969,
"debug/policy_rejected_logits": 29.466472625732422,
"debug/policy_rejected_logps": -449.44921875,
"debug/reference_chosen_logps": -407.29217529296875,
"debug/reference_rejected_logps": -445.6148681640625,
"epoch": 0.7083333333333334,
"grad_norm": 5.307136992049977,
"learning_rate": 1e-06,
"logits/chosen": 32.71680450439453,
"logits/rejected": 29.466472625732422,
"logps/chosen": -405.2165222167969,
"logps/rejected": -449.44921875,
"loss": 0.4894,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.020756645128130913,
"rewards/margins": 0.05910034105181694,
"rewards/rejected": -0.03834369778633118,
"step": 34
},
{
"debug/policy_chosen_logits": 25.695634841918945,
"debug/policy_chosen_logps": -404.29071044921875,
"debug/policy_rejected_logits": 27.330564498901367,
"debug/policy_rejected_logps": -454.2277526855469,
"debug/reference_chosen_logps": -405.4884338378906,
"debug/reference_rejected_logps": -449.79266357421875,
"epoch": 0.7291666666666666,
"grad_norm": 4.968733221673012,
"learning_rate": 1e-06,
"logits/chosen": 25.695634841918945,
"logits/rejected": 27.330564498901367,
"logps/chosen": -404.29071044921875,
"logps/rejected": -454.2277526855469,
"loss": 0.446,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.011977346614003181,
"rewards/margins": 0.05632827803492546,
"rewards/rejected": -0.04435092955827713,
"step": 35
},
{
"debug/policy_chosen_logits": 27.627330780029297,
"debug/policy_chosen_logps": -420.744873046875,
"debug/policy_rejected_logits": 28.073627471923828,
"debug/policy_rejected_logps": -437.855224609375,
"debug/reference_chosen_logps": -425.2474060058594,
"debug/reference_rejected_logps": -429.3760681152344,
"epoch": 0.75,
"grad_norm": 5.074316911246501,
"learning_rate": 1e-06,
"logits/chosen": 27.627330780029297,
"logits/rejected": 28.073627471923828,
"logps/chosen": -420.744873046875,
"logps/rejected": -437.855224609375,
"loss": 0.4719,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.04502537101507187,
"rewards/margins": 0.12981674075126648,
"rewards/rejected": -0.08479136973619461,
"step": 36
},
{
"debug/policy_chosen_logits": 26.058317184448242,
"debug/policy_chosen_logps": -418.7844543457031,
"debug/policy_rejected_logits": 27.601564407348633,
"debug/policy_rejected_logps": -430.4002380371094,
"debug/reference_chosen_logps": -421.88189697265625,
"debug/reference_rejected_logps": -429.6895446777344,
"epoch": 0.7708333333333334,
"grad_norm": 5.110386744264913,
"learning_rate": 1e-06,
"logits/chosen": 26.058317184448242,
"logits/rejected": 27.601564407348633,
"logps/chosen": -418.7844543457031,
"logps/rejected": -430.4002380371094,
"loss": 0.5027,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.030974578112363815,
"rewards/margins": 0.03808154910802841,
"rewards/rejected": -0.0071069709956645966,
"step": 37
},
{
"debug/policy_chosen_logits": 32.79731369018555,
"debug/policy_chosen_logps": -430.58941650390625,
"debug/policy_rejected_logits": 34.05510330200195,
"debug/policy_rejected_logps": -450.1502685546875,
"debug/reference_chosen_logps": -432.3368225097656,
"debug/reference_rejected_logps": -443.3255615234375,
"epoch": 0.7916666666666666,
"grad_norm": 4.973892389169695,
"learning_rate": 1e-06,
"logits/chosen": 32.79731369018555,
"logits/rejected": 34.05510330200195,
"logps/chosen": -430.58941650390625,
"logps/rejected": -450.1502685546875,
"loss": 0.4642,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.017473945394158363,
"rewards/margins": 0.08572139590978622,
"rewards/rejected": -0.06824745237827301,
"step": 38
},
{
"debug/policy_chosen_logits": 32.02721405029297,
"debug/policy_chosen_logps": -420.63739013671875,
"debug/policy_rejected_logits": 32.61655807495117,
"debug/policy_rejected_logps": -422.09625244140625,
"debug/reference_chosen_logps": -425.75048828125,
"debug/reference_rejected_logps": -412.3810729980469,
"epoch": 0.8125,
"grad_norm": 5.161463337049576,
"learning_rate": 1e-06,
"logits/chosen": 32.02721405029297,
"logits/rejected": 32.61655807495117,
"logps/chosen": -420.63739013671875,
"logps/rejected": -422.09625244140625,
"loss": 0.4611,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.0511307530105114,
"rewards/margins": 0.1482827365398407,
"rewards/rejected": -0.0971519872546196,
"step": 39
},
{
"debug/policy_chosen_logits": 30.953702926635742,
"debug/policy_chosen_logps": -415.52679443359375,
"debug/policy_rejected_logits": 30.896102905273438,
"debug/policy_rejected_logps": -432.7928161621094,
"debug/reference_chosen_logps": -418.53839111328125,
"debug/reference_rejected_logps": -433.76123046875,
"epoch": 0.8333333333333334,
"grad_norm": 5.104987202858757,
"learning_rate": 1e-06,
"logits/chosen": 30.953702926635742,
"logits/rejected": 30.896102905273438,
"logps/chosen": -415.52679443359375,
"logps/rejected": -432.7928161621094,
"loss": 0.4728,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.03011600486934185,
"rewards/margins": 0.020431898534297943,
"rewards/rejected": 0.009684105403721333,
"step": 40
},
{
"debug/policy_chosen_logits": 31.386384963989258,
"debug/policy_chosen_logps": -416.5565185546875,
"debug/policy_rejected_logits": 30.050193786621094,
"debug/policy_rejected_logps": -419.0783996582031,
"debug/reference_chosen_logps": -419.27752685546875,
"debug/reference_rejected_logps": -417.3736267089844,
"epoch": 0.8541666666666666,
"grad_norm": 4.8124075843398275,
"learning_rate": 1e-06,
"logits/chosen": 31.386384963989258,
"logits/rejected": 30.050193786621094,
"logps/chosen": -416.5565185546875,
"logps/rejected": -419.0783996582031,
"loss": 0.4822,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.027210043743252754,
"rewards/margins": 0.04425777494907379,
"rewards/rejected": -0.017047729343175888,
"step": 41
},
{
"debug/policy_chosen_logits": 27.5343074798584,
"debug/policy_chosen_logps": -416.6922607421875,
"debug/policy_rejected_logits": 29.844606399536133,
"debug/policy_rejected_logps": -427.3104553222656,
"debug/reference_chosen_logps": -422.4092102050781,
"debug/reference_rejected_logps": -425.1396484375,
"epoch": 0.875,
"grad_norm": 5.155061770516332,
"learning_rate": 1e-06,
"logits/chosen": 27.5343074798584,
"logits/rejected": 29.844606399536133,
"logps/chosen": -416.6922607421875,
"logps/rejected": -427.3104553222656,
"loss": 0.4577,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.05716957151889801,
"rewards/margins": 0.07887779176235199,
"rewards/rejected": -0.02170822024345398,
"step": 42
},
{
"debug/policy_chosen_logits": 27.6116943359375,
"debug/policy_chosen_logps": -421.576416015625,
"debug/policy_rejected_logits": 26.841312408447266,
"debug/policy_rejected_logps": -442.26361083984375,
"debug/reference_chosen_logps": -428.953125,
"debug/reference_rejected_logps": -441.0887756347656,
"epoch": 0.8958333333333334,
"grad_norm": 4.824458757028807,
"learning_rate": 1e-06,
"logits/chosen": 27.6116943359375,
"logits/rejected": 26.841312408447266,
"logps/chosen": -421.576416015625,
"logps/rejected": -442.26361083984375,
"loss": 0.4393,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.07376720011234283,
"rewards/margins": 0.08551543951034546,
"rewards/rejected": -0.01174823846668005,
"step": 43
},
{
"debug/policy_chosen_logits": 28.251888275146484,
"debug/policy_chosen_logps": -413.2276611328125,
"debug/policy_rejected_logits": 27.731847763061523,
"debug/policy_rejected_logps": -431.4485168457031,
"debug/reference_chosen_logps": -421.6210632324219,
"debug/reference_rejected_logps": -428.9674072265625,
"epoch": 0.9166666666666666,
"grad_norm": 4.765787027247756,
"learning_rate": 1e-06,
"logits/chosen": 28.251888275146484,
"logits/rejected": 27.731847763061523,
"logps/chosen": -413.2276611328125,
"logps/rejected": -431.4485168457031,
"loss": 0.4476,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.08393420279026031,
"rewards/margins": 0.10874545574188232,
"rewards/rejected": -0.02481124736368656,
"step": 44
},
{
"debug/policy_chosen_logits": 27.165687561035156,
"debug/policy_chosen_logps": -416.60101318359375,
"debug/policy_rejected_logits": 26.6442928314209,
"debug/policy_rejected_logps": -421.559326171875,
"debug/reference_chosen_logps": -422.03057861328125,
"debug/reference_rejected_logps": -421.613037109375,
"epoch": 0.9375,
"grad_norm": 5.601545923208478,
"learning_rate": 1e-06,
"logits/chosen": 27.165687561035156,
"logits/rejected": 26.6442928314209,
"logps/chosen": -416.60101318359375,
"logps/rejected": -421.559326171875,
"loss": 0.4536,
"rewards/accuracies": 0.75,
"rewards/chosen": 0.05429550260305405,
"rewards/margins": 0.05375823751091957,
"rewards/rejected": 0.0005372613668441772,
"step": 45
},
{
"debug/policy_chosen_logits": 30.84551429748535,
"debug/policy_chosen_logps": -420.135498046875,
"debug/policy_rejected_logits": 31.129411697387695,
"debug/policy_rejected_logps": -423.7474365234375,
"debug/reference_chosen_logps": -425.0477294921875,
"debug/reference_rejected_logps": -426.3204345703125,
"epoch": 0.9583333333333334,
"grad_norm": 5.0077997901080655,
"learning_rate": 1e-06,
"logits/chosen": 30.84551429748535,
"logits/rejected": 31.129411697387695,
"logps/chosen": -420.135498046875,
"logps/rejected": -423.7474365234375,
"loss": 0.4604,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.04912235215306282,
"rewards/margins": 0.02339225634932518,
"rewards/rejected": 0.025730092078447342,
"step": 46
},
{
"debug/policy_chosen_logits": 25.565441131591797,
"debug/policy_chosen_logps": -390.5962829589844,
"debug/policy_rejected_logits": 31.802610397338867,
"debug/policy_rejected_logps": -443.27691650390625,
"debug/reference_chosen_logps": -394.64532470703125,
"debug/reference_rejected_logps": -439.7212219238281,
"epoch": 0.9791666666666666,
"grad_norm": 5.104824777439459,
"learning_rate": 1e-06,
"logits/chosen": 25.565441131591797,
"logits/rejected": 31.802610397338867,
"logps/chosen": -390.5962829589844,
"logps/rejected": -443.27691650390625,
"loss": 0.4627,
"rewards/accuracies": 0.625,
"rewards/chosen": 0.04049030318856239,
"rewards/margins": 0.07604698091745377,
"rewards/rejected": -0.03555667772889137,
"step": 47
},
{
"debug/policy_chosen_logits": 30.14870262145996,
"debug/policy_chosen_logps": -410.57965087890625,
"debug/policy_rejected_logits": 30.24129295349121,
"debug/policy_rejected_logps": -411.0328369140625,
"debug/reference_chosen_logps": -416.5841979980469,
"debug/reference_rejected_logps": -415.076904296875,
"epoch": 1.0,
"grad_norm": 5.41196633275443,
"learning_rate": 1e-06,
"logits/chosen": 30.14870262145996,
"logits/rejected": 30.24129295349121,
"logps/chosen": -410.57965087890625,
"logps/rejected": -411.0328369140625,
"loss": 0.4479,
"rewards/accuracies": 0.375,
"rewards/chosen": 0.06004543602466583,
"rewards/margins": 0.01960449479520321,
"rewards/rejected": 0.040440939366817474,
"step": 48
},
{
"epoch": 1.0,
"step": 48,
"total_flos": 0.0,
"train_loss": 0.4809455207238595,
"train_runtime": 486.3354,
"train_samples_per_second": 6.271,
"train_steps_per_second": 0.099
}
],
"logging_steps": 1,
"max_steps": 48,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 500,
"stateful_callbacks": {
"TrainerControl": {
"args": {
"should_epoch_stop": false,
"should_evaluate": false,
"should_log": false,
"should_save": true,
"should_training_stop": true
},
"attributes": {}
}
},
"total_flos": 0.0,
"train_batch_size": 8,
"trial_name": null,
"trial_params": null
}
|