HuggingFaceH4/zephyr-7b-beta
1.9k61k
1{2 "best_metric": null,3 "best_model_checkpoint": null,4 "epoch": 3.0,5 "eval_steps": 100,6 "global_step": 5811,7 "is_hyper_param_search": false,8 "is_local_process_zero": true,9 "is_world_process_zero": true,10 "log_history": [11 {12 "epoch": 0.0,13 "learning_rate": 8.591065292096219e-10,14 "logits/chosen": -2.5129990577697754,15 "logits/rejected": -2.4275057315826416,16 "logps/chosen": -96.6673583984375,17 "logps/rejected": -105.15755462646484,18 "loss": 0.6931,19 "rewards/accuracies": 0.0,20 "rewards/chosen": 0.0,21 "rewards/margins": 0.0,22 "rewards/rejected": 0.0,23 "step": 124 },25 {26 "epoch": 0.01,27 "learning_rate": 8.59106529209622e-09,28 "logits/chosen": -2.988718271255493,29 "logits/rejected": -2.9780874252319336,30 "logps/chosen": -302.4128723144531,31 "logps/rejected": -225.56951904296875,32 "loss": 0.6947,33 "rewards/accuracies": 0.5,34 "rewards/chosen": -0.007120599504560232,35 "rewards/margins": -0.004252635408192873,36 "rewards/rejected": -0.002867964096367359,37 "step": 1038 },39 {40 "epoch": 0.01,41 "learning_rate": 1.718213058419244e-08,42 "logits/chosen": -2.8921194076538086,43 "logits/rejected": -2.7121551036834717,44 "logps/chosen": -287.7423400878906,45 "logps/rejected": -217.6292724609375,46 "loss": 0.6906,47 "rewards/accuracies": 0.550000011920929,48 "rewards/chosen": 0.0037767409812659025,49 "rewards/margins": 0.010759315453469753,50 "rewards/rejected": -0.006982574705034494,51 "step": 2052 },53 {54 "epoch": 0.02,55 "learning_rate": 2.5773195876288656e-08,56 "logits/chosen": -3.015655994415283,57 "logits/rejected": -2.9962334632873535,58 "logps/chosen": -297.9928283691406,59 "logps/rejected": -203.88180541992188,60 "loss": 0.6891,61 "rewards/accuracies": 0.6499999761581421,62 "rewards/chosen": 0.025857295840978622,63 "rewards/margins": 0.025261688977479935,64 "rewards/rejected": 0.0005956076784059405,65 "step": 3066 },67 {68 "epoch": 0.02,69 "learning_rate": 3.436426116838488e-08,70 "logits/chosen": -2.8478853702545166,71 "logits/rejected": -2.9214625358581543,72 "logps/chosen": -267.7845153808594,73 "logps/rejected": -250.1910400390625,74 "loss": 0.6861,75 "rewards/accuracies": 0.75,76 "rewards/chosen": 0.027253543958067894,77 "rewards/margins": 0.02428315207362175,78 "rewards/rejected": 0.002970390487462282,79 "step": 4080 },81 {82 "epoch": 0.03,83 "learning_rate": 4.29553264604811e-08,84 "logits/chosen": -3.0094895362854004,85 "logits/rejected": -2.9605789184570312,86 "logps/chosen": -322.73681640625,87 "logps/rejected": -245.77450561523438,88 "loss": 0.6839,89 "rewards/accuracies": 0.6499999761581421,90 "rewards/chosen": 0.022152891382575035,91 "rewards/margins": 0.04998321458697319,92 "rewards/rejected": -0.027830326929688454,93 "step": 5094 },95 {96 "epoch": 0.03,97 "learning_rate": 5.154639175257731e-08,98 "logits/chosen": -2.9751992225646973,99 "logits/rejected": -2.9595389366149902,100 "logps/chosen": -308.54351806640625,101 "logps/rejected": -224.53707885742188,102 "loss": 0.6688,103 "rewards/accuracies": 0.75,104 "rewards/chosen": 0.04110954329371452,105 "rewards/margins": 0.11852701753377914,106 "rewards/rejected": -0.07741747796535492,107 "step": 60108 },109 {110 "epoch": 0.04,111 "learning_rate": 6.013745704467354e-08,112 "logits/chosen": -2.9823076725006104,113 "logits/rejected": -3.0206565856933594,114 "logps/chosen": -375.16925048828125,115 "logps/rejected": -224.032958984375,116 "loss": 0.6642,117 "rewards/accuracies": 0.800000011920929,118 "rewards/chosen": 0.07654228806495667,119 "rewards/margins": 0.1424637734889984,120 "rewards/rejected": -0.06592147052288055,121 "step": 70122 },123 {124 "epoch": 0.04,125 "learning_rate": 6.872852233676976e-08,126 "logits/chosen": -3.0697617530822754,127 "logits/rejected": -3.036527156829834,128 "logps/chosen": -353.6755676269531,129 "logps/rejected": -216.9717559814453,130 "loss": 0.6418,131 "rewards/accuracies": 0.699999988079071,132 "rewards/chosen": 0.04413590952754021,133 "rewards/margins": 0.10847017914056778,134 "rewards/rejected": -0.06433425843715668,135 "step": 80136 },137 {138 "epoch": 0.05,139 "learning_rate": 7.731958762886598e-08,140 "logits/chosen": -2.9876997470855713,141 "logits/rejected": -2.9616377353668213,142 "logps/chosen": -291.57012939453125,143 "logps/rejected": -193.0994873046875,144 "loss": 0.6367,145 "rewards/accuracies": 0.800000011920929,146 "rewards/chosen": 0.08899353444576263,147 "rewards/margins": 0.24147820472717285,148 "rewards/rejected": -0.1524846851825714,149 "step": 90150 },151 {152 "epoch": 0.05,153 "learning_rate": 8.59106529209622e-08,154 "logits/chosen": -2.8401777744293213,155 "logits/rejected": -2.7715401649475098,156 "logps/chosen": -261.7100524902344,157 "logps/rejected": -255.4248046875,158 "loss": 0.6284,159 "rewards/accuracies": 0.699999988079071,160 "rewards/chosen": 0.03916650265455246,161 "rewards/margins": 0.1420799195766449,162 "rewards/rejected": -0.10291342437267303,163 "step": 100164 },165 {166 "epoch": 0.05,167 "eval_logits/chosen": -2.823406934738159,168 "eval_logits/rejected": -2.797581672668457,169 "eval_logps/chosen": -253.8098907470703,170 "eval_logps/rejected": -258.8415832519531,171 "eval_loss": 0.6098471879959106,172 "eval_rewards/accuracies": 0.734375,173 "eval_rewards/chosen": 0.04252301901578903,174 "eval_rewards/margins": 0.22968964278697968,175 "eval_rewards/rejected": -0.18716664612293243,176 "eval_runtime": 58.4622,177 "eval_samples_per_second": 17.105,178 "eval_steps_per_second": 0.274,179 "step": 100180 },181 {182 "epoch": 0.06,183 "learning_rate": 9.450171821305841e-08,184 "logits/chosen": -3.0175564289093018,185 "logits/rejected": -3.084195137023926,186 "logps/chosen": -344.5015869140625,187 "logps/rejected": -294.0466613769531,188 "loss": 0.6208,189 "rewards/accuracies": 0.75,190 "rewards/chosen": 0.07540851831436157,191 "rewards/margins": 0.3283361792564392,192 "rewards/rejected": -0.25292766094207764,193 "step": 110194 },195 {196 "epoch": 0.06,197 "learning_rate": 1.0309278350515462e-07,198 "logits/chosen": -2.831212043762207,199 "logits/rejected": -2.7832601070404053,200 "logps/chosen": -184.1349639892578,201 "logps/rejected": -206.84634399414062,202 "loss": 0.587,203 "rewards/accuracies": 0.6499999761581421,204 "rewards/chosen": -0.06816364824771881,205 "rewards/margins": 0.0881614089012146,206 "rewards/rejected": -0.1563250720500946,207 "step": 120208 },209 {210 "epoch": 0.07,211 "learning_rate": 1.1168384879725086e-07,212 "logits/chosen": -3.0061099529266357,213 "logits/rejected": -2.8498525619506836,214 "logps/chosen": -333.06072998046875,215 "logps/rejected": -189.4818115234375,216 "loss": 0.5832,217 "rewards/accuracies": 0.800000011920929,218 "rewards/chosen": 0.19702570140361786,219 "rewards/margins": 0.5247530341148376,220 "rewards/rejected": -0.3277273178100586,221 "step": 130222 },223 {224 "epoch": 0.07,225 "learning_rate": 1.202749140893471e-07,226 "logits/chosen": -2.8609023094177246,227 "logits/rejected": -2.77339243888855,228 "logps/chosen": -297.0363464355469,229 "logps/rejected": -242.37255859375,230 "loss": 0.5649,231 "rewards/accuracies": 0.699999988079071,232 "rewards/chosen": 0.016815107315778732,233 "rewards/margins": 0.422064870595932,234 "rewards/rejected": -0.40524977445602417,235 "step": 140236 },237 {238 "epoch": 0.08,239 "learning_rate": 1.2886597938144328e-07,240 "logits/chosen": -3.088327407836914,241 "logits/rejected": -2.9465346336364746,242 "logps/chosen": -305.6724548339844,243 "logps/rejected": -314.7848205566406,244 "loss": 0.5548,245 "rewards/accuracies": 0.8999999761581421,246 "rewards/chosen": 0.12263361364603043,247 "rewards/margins": 0.6976320147514343,248 "rewards/rejected": -0.5749984979629517,249 "step": 150250 },251 {252 "epoch": 0.08,253 "learning_rate": 1.3745704467353952e-07,254 "logits/chosen": -2.883831024169922,255 "logits/rejected": -2.8376777172088623,256 "logps/chosen": -267.89154052734375,257 "logps/rejected": -199.8636474609375,258 "loss": 0.5362,259 "rewards/accuracies": 0.6499999761581421,260 "rewards/chosen": 0.17647962272167206,261 "rewards/margins": 0.5636450052261353,262 "rewards/rejected": -0.387165367603302,263 "step": 160264 },265 {266 "epoch": 0.09,267 "learning_rate": 1.4604810996563573e-07,268 "logits/chosen": -2.823948383331299,269 "logits/rejected": -2.7283661365509033,270 "logps/chosen": -234.5882568359375,271 "logps/rejected": -194.86480712890625,272 "loss": 0.4987,273 "rewards/accuracies": 0.75,274 "rewards/chosen": 0.09966392815113068,275 "rewards/margins": 0.7896274328231812,276 "rewards/rejected": -0.6899635791778564,277 "step": 170278 },279 {280 "epoch": 0.09,281 "learning_rate": 1.5463917525773197e-07,282 "logits/chosen": -2.9629111289978027,283 "logits/rejected": -2.9428882598876953,284 "logps/chosen": -232.97244262695312,285 "logps/rejected": -183.2829132080078,286 "loss": 0.5185,287 "rewards/accuracies": 0.800000011920929,288 "rewards/chosen": 0.2645714282989502,289 "rewards/margins": 0.8501029014587402,290 "rewards/rejected": -0.5855314135551453,291 "step": 180292 },293 {294 "epoch": 0.1,295 "learning_rate": 1.6323024054982818e-07,296 "logits/chosen": -2.9642796516418457,297 "logits/rejected": -2.97268009185791,298 "logps/chosen": -275.6226501464844,299 "logps/rejected": -233.35537719726562,300 "loss": 0.5748,301 "rewards/accuracies": 0.75,302 "rewards/chosen": 0.1616288721561432,303 "rewards/margins": 0.4936322569847107,304 "rewards/rejected": -0.3320034146308899,305 "step": 190306 },307 {308 "epoch": 0.1,309 "learning_rate": 1.718213058419244e-07,310 "logits/chosen": -2.9080729484558105,311 "logits/rejected": -2.9043314456939697,312 "logps/chosen": -282.22369384765625,313 "logps/rejected": -235.44992065429688,314 "loss": 0.4908,315 "rewards/accuracies": 0.800000011920929,316 "rewards/chosen": 0.13387183845043182,317 "rewards/margins": 0.7095439434051514,318 "rewards/rejected": -0.5756720900535583,319 "step": 200320 },321 {322 "epoch": 0.1,323 "eval_logits/chosen": -2.7959609031677246,324 "eval_logits/rejected": -2.7718665599823,325 "eval_logps/chosen": -254.51446533203125,326 "eval_logps/rejected": -263.8123779296875,327 "eval_loss": 0.5425560474395752,328 "eval_rewards/accuracies": 0.75,329 "eval_rewards/chosen": -0.027933437377214432,330 "eval_rewards/margins": 0.6563125252723694,331 "eval_rewards/rejected": -0.6842460036277771,332 "eval_runtime": 58.0136,333 "eval_samples_per_second": 17.237,334 "eval_steps_per_second": 0.276,335 "step": 200336 },337 {338 "epoch": 0.11,339 "learning_rate": 1.804123711340206e-07,340 "logits/chosen": -2.664795160293579,341 "logits/rejected": -2.427393674850464,342 "logps/chosen": -297.56488037109375,343 "logps/rejected": -226.8320770263672,344 "loss": 0.5682,345 "rewards/accuracies": 0.800000011920929,346 "rewards/chosen": -0.22673270106315613,347 "rewards/margins": 0.919518768787384,348 "rewards/rejected": -1.1462514400482178,349 "step": 210350 },351 {352 "epoch": 0.11,353 "learning_rate": 1.8900343642611682e-07,354 "logits/chosen": -2.8164966106414795,355 "logits/rejected": -2.7533140182495117,356 "logps/chosen": -316.3358459472656,357 "logps/rejected": -248.8792724609375,358 "loss": 0.5418,359 "rewards/accuracies": 0.75,360 "rewards/chosen": -0.2400936633348465,361 "rewards/margins": 0.5202454924583435,362 "rewards/rejected": -0.7603391408920288,363 "step": 220364 },365 {366 "epoch": 0.12,367 "learning_rate": 1.9759450171821303e-07,368 "logits/chosen": -2.8455495834350586,369 "logits/rejected": -2.815950870513916,370 "logps/chosen": -291.536376953125,371 "logps/rejected": -252.3511199951172,372 "loss": 0.516,373 "rewards/accuracies": 0.800000011920929,374 "rewards/chosen": -0.05435393005609512,375 "rewards/margins": 0.677357017993927,376 "rewards/rejected": -0.7317109107971191,377 "step": 230378 },379 {380 "epoch": 0.12,381 "learning_rate": 2.0618556701030925e-07,382 "logits/chosen": -2.884962558746338,383 "logits/rejected": -2.9899585247039795,384 "logps/chosen": -362.83612060546875,385 "logps/rejected": -246.82815551757812,386 "loss": 0.5416,387 "rewards/accuracies": 0.5,388 "rewards/chosen": -0.4428789019584656,389 "rewards/margins": 0.2312956303358078,390 "rewards/rejected": -0.6741746068000793,391 "step": 240392 },393 {394 "epoch": 0.13,395 "learning_rate": 2.1477663230240549e-07,396 "logits/chosen": -2.979492425918579,397 "logits/rejected": -2.9899439811706543,398 "logps/chosen": -232.15756225585938,399 "logps/rejected": -157.3478240966797,400 "loss": 0.5141,401 "rewards/accuracies": 0.8999999761581421,402 "rewards/chosen": 0.26762503385543823,403 "rewards/margins": 1.1515061855316162,404 "rewards/rejected": -0.8838812112808228,405 "step": 250406 },407 {408 "epoch": 0.13,409 "learning_rate": 2.2336769759450173e-07,410 "logits/chosen": -3.0052077770233154,411 "logits/rejected": -2.9878716468811035,412 "logps/chosen": -309.3619689941406,413 "logps/rejected": -189.45968627929688,414 "loss": 0.5321,415 "rewards/accuracies": 0.75,416 "rewards/chosen": 0.4304015040397644,417 "rewards/margins": 1.0413486957550049,418 "rewards/rejected": -0.6109471917152405,419 "step": 260420 },421 {422 "epoch": 0.14,423 "learning_rate": 2.3195876288659794e-07,424 "logits/chosen": -2.8794291019439697,425 "logits/rejected": -2.831512928009033,426 "logps/chosen": -306.6054992675781,427 "logps/rejected": -237.39382934570312,428 "loss": 0.4909,429 "rewards/accuracies": 0.6000000238418579,430 "rewards/chosen": 0.18433420360088348,431 "rewards/margins": 0.8596351742744446,432 "rewards/rejected": -0.6753008365631104,433 "step": 270434 },435 {436 "epoch": 0.14,437 "learning_rate": 2.405498281786942e-07,438 "logits/chosen": -2.9824016094207764,439 "logits/rejected": -2.9367408752441406,440 "logps/chosen": -350.133056640625,441 "logps/rejected": -254.4954071044922,442 "loss": 0.5336,443 "rewards/accuracies": 0.699999988079071,444 "rewards/chosen": 0.20658831298351288,445 "rewards/margins": 0.7473801374435425,446 "rewards/rejected": -0.5407918691635132,447 "step": 280448 },449 {450 "epoch": 0.15,451 "learning_rate": 2.4914089347079036e-07,452 "logits/chosen": -2.7565178871154785,453 "logits/rejected": -2.944960832595825,454 "logps/chosen": -242.48397827148438,455 "logps/rejected": -227.69107055664062,456 "loss": 0.5056,457 "rewards/accuracies": 0.6000000238418579,458 "rewards/chosen": 0.699475109577179,459 "rewards/margins": 1.3677313327789307,460 "rewards/rejected": -0.6682561635971069,461 "step": 290462 },463 {464 "epoch": 0.15,465 "learning_rate": 2.5773195876288655e-07,466 "logits/chosen": -2.779125928878784,467 "logits/rejected": -2.943162679672241,468 "logps/chosen": -411.8221130371094,469 "logps/rejected": -222.3397216796875,470 "loss": 0.5264,471 "rewards/accuracies": 0.699999988079071,472 "rewards/chosen": 0.34889036417007446,473 "rewards/margins": 1.2339386940002441,474 "rewards/rejected": -0.8850483894348145,475 "step": 300476 },477 {478 "epoch": 0.15,479 "eval_logits/chosen": -2.812185525894165,480 "eval_logits/rejected": -2.789177656173706,481 "eval_logps/chosen": -253.82086181640625,482 "eval_logps/rejected": -266.7626953125,483 "eval_loss": 0.5323615670204163,484 "eval_rewards/accuracies": 0.765625,485 "eval_rewards/chosen": 0.04142449051141739,486 "eval_rewards/margins": 1.0207018852233887,487 "eval_rewards/rejected": -0.9792775511741638,488 "eval_runtime": 59.6543,489 "eval_samples_per_second": 16.763,490 "eval_steps_per_second": 0.268,491 "step": 300492 },493 {494 "epoch": 0.16,495 "learning_rate": 2.663230240549828e-07,496 "logits/chosen": -2.9955785274505615,497 "logits/rejected": -2.9795451164245605,498 "logps/chosen": -318.2289123535156,499 "logps/rejected": -216.7342071533203,500 "loss": 0.5172,501 "rewards/accuracies": 0.800000011920929,502 "rewards/chosen": 0.10595469176769257,503 "rewards/margins": 1.0604875087738037,504 "rewards/rejected": -0.9545329213142395,505 "step": 310506 },507 {508 "epoch": 0.17,509 "learning_rate": 2.7491408934707903e-07,510 "logits/chosen": -2.9341654777526855,511 "logits/rejected": -2.9966204166412354,512 "logps/chosen": -379.42572021484375,513 "logps/rejected": -282.17291259765625,514 "loss": 0.4682,515 "rewards/accuracies": 0.8500000238418579,516 "rewards/chosen": 0.08291205763816833,517 "rewards/margins": 1.1605613231658936,518 "rewards/rejected": -1.0776493549346924,519 "step": 320520 },521 {522 "epoch": 0.17,523 "learning_rate": 2.835051546391752e-07,524 "logits/chosen": -2.8474197387695312,525 "logits/rejected": -2.844364643096924,526 "logps/chosen": -331.90802001953125,527 "logps/rejected": -223.3518524169922,528 "loss": 0.4881,529 "rewards/accuracies": 0.8500000238418579,530 "rewards/chosen": -0.02964567206799984,531 "rewards/margins": 1.4632409811019897,532 "rewards/rejected": -1.4928867816925049,533 "step": 330534 },535 {536 "epoch": 0.18,537 "learning_rate": 2.9209621993127146e-07,538 "logits/chosen": -2.68745756149292,539 "logits/rejected": -2.817155361175537,540 "logps/chosen": -232.6031036376953,541 "logps/rejected": -246.84768676757812,542 "loss": 0.5196,543 "rewards/accuracies": 0.699999988079071,544 "rewards/chosen": 0.5186244249343872,545 "rewards/margins": 1.0158860683441162,546 "rewards/rejected": -0.49726182222366333,547 "step": 340548 },549 {550 "epoch": 0.18,551 "learning_rate": 3.006872852233677e-07,552 "logits/chosen": -3.0304269790649414,553 "logits/rejected": -2.9698691368103027,554 "logps/chosen": -159.0189208984375,555 "logps/rejected": -212.7183380126953,556 "loss": 0.4873,557 "rewards/accuracies": 0.6499999761581421,558 "rewards/chosen": -0.310208797454834,559 "rewards/margins": 0.5256294012069702,560 "rewards/rejected": -0.8358383178710938,561 "step": 350562 },563 {564 "epoch": 0.19,565 "learning_rate": 3.0927835051546394e-07,566 "logits/chosen": -3.0391955375671387,567 "logits/rejected": -3.0694854259490967,568 "logps/chosen": -381.39715576171875,569 "logps/rejected": -347.92559814453125,570 "loss": 0.4607,571 "rewards/accuracies": 0.8999999761581421,572 "rewards/chosen": 0.43349432945251465,573 "rewards/margins": 1.4752476215362549,574 "rewards/rejected": -1.0417532920837402,575 "step": 360576 },577 {578 "epoch": 0.19,579 "learning_rate": 3.178694158075601e-07,580 "logits/chosen": -2.985565662384033,581 "logits/rejected": -2.951699733734131,582 "logps/chosen": -158.28598022460938,583 "logps/rejected": -127.53106689453125,584 "loss": 0.4825,585 "rewards/accuracies": 0.8500000238418579,586 "rewards/chosen": -0.025753701105713844,587 "rewards/margins": 1.3051038980484009,588 "rewards/rejected": -1.330857515335083,589 "step": 370590 },591 {592 "epoch": 0.2,593 "learning_rate": 3.2646048109965636e-07,594 "logits/chosen": -2.8883204460144043,595 "logits/rejected": -2.7797765731811523,596 "logps/chosen": -300.84283447265625,597 "logps/rejected": -306.0265197753906,598 "loss": 0.5606,599 "rewards/accuracies": 0.6499999761581421,600 "rewards/chosen": 0.03248428553342819,601 "rewards/margins": 0.8653362393379211,602 "rewards/rejected": -0.8328520655632019,603 "step": 380604 },605 {606 "epoch": 0.2,607 "learning_rate": 3.3505154639175255e-07,608 "logits/chosen": -2.8869693279266357,609 "logits/rejected": -2.8558154106140137,610 "logps/chosen": -295.94268798828125,611 "logps/rejected": -245.67544555664062,612 "loss": 0.4789,613 "rewards/accuracies": 0.75,614 "rewards/chosen": 0.0472743920981884,615 "rewards/margins": 1.3127429485321045,616 "rewards/rejected": -1.2654683589935303,617 "step": 390618 },619 {620 "epoch": 0.21,621 "learning_rate": 3.436426116838488e-07,622 "logits/chosen": -3.0791659355163574,623 "logits/rejected": -3.0203123092651367,624 "logps/chosen": -251.6421661376953,625 "logps/rejected": -219.4331512451172,626 "loss": 0.5536,627 "rewards/accuracies": 0.6000000238418579,628 "rewards/chosen": -0.3093084990978241,629 "rewards/margins": 0.4000861644744873,630 "rewards/rejected": -0.7093946933746338,631 "step": 400632 },633 {634 "epoch": 0.21,635 "eval_logits/chosen": -2.8764305114746094,636 "eval_logits/rejected": -2.8541693687438965,637 "eval_logps/chosen": -254.42034912109375,638 "eval_logps/rejected": -272.24603271484375,639 "eval_loss": 0.4957379102706909,640 "eval_rewards/accuracies": 0.796875,641 "eval_rewards/chosen": -0.01852385140955448,642 "eval_rewards/margins": 1.5090851783752441,643 "eval_rewards/rejected": -1.5276089906692505,644 "eval_runtime": 56.3835,645 "eval_samples_per_second": 17.736,646 "eval_steps_per_second": 0.284,647 "step": 400648 },649 {650 "epoch": 0.21,651 "learning_rate": 3.5223367697594503e-07,652 "logits/chosen": -2.8821568489074707,653 "logits/rejected": -2.8333544731140137,654 "logps/chosen": -320.3736267089844,655 "logps/rejected": -205.11056518554688,656 "loss": 0.4277,657 "rewards/accuracies": 0.800000011920929,658 "rewards/chosen": 0.0024402737617492676,659 "rewards/margins": 1.244533658027649,660 "rewards/rejected": -1.2420933246612549,661 "step": 410662 },663 {664 "epoch": 0.22,665 "learning_rate": 3.608247422680412e-07,666 "logits/chosen": -2.9139723777770996,667 "logits/rejected": -2.8578293323516846,668 "logps/chosen": -301.2723693847656,669 "logps/rejected": -248.91744995117188,670 "loss": 0.5208,671 "rewards/accuracies": 0.699999988079071,672 "rewards/chosen": -0.27292880415916443,673 "rewards/margins": 0.9970871210098267,674 "rewards/rejected": -1.2700159549713135,675 "step": 420676 },677 {678 "epoch": 0.22,679 "learning_rate": 3.6941580756013745e-07,680 "logits/chosen": -2.9251325130462646,681 "logits/rejected": -2.8964738845825195,682 "logps/chosen": -210.9687042236328,683 "logps/rejected": -185.3360137939453,684 "loss": 0.5229,685 "rewards/accuracies": 0.75,686 "rewards/chosen": -0.43344053626060486,687 "rewards/margins": 0.8561422228813171,688 "rewards/rejected": -1.2895828485488892,689 "step": 430690 },691 {692 "epoch": 0.23,693 "learning_rate": 3.7800687285223364e-07,694 "logits/chosen": -2.901094436645508,695 "logits/rejected": -2.8542165756225586,696 "logps/chosen": -348.6666259765625,697 "logps/rejected": -329.27294921875,698 "loss": 0.6022,699 "rewards/accuracies": 0.800000011920929,700 "rewards/chosen": 0.014251199550926685,701 "rewards/margins": 0.753572940826416,702 "rewards/rejected": -0.7393215298652649,703 "step": 440704 },705 {706 "epoch": 0.23,707 "learning_rate": 3.865979381443299e-07,708 "logits/chosen": -2.936382532119751,709 "logits/rejected": -2.9940216541290283,710 "logps/chosen": -308.2112731933594,711 "logps/rejected": -232.1812744140625,712 "loss": 0.504,713 "rewards/accuracies": 0.44999998807907104,714 "rewards/chosen": -0.3824332356452942,715 "rewards/margins": 0.28821295499801636,716 "rewards/rejected": -0.6706462502479553,717 "step": 450718 },719 {720 "epoch": 0.24,721 "learning_rate": 3.9518900343642607e-07,722 "logits/chosen": -2.951936960220337,723 "logits/rejected": -3.0050208568573,724 "logps/chosen": -326.07659912109375,725 "logps/rejected": -301.6195983886719,726 "loss": 0.5801,727 "rewards/accuracies": 0.75,728 "rewards/chosen": 0.1851659119129181,729 "rewards/margins": 1.5346710681915283,730 "rewards/rejected": -1.349505066871643,731 "step": 460732 },733 {734 "epoch": 0.24,735 "learning_rate": 4.037800687285223e-07,736 "logits/chosen": -2.8154656887054443,737 "logits/rejected": -2.8765406608581543,738 "logps/chosen": -320.0531311035156,739 "logps/rejected": -226.99124145507812,740 "loss": 0.4564,741 "rewards/accuracies": 0.6000000238418579,742 "rewards/chosen": -0.22784185409545898,743 "rewards/margins": 0.7857998013496399,744 "rewards/rejected": -1.013641595840454,745 "step": 470746 },747 {748 "epoch": 0.25,749 "learning_rate": 4.123711340206185e-07,750 "logits/chosen": -3.0402634143829346,751 "logits/rejected": -3.021247625350952,752 "logps/chosen": -284.4671936035156,753 "logps/rejected": -287.5126647949219,754 "loss": 0.4915,755 "rewards/accuracies": 0.5,756 "rewards/chosen": -0.03487253934144974,757 "rewards/margins": 0.45380640029907227,758 "rewards/rejected": -0.4886789321899414,759 "step": 480760 },761 {762 "epoch": 0.25,763 "learning_rate": 4.209621993127148e-07,764 "logits/chosen": -2.9354074001312256,765 "logits/rejected": -2.920379638671875,766 "logps/chosen": -311.0786437988281,767 "logps/rejected": -246.13339233398438,768 "loss": 0.4588,769 "rewards/accuracies": 0.75,770 "rewards/chosen": -0.48857221007347107,771 "rewards/margins": 0.9489312171936035,772 "rewards/rejected": -1.4375033378601074,773 "step": 490774 },775 {776 "epoch": 0.26,777 "learning_rate": 4.2955326460481097e-07,778 "logits/chosen": -3.123109817504883,779 "logits/rejected": -3.0762407779693604,780 "logps/chosen": -309.3453063964844,781 "logps/rejected": -281.5166015625,782 "loss": 0.5362,783 "rewards/accuracies": 0.75,784 "rewards/chosen": 0.25006183981895447,785 "rewards/margins": 1.4525038003921509,786 "rewards/rejected": -1.202441930770874,787 "step": 500788 },789 {790 "epoch": 0.26,791 "eval_logits/chosen": -2.8957557678222656,792 "eval_logits/rejected": -2.87016224861145,793 "eval_logps/chosen": -256.86529541015625,794 "eval_logps/rejected": -272.88690185546875,795 "eval_loss": 0.503109335899353,796 "eval_rewards/accuracies": 0.78125,797 "eval_rewards/chosen": -0.2630198001861572,798 "eval_rewards/margins": 1.3286765813827515,799 "eval_rewards/rejected": -1.5916962623596191,800 "eval_runtime": 55.3853,801 "eval_samples_per_second": 18.055,802 "eval_steps_per_second": 0.289,803 "step": 500804 },805 {806 "epoch": 0.26,807 "learning_rate": 4.381443298969072e-07,808 "logits/chosen": -2.6036550998687744,809 "logits/rejected": -2.6383635997772217,810 "logps/chosen": -252.81375122070312,811 "logps/rejected": -243.0044708251953,812 "loss": 0.5633,813 "rewards/accuracies": 0.6499999761581421,814 "rewards/chosen": -0.6361426115036011,815 "rewards/margins": 0.3173540532588959,816 "rewards/rejected": -0.9534965753555298,817 "step": 510818 },819 {820 "epoch": 0.27,821 "learning_rate": 4.4673539518900345e-07,822 "logits/chosen": -2.96708607673645,823 "logits/rejected": -3.0084481239318848,824 "logps/chosen": -186.54592895507812,825 "logps/rejected": -187.34884643554688,826 "loss": 0.5443,827 "rewards/accuracies": 0.8999999761581421,828 "rewards/chosen": -0.1140596866607666,829 "rewards/margins": 1.242305874824524,830 "rewards/rejected": -1.356365442276001,831 "step": 520832 },833 {834 "epoch": 0.27,835 "learning_rate": 4.5532646048109964e-07,836 "logits/chosen": -2.987997531890869,837 "logits/rejected": -2.9607906341552734,838 "logps/chosen": -292.63690185546875,839 "logps/rejected": -240.1947479248047,840 "loss": 0.5091,841 "rewards/accuracies": 0.8500000238418579,842 "rewards/chosen": 0.1345333755016327,843 "rewards/margins": 1.3595573902130127,844 "rewards/rejected": -1.2250239849090576,845 "step": 530846 },847 {848 "epoch": 0.28,849 "learning_rate": 4.639175257731959e-07,850 "logits/chosen": -2.8647749423980713,851 "logits/rejected": -2.868330955505371,852 "logps/chosen": -186.86167907714844,853 "logps/rejected": -243.17910766601562,854 "loss": 0.6201,855 "rewards/accuracies": 0.6000000238418579,856 "rewards/chosen": -0.7495313882827759,857 "rewards/margins": 0.6188509464263916,858 "rewards/rejected": -1.3683823347091675,859 "step": 540860 },861 {862 "epoch": 0.28,863 "learning_rate": 4.7250859106529206e-07,864 "logits/chosen": -2.9979634284973145,865 "logits/rejected": -2.9638993740081787,866 "logps/chosen": -349.7961730957031,867 "logps/rejected": -288.20062255859375,868 "loss": 0.6153,869 "rewards/accuracies": 0.800000011920929,870 "rewards/chosen": -0.04092409461736679,871 "rewards/margins": 1.4131947755813599,872 "rewards/rejected": -1.4541189670562744,873 "step": 550874 },875 {876 "epoch": 0.29,877 "learning_rate": 4.810996563573884e-07,878 "logits/chosen": -3.0216901302337646,879 "logits/rejected": -3.0451061725616455,880 "logps/chosen": -326.0102844238281,881 "logps/rejected": -307.83367919921875,882 "loss": 0.539,883 "rewards/accuracies": 0.800000011920929,884 "rewards/chosen": -0.6230143904685974,885 "rewards/margins": 0.9213398098945618,886 "rewards/rejected": -1.5443540811538696,887 "step": 560888 },889 {890 "epoch": 0.29,891 "learning_rate": 4.896907216494845e-07,892 "logits/chosen": -2.990562677383423,893 "logits/rejected": -2.9301705360412598,894 "logps/chosen": -336.96826171875,895 "logps/rejected": -225.23599243164062,896 "loss": 0.5141,897 "rewards/accuracies": 0.75,898 "rewards/chosen": -0.8188888430595398,899 "rewards/margins": 1.2901289463043213,900 "rewards/rejected": -2.109017848968506,901 "step": 570902 },903 {904 "epoch": 0.3,905 "learning_rate": 4.982817869415807e-07,906 "logits/chosen": -2.900038003921509,907 "logits/rejected": -2.9860446453094482,908 "logps/chosen": -322.78240966796875,909 "logps/rejected": -231.99667358398438,910 "loss": 0.5668,911 "rewards/accuracies": 0.6000000238418579,912 "rewards/chosen": -0.6513129472732544,913 "rewards/margins": 0.49615031480789185,914 "rewards/rejected": -1.147463321685791,915 "step": 580916 },917 {918 "epoch": 0.3,919 "learning_rate": 4.992350353796136e-07,920 "logits/chosen": -2.936190605163574,921 "logits/rejected": -2.8864665031433105,922 "logps/chosen": -251.67172241210938,923 "logps/rejected": -241.36318969726562,924 "loss": 0.4964,925 "rewards/accuracies": 0.8999999761581421,926 "rewards/chosen": 0.027965422719717026,927 "rewards/margins": 1.3970218896865845,928 "rewards/rejected": -1.3690563440322876,929 "step": 590930 },931 {932 "epoch": 0.31,933 "learning_rate": 4.982788296041308e-07,934 "logits/chosen": -2.9233040809631348,935 "logits/rejected": -2.961263656616211,936 "logps/chosen": -199.49600219726562,937 "logps/rejected": -255.68612670898438,938 "loss": 0.5966,939 "rewards/accuracies": 0.75,940 "rewards/chosen": -0.13642588257789612,941 "rewards/margins": 1.8362632989883423,942 "rewards/rejected": -1.9726893901824951,943 "step": 600944 },945 {946 "epoch": 0.31,947 "eval_logits/chosen": -2.8986048698425293,948 "eval_logits/rejected": -2.8777544498443604,949 "eval_logps/chosen": -257.2279357910156,950 "eval_logps/rejected": -273.46136474609375,951 "eval_loss": 0.5963188409805298,952 "eval_rewards/accuracies": 0.78125,953 "eval_rewards/chosen": -0.29928162693977356,954 "eval_rewards/margins": 1.349860429763794,955 "eval_rewards/rejected": -1.6491420269012451,956 "eval_runtime": 54.6151,957 "eval_samples_per_second": 18.31,958 "eval_steps_per_second": 0.293,959 "step": 600960 },961 {962 "epoch": 0.31,963 "learning_rate": 4.973226238286479e-07,964 "logits/chosen": -2.9391376972198486,965 "logits/rejected": -2.9422051906585693,966 "logps/chosen": -302.2151184082031,967 "logps/rejected": -287.15606689453125,968 "loss": 0.5368,969 "rewards/accuracies": 0.699999988079071,970 "rewards/chosen": -0.4417892098426819,971 "rewards/margins": 1.2578237056732178,972 "rewards/rejected": -1.6996129751205444,973 "step": 610974 },975 {976 "epoch": 0.32,977 "learning_rate": 4.96366418053165e-07,978 "logits/chosen": -3.0650055408477783,979 "logits/rejected": -3.0660297870635986,980 "logps/chosen": -334.0442810058594,981 "logps/rejected": -252.632080078125,982 "loss": 0.5857,983 "rewards/accuracies": 0.75,984 "rewards/chosen": 0.07565183192491531,985 "rewards/margins": 0.7010248899459839,986 "rewards/rejected": -0.6253730654716492,987 "step": 620988 },989 {990 "epoch": 0.33,991 "learning_rate": 4.954102122776821e-07,992 "logits/chosen": -2.918349027633667,993 "logits/rejected": -2.8907716274261475,994 "logps/chosen": -195.51907348632812,995 "logps/rejected": -167.48745727539062,996 "loss": 0.5561,997 "rewards/accuracies": 0.75,998 "rewards/chosen": -0.20618407428264618,999 "rewards/margins": 1.305176019668579,1000 "rewards/rejected": -1.5113601684570312,1001 "step": 6301002 },1003 {1004 "epoch": 0.33,1005 "learning_rate": 4.944540065021993e-07,1006 "logits/chosen": -2.8896799087524414,1007 "logits/rejected": -2.9980359077453613,1008 "logps/chosen": -264.43023681640625,1009 "logps/rejected": -230.43008422851562,1010 "loss": 0.5386,1011 "rewards/accuracies": 0.8999999761581421,1012 "rewards/chosen": -0.18526920676231384,1013 "rewards/margins": 1.8127784729003906,1014 "rewards/rejected": -1.9980475902557373,1015 "step": 6401016 },1017 {1018 "epoch": 0.34,1019 "learning_rate": 4.934978007267163e-07,1020 "logits/chosen": -2.959494113922119,1021 "logits/rejected": -2.982419967651367,1022 "logps/chosen": -242.4766082763672,1023 "logps/rejected": -274.0234680175781,1024 "loss": 0.5399,1025 "rewards/accuracies": 0.75,1026 "rewards/chosen": -0.2960631847381592,1027 "rewards/margins": 1.0227611064910889,1028 "rewards/rejected": -1.318824291229248,1029 "step": 6501030 },1031 {1032 "epoch": 0.34,1033 "learning_rate": 4.925415949512335e-07,1034 "logits/chosen": -2.9986279010772705,1035 "logits/rejected": -2.966939926147461,1036 "logps/chosen": -328.2708435058594,1037 "logps/rejected": -273.13006591796875,1038 "loss": 0.5061,1039 "rewards/accuracies": 0.75,1040 "rewards/chosen": -0.47610530257225037,1041 "rewards/margins": 1.19678795337677,1042 "rewards/rejected": -1.6728931665420532,1043 "step": 6601044 },1045 {1046 "epoch": 0.35,1047 "learning_rate": 4.915853891757506e-07,1048 "logits/chosen": -2.902583360671997,1049 "logits/rejected": -2.941610336303711,1050 "logps/chosen": -197.52853393554688,1051 "logps/rejected": -190.82029724121094,1052 "loss": 0.6096,1053 "rewards/accuracies": 0.6000000238418579,1054 "rewards/chosen": -0.34461653232574463,1055 "rewards/margins": 0.8742098808288574,1056 "rewards/rejected": -1.2188262939453125,1057 "step": 6701058 },1059 {1060 "epoch": 0.35,1061 "learning_rate": 4.906291834002677e-07,1062 "logits/chosen": -2.8424625396728516,1063 "logits/rejected": -2.791315793991089,1064 "logps/chosen": -273.73455810546875,1065 "logps/rejected": -229.92031860351562,1066 "loss": 0.5023,1067 "rewards/accuracies": 0.699999988079071,1068 "rewards/chosen": -0.6406866312026978,1069 "rewards/margins": 0.9865404367446899,1070 "rewards/rejected": -1.6272270679473877,1071 "step": 6801072 },1073 {1074 "epoch": 0.36,1075 "learning_rate": 4.896729776247848e-07,1076 "logits/chosen": -3.039944648742676,1077 "logits/rejected": -2.9114279747009277,1078 "logps/chosen": -344.2494201660156,1079 "logps/rejected": -179.3026580810547,1080 "loss": 0.4748,1081 "rewards/accuracies": 0.8500000238418579,1082 "rewards/chosen": 0.01536635123193264,1083 "rewards/margins": 1.3766069412231445,1084 "rewards/rejected": -1.3612406253814697,1085 "step": 6901086 },1087 {1088 "epoch": 0.36,1089 "learning_rate": 4.88716771849302e-07,1090 "logits/chosen": -2.9146389961242676,1091 "logits/rejected": -2.918255090713501,1092 "logps/chosen": -441.6365661621094,1093 "logps/rejected": -344.4063720703125,1094 "loss": 0.5014,1095 "rewards/accuracies": 0.75,1096 "rewards/chosen": -0.2016751766204834,1097 "rewards/margins": 1.255171775817871,1098 "rewards/rejected": -1.456847071647644,1099 "step": 7001100 },1101 {1102 "epoch": 0.36,1103 "eval_logits/chosen": -2.7868502140045166,1104 "eval_logits/rejected": -2.7659108638763428,1105 "eval_logps/chosen": -257.09423828125,1106 "eval_logps/rejected": -271.72039794921875,1107 "eval_loss": 0.5382026433944702,1108 "eval_rewards/accuracies": 0.75,1109 "eval_rewards/chosen": -0.28591296076774597,1110 "eval_rewards/margins": 1.189131736755371,1111 "eval_rewards/rejected": -1.4750447273254395,1112 "eval_runtime": 57.4875,1113 "eval_samples_per_second": 17.395,1114 "eval_steps_per_second": 0.278,1115 "step": 7001116 },1117 {1118 "epoch": 0.37,1119 "learning_rate": 4.87760566073819e-07,1120 "logits/chosen": -2.8412322998046875,1121 "logits/rejected": -2.9222323894500732,1122 "logps/chosen": -265.5148010253906,1123 "logps/rejected": -250.9593963623047,1124 "loss": 0.5254,1125 "rewards/accuracies": 0.75,1126 "rewards/chosen": -0.7364897131919861,1127 "rewards/margins": 1.0006908178329468,1128 "rewards/rejected": -1.7371807098388672,1129 "step": 7101130 },1131 {1132 "epoch": 0.37,1133 "learning_rate": 4.868043602983362e-07,1134 "logits/chosen": -2.9580254554748535,1135 "logits/rejected": -2.9545352458953857,1136 "logps/chosen": -275.90625,1137 "logps/rejected": -375.8464660644531,1138 "loss": 0.5059,1139 "rewards/accuracies": 0.8999999761581421,1140 "rewards/chosen": -0.17938612401485443,1141 "rewards/margins": 2.483105421066284,1142 "rewards/rejected": -2.662491798400879,1143 "step": 7201144 },1145 {1146 "epoch": 0.38,1147 "learning_rate": 4.858481545228533e-07,1148 "logits/chosen": -2.9665563106536865,1149 "logits/rejected": -2.962049722671509,1150 "logps/chosen": -315.9619140625,1151 "logps/rejected": -332.67608642578125,1152 "loss": 0.5005,1153 "rewards/accuracies": 0.6499999761581421,1154 "rewards/chosen": -0.3134005665779114,1155 "rewards/margins": 1.136232614517212,1156 "rewards/rejected": -1.4496333599090576,1157 "step": 7301158 },1159 {1160 "epoch": 0.38,1161 "learning_rate": 4.848919487473704e-07,1162 "logits/chosen": -2.9164295196533203,1163 "logits/rejected": -2.856682538986206,1164 "logps/chosen": -322.0476379394531,1165 "logps/rejected": -230.6309356689453,1166 "loss": 0.5886,1167 "rewards/accuracies": 0.8500000238418579,1168 "rewards/chosen": -0.5171477198600769,1169 "rewards/margins": 1.454939603805542,1170 "rewards/rejected": -1.9720872640609741,1171 "step": 7401172 },1173 {1174 "epoch": 0.39,1175 "learning_rate": 4.839357429718875e-07,1176 "logits/chosen": -2.884716510772705,1177 "logits/rejected": -2.9519991874694824,1178 "logps/chosen": -297.78839111328125,1179 "logps/rejected": -239.98959350585938,1180 "loss": 0.5356,1181 "rewards/accuracies": 0.8500000238418579,1182 "rewards/chosen": -0.3978341817855835,1183 "rewards/margins": 1.704395055770874,1184 "rewards/rejected": -2.102229356765747,1185 "step": 7501186 },1187 {1188 "epoch": 0.39,1189 "learning_rate": 4.829795371964047e-07,1190 "logits/chosen": -2.831848621368408,1191 "logits/rejected": -2.817645311355591,1192 "logps/chosen": -249.10452270507812,1193 "logps/rejected": -241.6534881591797,1194 "loss": 0.8766,1195 "rewards/accuracies": 0.8999999761581421,1196 "rewards/chosen": -0.2761456370353699,1197 "rewards/margins": 1.4581564664840698,1198 "rewards/rejected": -1.7343019247055054,1199 "step": 7601200 },