kevinpro/MathOctopus-MAPO-DPO-13B
020
1{2 "best_metric": 0.6881732940673828,3 "best_model_checkpoint": "/mnt/data/shesj/Trained/RL4CoT/DPO/Parallel_13B_numglueCorrect_extend_10lang_v3_iter2.json/checkpoint-1000",4 "epoch": 0.2508938091952581,5 "eval_steps": 100,6 "global_step": 1000,7 "is_hyper_param_search": false,8 "is_local_process_zero": true,9 "is_world_process_zero": true,10 "log_history": [11 {12 "epoch": 0.0,13 "learning_rate": 1e-08,14 "logits/chosen": -1.5961366891860962,15 "logits/rejected": -1.4505422115325928,16 "logps/chosen": -5.596881866455078,17 "logps/rejected": -9.411199569702148,18 "loss": 0.6934,19 "rewards/accuracies": 0.26249998807907104,20 "rewards/chosen": -0.0024300559889525175,21 "rewards/margins": -0.003987783100455999,22 "rewards/rejected": 0.00155772699508816,23 "step": 524 },25 {26 "epoch": 0.0,27 "learning_rate": 2e-08,28 "logits/chosen": -1.7575080394744873,29 "logits/rejected": -1.5709590911865234,30 "logps/chosen": -5.7118635177612305,31 "logps/rejected": -8.229207992553711,32 "loss": 0.6932,33 "rewards/accuracies": 0.6625000238418579,34 "rewards/chosen": 0.005083759315311909,35 "rewards/margins": 0.007960619404911995,36 "rewards/rejected": -0.0028768605552613735,37 "step": 1038 },39 {40 "epoch": 0.0,41 "learning_rate": 3e-08,42 "logits/chosen": -1.3776355981826782,43 "logits/rejected": -1.1297904253005981,44 "logps/chosen": -5.6487884521484375,45 "logps/rejected": -6.3460259437561035,46 "loss": 0.6928,47 "rewards/accuracies": 0.550000011920929,48 "rewards/chosen": 0.0028433147817850113,49 "rewards/margins": 0.004021753557026386,50 "rewards/rejected": -0.0011784390080720186,51 "step": 1552 },53 {54 "epoch": 0.01,55 "learning_rate": 4e-08,56 "logits/chosen": -1.3416035175323486,57 "logits/rejected": -1.4808504581451416,58 "logps/chosen": -6.05244255065918,59 "logps/rejected": -6.185896873474121,60 "loss": 0.6939,61 "rewards/accuracies": 0.44999998807907104,62 "rewards/chosen": -0.002370675327256322,63 "rewards/margins": -0.003694503800943494,64 "rewards/rejected": 0.0013238281244412065,65 "step": 2066 },67 {68 "epoch": 0.01,69 "learning_rate": 5e-08,70 "logits/chosen": -1.5403258800506592,71 "logits/rejected": -1.6286535263061523,72 "logps/chosen": -5.387463569641113,73 "logps/rejected": -6.760479927062988,74 "loss": 0.6931,75 "rewards/accuracies": 0.42500001192092896,76 "rewards/chosen": -0.002958612982183695,77 "rewards/margins": -0.0027186197694391012,78 "rewards/rejected": -0.0002399933582637459,79 "step": 2580 },81 {82 "epoch": 0.01,83 "learning_rate": 6e-08,84 "logits/chosen": -1.3823367357254028,85 "logits/rejected": -1.3627607822418213,86 "logps/chosen": -5.9833083152771,87 "logps/rejected": -5.850650787353516,88 "loss": 0.6936,89 "rewards/accuracies": 0.5249999761581421,90 "rewards/chosen": 0.002619728446006775,91 "rewards/margins": 0.004939082078635693,92 "rewards/rejected": -0.002319354098290205,93 "step": 3094 },95 {96 "epoch": 0.01,97 "learning_rate": 6.999999999999999e-08,98 "logits/chosen": -1.4706413745880127,99 "logits/rejected": -1.4250898361206055,100 "logps/chosen": -5.871586799621582,101 "logps/rejected": -7.329322814941406,102 "loss": 0.6927,103 "rewards/accuracies": 0.550000011920929,104 "rewards/chosen": 0.0053069936111569405,105 "rewards/margins": 0.006646360270678997,106 "rewards/rejected": -0.001339366426691413,107 "step": 35108 },109 {110 "epoch": 0.01,111 "learning_rate": 8e-08,112 "logits/chosen": -1.3677151203155518,113 "logits/rejected": -1.4711055755615234,114 "logps/chosen": -5.220858573913574,115 "logps/rejected": -6.019894599914551,116 "loss": 0.6922,117 "rewards/accuracies": 0.48750001192092896,118 "rewards/chosen": 0.0007630128529854119,119 "rewards/margins": 0.0009836136596277356,120 "rewards/rejected": -0.0002206008939538151,121 "step": 40122 },123 {124 "epoch": 0.01,125 "learning_rate": 9e-08,126 "logits/chosen": -1.5524793863296509,127 "logits/rejected": -1.5295554399490356,128 "logps/chosen": -5.439484596252441,129 "logps/rejected": -6.122335433959961,130 "loss": 0.6937,131 "rewards/accuracies": 0.4625000059604645,132 "rewards/chosen": -0.0029087450820952654,133 "rewards/margins": -0.0008771896245889366,134 "rewards/rejected": -0.002031555399298668,135 "step": 45136 },137 {138 "epoch": 0.01,139 "learning_rate": 1e-07,140 "logits/chosen": -1.5465644598007202,141 "logits/rejected": -1.5324052572250366,142 "logps/chosen": -5.974602699279785,143 "logps/rejected": -6.071439743041992,144 "loss": 0.6932,145 "rewards/accuracies": 0.4749999940395355,146 "rewards/chosen": 0.00048004291602410376,147 "rewards/margins": -0.0023091284092515707,148 "rewards/rejected": 0.0027891716454178095,149 "step": 50150 },151 {152 "epoch": 0.01,153 "learning_rate": 1.1e-07,154 "logits/chosen": -1.4663435220718384,155 "logits/rejected": -1.4614862203598022,156 "logps/chosen": -5.4424333572387695,157 "logps/rejected": -6.563809394836426,158 "loss": 0.6937,159 "rewards/accuracies": 0.512499988079071,160 "rewards/chosen": 0.0017671022797003388,161 "rewards/margins": 0.00013901680358685553,162 "rewards/rejected": 0.0016280856216326356,163 "step": 55164 },165 {166 "epoch": 0.02,167 "learning_rate": 1.2e-07,168 "logits/chosen": -1.3572139739990234,169 "logits/rejected": -1.5018689632415771,170 "logps/chosen": -5.387210845947266,171 "logps/rejected": -5.628620147705078,172 "loss": 0.6934,173 "rewards/accuracies": 0.5,174 "rewards/chosen": -0.0009455516701564193,175 "rewards/margins": -0.0012193130096420646,176 "rewards/rejected": 0.0002737622708082199,177 "step": 60178 },179 {180 "epoch": 0.02,181 "learning_rate": 1.3e-07,182 "logits/chosen": -1.4500153064727783,183 "logits/rejected": -1.5059070587158203,184 "logps/chosen": -5.2743024826049805,185 "logps/rejected": -5.707627773284912,186 "loss": 0.6932,187 "rewards/accuracies": 0.44999998807907104,188 "rewards/chosen": -8.441717363893986e-05,189 "rewards/margins": -0.0039388397708535194,190 "rewards/rejected": 0.0038544225972145796,191 "step": 65192 },193 {194 "epoch": 0.02,195 "learning_rate": 1.3999999999999998e-07,196 "logits/chosen": -1.400269627571106,197 "logits/rejected": -1.4793713092803955,198 "logps/chosen": -5.212708473205566,199 "logps/rejected": -7.475738525390625,200 "loss": 0.6938,201 "rewards/accuracies": 0.42500001192092896,202 "rewards/chosen": -0.0009079872397705913,203 "rewards/margins": -0.004652161151170731,204 "rewards/rejected": 0.0037441744934767485,205 "step": 70206 },207 {208 "epoch": 0.02,209 "learning_rate": 1.5e-07,210 "logits/chosen": -1.6910864114761353,211 "logits/rejected": -1.6056768894195557,212 "logps/chosen": -5.710541725158691,213 "logps/rejected": -6.734696388244629,214 "loss": 0.6925,215 "rewards/accuracies": 0.5375000238418579,216 "rewards/chosen": -0.0002835780323948711,217 "rewards/margins": 0.001549507724121213,218 "rewards/rejected": -0.0018330859020352364,219 "step": 75220 },221 {222 "epoch": 0.02,223 "learning_rate": 1.6e-07,224 "logits/chosen": -1.348565936088562,225 "logits/rejected": -1.3284913301467896,226 "logps/chosen": -5.4736833572387695,227 "logps/rejected": -6.32749080657959,228 "loss": 0.6916,229 "rewards/accuracies": 0.574999988079071,230 "rewards/chosen": 0.004232033155858517,231 "rewards/margins": 0.008012665435671806,232 "rewards/rejected": -0.003780632745474577,233 "step": 80234 },235 {236 "epoch": 0.02,237 "learning_rate": 1.7e-07,238 "logits/chosen": -1.4623692035675049,239 "logits/rejected": -1.3986611366271973,240 "logps/chosen": -5.681046485900879,241 "logps/rejected": -6.193971157073975,242 "loss": 0.6923,243 "rewards/accuracies": 0.550000011920929,244 "rewards/chosen": -0.003290227148681879,245 "rewards/margins": 0.00016479431360494345,246 "rewards/rejected": -0.003455021884292364,247 "step": 85248 },249 {250 "epoch": 0.02,251 "learning_rate": 1.8e-07,252 "logits/chosen": -1.6442101001739502,253 "logits/rejected": -1.7291923761367798,254 "logps/chosen": -6.625936985015869,255 "logps/rejected": -8.430340766906738,256 "loss": 0.6935,257 "rewards/accuracies": 0.625,258 "rewards/chosen": 0.004968828056007624,259 "rewards/margins": 0.004905154462903738,260 "rewards/rejected": 6.367354944813997e-05,261 "step": 90262 },263 {264 "epoch": 0.02,265 "learning_rate": 1.8999999999999998e-07,266 "logits/chosen": -1.3575623035430908,267 "logits/rejected": -1.3733503818511963,268 "logps/chosen": -5.06931209564209,269 "logps/rejected": -5.564260005950928,270 "loss": 0.6925,271 "rewards/accuracies": 0.574999988079071,272 "rewards/chosen": 0.0030343024991452694,273 "rewards/margins": 0.004197937436401844,274 "rewards/rejected": -0.0011636342387646437,275 "step": 95276 },277 {278 "epoch": 0.03,279 "learning_rate": 2e-07,280 "logits/chosen": -1.4199776649475098,281 "logits/rejected": -1.3951759338378906,282 "logps/chosen": -4.820796012878418,283 "logps/rejected": -5.737803936004639,284 "loss": 0.6933,285 "rewards/accuracies": 0.4625000059604645,286 "rewards/chosen": -0.0034723032731562853,287 "rewards/margins": -0.003412929829210043,288 "rewards/rejected": -5.9372840041760355e-05,289 "step": 100290 },291 {292 "epoch": 0.03,293 "eval_logits/chosen": -3.167680501937866,294 "eval_logits/rejected": -3.1470484733581543,295 "eval_logps/chosen": -6.138812065124512,296 "eval_logps/rejected": -6.94625997543335,297 "eval_loss": 0.6930756568908691,298 "eval_rewards/accuracies": 0.5112179517745972,299 "eval_rewards/chosen": 0.00018124215421266854,300 "eval_rewards/margins": 0.00026574215735308826,301 "eval_rewards/rejected": -8.450019231531769e-05,302 "eval_runtime": 615.2356,303 "eval_samples_per_second": 32.401,304 "eval_steps_per_second": 0.507,305 "step": 100306 },307 {308 "epoch": 0.03,309 "learning_rate": 1.9999658256641745e-07,310 "logits/chosen": -1.5726451873779297,311 "logits/rejected": -1.4517858028411865,312 "logps/chosen": -6.400355339050293,313 "logps/rejected": -8.746145248413086,314 "loss": 0.6923,315 "rewards/accuracies": 0.48750001192092896,316 "rewards/chosen": 0.0017850773874670267,317 "rewards/margins": 0.003671336220577359,318 "rewards/rejected": -0.001886258483864367,319 "step": 105320 },321 {322 "epoch": 0.03,323 "learning_rate": 1.999863304992469e-07,324 "logits/chosen": -1.4806731939315796,325 "logits/rejected": -1.4488584995269775,326 "logps/chosen": -4.718806266784668,327 "logps/rejected": -6.621995449066162,328 "loss": 0.6931,329 "rewards/accuracies": 0.512499988079071,330 "rewards/chosen": -0.003079561050981283,331 "rewards/margins": -0.0010764991166070104,332 "rewards/rejected": -0.002003061817958951,333 "step": 110334 },335 {336 "epoch": 0.03,337 "learning_rate": 1.9996924449920347e-07,338 "logits/chosen": -1.3951839208602905,339 "logits/rejected": -1.5014991760253906,340 "logps/chosen": -5.849172115325928,341 "logps/rejected": -6.110888957977295,342 "loss": 0.6916,343 "rewards/accuracies": 0.5249999761581421,344 "rewards/chosen": -0.002346306573599577,345 "rewards/margins": -0.0002563331217970699,346 "rewards/rejected": -0.0020899735391139984,347 "step": 115348 },349 {350 "epoch": 0.03,351 "learning_rate": 1.999453257340926e-07,352 "logits/chosen": -1.709111213684082,353 "logits/rejected": -1.6502996683120728,354 "logps/chosen": -5.375964164733887,355 "logps/rejected": -6.7974419593811035,356 "loss": 0.6908,357 "rewards/accuracies": 0.5,358 "rewards/chosen": 0.0006786620942875743,359 "rewards/margins": 0.002161815296858549,360 "rewards/rejected": -0.0014831533189862967,361 "step": 120362 },363 {364 "epoch": 0.03,365 "learning_rate": 1.9991457583873009e-07,366 "logits/chosen": -1.6699622869491577,367 "logits/rejected": -1.6052825450897217,368 "logps/chosen": -5.923104286193848,369 "logps/rejected": -6.845878601074219,370 "loss": 0.6904,371 "rewards/accuracies": 0.5375000238418579,372 "rewards/chosen": 0.002157662995159626,373 "rewards/margins": 0.0037304076831787825,374 "rewards/rejected": -0.0015727445716038346,375 "step": 125376 },377 {378 "epoch": 0.03,379 "learning_rate": 1.9987699691483047e-07,380 "logits/chosen": -1.5257129669189453,381 "logits/rejected": -1.5561866760253906,382 "logps/chosen": -4.64754581451416,383 "logps/rejected": -6.103314399719238,384 "loss": 0.69,385 "rewards/accuracies": 0.637499988079071,386 "rewards/chosen": 0.00055070681264624,387 "rewards/margins": 0.008159702643752098,388 "rewards/rejected": -0.0076089962385594845,389 "step": 130390 },391 {392 "epoch": 0.03,393 "learning_rate": 1.9983259153086325e-07,394 "logits/chosen": -1.5833688974380493,395 "logits/rejected": -1.5711325407028198,396 "logps/chosen": -5.653990745544434,397 "logps/rejected": -7.226287841796875,398 "loss": 0.6918,399 "rewards/accuracies": 0.625,400 "rewards/chosen": -0.0021617007441818714,401 "rewards/margins": 0.005132616497576237,402 "rewards/rejected": -0.007294316776096821,403 "step": 135404 },405 {406 "epoch": 0.04,407 "learning_rate": 1.9978136272187745e-07,408 "logits/chosen": -1.3831666707992554,409 "logits/rejected": -1.3798400163650513,410 "logps/chosen": -4.3581929206848145,411 "logps/rejected": -5.750641345977783,412 "loss": 0.6905,413 "rewards/accuracies": 0.550000011920929,414 "rewards/chosen": 0.0044810837134718895,415 "rewards/margins": 0.002624013228341937,416 "rewards/rejected": 0.0018570702522993088,417 "step": 140418 },419 {420 "epoch": 0.04,421 "learning_rate": 1.997233139892941e-07,422 "logits/chosen": -1.4784475564956665,423 "logits/rejected": -1.453616976737976,424 "logps/chosen": -5.93430757522583,425 "logps/rejected": -6.510165214538574,426 "loss": 0.6904,427 "rewards/accuracies": 0.5625,428 "rewards/chosen": 0.0023500092793256044,429 "rewards/margins": 0.007515914738178253,430 "rewards/rejected": -0.005165906623005867,431 "step": 145432 },433 {434 "epoch": 0.04,435 "learning_rate": 1.9965844930066698e-07,436 "logits/chosen": -1.4583405256271362,437 "logits/rejected": -1.611533761024475,438 "logps/chosen": -5.597588539123535,439 "logps/rejected": -7.059754848480225,440 "loss": 0.6884,441 "rewards/accuracies": 0.574999988079071,442 "rewards/chosen": -0.0004460577911231667,443 "rewards/margins": 0.008472367189824581,444 "rewards/rejected": -0.00891842506825924,445 "step": 150446 },447 {448 "epoch": 0.04,449 "learning_rate": 1.9958677308941136e-07,450 "logits/chosen": -1.590573787689209,451 "logits/rejected": -1.5703933238983154,452 "logps/chosen": -8.305377006530762,453 "logps/rejected": -6.981376647949219,454 "loss": 0.6898,455 "rewards/accuracies": 0.5874999761581421,456 "rewards/chosen": 0.0003787805908359587,457 "rewards/margins": 0.006806156598031521,458 "rewards/rejected": -0.006427376065403223,459 "step": 155460 },461 {462 "epoch": 0.04,463 "learning_rate": 1.9950829025450114e-07,464 "logits/chosen": -1.6168692111968994,465 "logits/rejected": -1.3941491842269897,466 "logps/chosen": -4.724443435668945,467 "logps/rejected": -7.058165073394775,468 "loss": 0.6897,469 "rewards/accuracies": 0.550000011920929,470 "rewards/chosen": 0.002109301509335637,471 "rewards/margins": 0.008472011424601078,472 "rewards/rejected": -0.00636270921677351,473 "step": 160474 },475 {476 "epoch": 0.04,477 "learning_rate": 1.9942300616013377e-07,478 "logits/chosen": -1.7095611095428467,479 "logits/rejected": -1.7674989700317383,480 "logps/chosen": -5.134617328643799,481 "logps/rejected": -5.987712383270264,482 "loss": 0.6906,483 "rewards/accuracies": 0.5375000238418579,484 "rewards/chosen": -0.0022057683672755957,485 "rewards/margins": 0.0033194683492183685,486 "rewards/rejected": -0.0055252364836633205,487 "step": 165488 },489 {490 "epoch": 0.04,491 "learning_rate": 1.993309266353638e-07,492 "logits/chosen": -1.3613307476043701,493 "logits/rejected": -1.4475274085998535,494 "logps/chosen": -6.0252180099487305,495 "logps/rejected": -6.769097328186035,496 "loss": 0.6906,497 "rewards/accuracies": 0.5874999761581421,498 "rewards/chosen": -0.00019303560839034617,499 "rewards/margins": 0.0034273392520844936,500 "rewards/rejected": -0.0036203742492944,501 "step": 170502 },503 {504 "epoch": 0.04,505 "learning_rate": 1.992320579737045e-07,506 "logits/chosen": -1.2607237100601196,507 "logits/rejected": -1.2645083665847778,508 "logps/chosen": -5.982339859008789,509 "logps/rejected": -5.792677879333496,510 "loss": 0.687,511 "rewards/accuracies": 0.6875,512 "rewards/chosen": 0.006508027669042349,513 "rewards/margins": 0.01678382232785225,514 "rewards/rejected": -0.010275794193148613,515 "step": 175516 },517 {518 "epoch": 0.05,519 "learning_rate": 1.9912640693269751e-07,520 "logits/chosen": -1.4913660287857056,521 "logits/rejected": -1.467761516571045,522 "logps/chosen": -4.889002799987793,523 "logps/rejected": -6.567469120025635,524 "loss": 0.6895,525 "rewards/accuracies": 0.6625000238418579,526 "rewards/chosen": 0.007032574620097876,527 "rewards/margins": 0.015939272940158844,528 "rewards/rejected": -0.008906697854399681,529 "step": 180530 },531 {532 "epoch": 0.05,533 "learning_rate": 1.9901398073345117e-07,534 "logits/chosen": -1.6264896392822266,535 "logits/rejected": -1.787503957748413,536 "logps/chosen": -5.5176873207092285,537 "logps/rejected": -7.238592624664307,538 "loss": 0.6888,539 "rewards/accuracies": 0.6625000238418579,540 "rewards/chosen": 0.0024650241248309612,541 "rewards/margins": 0.014559340663254261,542 "rewards/rejected": -0.012094316072762012,543 "step": 185544 },545 {546 "epoch": 0.05,547 "learning_rate": 1.9889478706014683e-07,548 "logits/chosen": -1.5220094919204712,549 "logits/rejected": -1.3731660842895508,550 "logps/chosen": -4.48035192489624,551 "logps/rejected": -5.973559856414795,552 "loss": 0.6892,553 "rewards/accuracies": 0.6499999761581421,554 "rewards/chosen": 0.002238192595541477,555 "rewards/margins": 0.0097486088052392,556 "rewards/rejected": -0.007510416209697723,557 "step": 190558 },559 {560 "epoch": 0.05,561 "learning_rate": 1.9876883405951376e-07,562 "logits/chosen": -1.3608970642089844,563 "logits/rejected": -1.2253252267837524,564 "logps/chosen": -6.359659194946289,565 "logps/rejected": -6.143754005432129,566 "loss": 0.6891,567 "rewards/accuracies": 0.6499999761581421,568 "rewards/chosen": 0.0014768632827326655,569 "rewards/margins": 0.004477277398109436,570 "rewards/rejected": -0.0030004139989614487,571 "step": 195572 },573 {574 "epoch": 0.05,575 "learning_rate": 1.9863613034027222e-07,576 "logits/chosen": -1.3653035163879395,577 "logits/rejected": -1.3042250871658325,578 "logps/chosen": -5.9716901779174805,579 "logps/rejected": -6.985511779785156,580 "loss": 0.6853,581 "rewards/accuracies": 0.5625,582 "rewards/chosen": 0.003856885712593794,583 "rewards/margins": 0.011315730400383472,584 "rewards/rejected": -0.0074588460847735405,585 "step": 200586 },587 {588 "epoch": 0.05,589 "eval_logits/chosen": -3.174004077911377,590 "eval_logits/rejected": -3.153486490249634,591 "eval_logps/chosen": -6.161444664001465,592 "eval_logps/rejected": -6.977956771850586,593 "eval_loss": 0.6930607557296753,594 "eval_rewards/accuracies": 0.5140224099159241,595 "eval_rewards/chosen": -0.0020819876808673143,596 "eval_rewards/margins": 0.0011722741182893515,597 "eval_rewards/rejected": -0.0032542620319873095,598 "eval_runtime": 618.1553,599 "eval_samples_per_second": 32.248,600 "eval_steps_per_second": 0.505,601 "step": 200602 },603 {604 "epoch": 0.05,605 "learning_rate": 1.9849668497254518e-07,606 "logits/chosen": -1.366420030593872,607 "logits/rejected": -1.418501377105713,608 "logps/chosen": -4.841452121734619,609 "logps/rejected": -7.0436296463012695,610 "loss": 0.6882,611 "rewards/accuracies": 0.637499988079071,612 "rewards/chosen": 0.0067129782401025295,613 "rewards/margins": 0.013699628412723541,614 "rewards/rejected": -0.006986652500927448,615 "step": 205616 },617 {618 "epoch": 0.05,619 "learning_rate": 1.9835050748723822e-07,620 "logits/chosen": -1.5236493349075317,621 "logits/rejected": -1.4237031936645508,622 "logps/chosen": -5.028156280517578,623 "logps/rejected": -6.966952323913574,624 "loss": 0.6862,625 "rewards/accuracies": 0.6000000238418579,626 "rewards/chosen": 0.004889342468231916,627 "rewards/margins": 0.01445357222110033,628 "rewards/rejected": -0.009564228355884552,629 "step": 210630 },631 {632 "epoch": 0.05,633 "learning_rate": 1.9819760787538837e-07,634 "logits/chosen": -1.7881724834442139,635 "logits/rejected": -1.702300786972046,636 "logps/chosen": -5.008362293243408,637 "logps/rejected": -7.346070766448975,638 "loss": 0.6862,639 "rewards/accuracies": 0.6499999761581421,640 "rewards/chosen": -0.003184701083227992,641 "rewards/margins": 0.009327715262770653,642 "rewards/rejected": -0.012512415647506714,643 "step": 215644 },645 {646 "epoch": 0.06,647 "learning_rate": 1.9803799658748093e-07,648 "logits/chosen": -1.7177143096923828,649 "logits/rejected": -1.579524278640747,650 "logps/chosen": -5.551783561706543,651 "logps/rejected": -8.588147163391113,652 "loss": 0.6859,653 "rewards/accuracies": 0.5625,654 "rewards/chosen": -0.00936056487262249,655 "rewards/margins": -0.0009745029965415597,656 "rewards/rejected": -0.00838606245815754,657 "step": 220658 },659 {660 "epoch": 0.06,661 "learning_rate": 1.9787168453273545e-07,662 "logits/chosen": -1.4372203350067139,663 "logits/rejected": -1.4148085117340088,664 "logps/chosen": -5.2672553062438965,665 "logps/rejected": -5.7078680992126465,666 "loss": 0.685,667 "rewards/accuracies": 0.637499988079071,668 "rewards/chosen": 0.004485909827053547,669 "rewards/margins": 0.01839793473482132,670 "rewards/rejected": -0.013912022113800049,671 "step": 225672 },673 {674 "epoch": 0.06,675 "learning_rate": 1.9769868307835993e-07,676 "logits/chosen": -1.6145379543304443,677 "logits/rejected": -1.536407709121704,678 "logps/chosen": -5.295413970947266,679 "logps/rejected": -7.089319705963135,680 "loss": 0.6852,681 "rewards/accuracies": 0.550000011920929,682 "rewards/chosen": 0.00589085603132844,683 "rewards/margins": 0.01651725545525551,684 "rewards/rejected": -0.010626398026943207,685 "step": 230686 },687 {688 "epoch": 0.06,689 "learning_rate": 1.9751900404877398e-07,690 "logits/chosen": -1.5907440185546875,691 "logits/rejected": -1.5271718502044678,692 "logps/chosen": -4.938467979431152,693 "logps/rejected": -5.892498970031738,694 "loss": 0.6846,695 "rewards/accuracies": 0.6000000238418579,696 "rewards/chosen": 0.0029959643725305796,697 "rewards/margins": 0.021143438294529915,698 "rewards/rejected": -0.01814747229218483,699 "step": 235700 },701 {702 "epoch": 0.06,703 "learning_rate": 1.9733265972480058e-07,704 "logits/chosen": -1.6651685237884521,705 "logits/rejected": -1.5356318950653076,706 "logps/chosen": -7.418261528015137,707 "logps/rejected": -8.457880973815918,708 "loss": 0.6852,709 "rewards/accuracies": 0.637499988079071,710 "rewards/chosen": 0.0033951636869460344,711 "rewards/margins": 0.026372741907835007,712 "rewards/rejected": -0.02297757938504219,713 "step": 240714 },715 {716 "epoch": 0.06,717 "learning_rate": 1.9713966284282674e-07,718 "logits/chosen": -1.4138095378875732,719 "logits/rejected": -1.3163411617279053,720 "logps/chosen": -6.462838172912598,721 "logps/rejected": -7.8138837814331055,722 "loss": 0.6839,723 "rewards/accuracies": 0.5625,724 "rewards/chosen": -0.007619897834956646,725 "rewards/margins": 0.0050119319930672646,726 "rewards/rejected": -0.012631828896701336,727 "step": 245728 },729 {730 "epoch": 0.06,731 "learning_rate": 1.9694002659393302e-07,732 "logits/chosen": -1.5196382999420166,733 "logits/rejected": -1.477994680404663,734 "logps/chosen": -5.463156700134277,735 "logps/rejected": -6.823214530944824,736 "loss": 0.6831,737 "rewards/accuracies": 0.5874999761581421,738 "rewards/chosen": -0.0001128343865275383,739 "rewards/margins": 0.022026551887392998,740 "rewards/rejected": -0.02213938534259796,741 "step": 250742 },743 {744 "epoch": 0.06,745 "learning_rate": 1.9673376462299182e-07,746 "logits/chosen": -1.522629976272583,747 "logits/rejected": -1.4759032726287842,748 "logps/chosen": -6.2718963623046875,749 "logps/rejected": -7.001539707183838,750 "loss": 0.6825,751 "rewards/accuracies": 0.675000011920929,752 "rewards/chosen": 0.00030401311232708395,753 "rewards/margins": 0.021813327446579933,754 "rewards/rejected": -0.021509312093257904,755 "step": 255756 },757 {758 "epoch": 0.07,759 "learning_rate": 1.9652089102773487e-07,760 "logits/chosen": -1.815498948097229,761 "logits/rejected": -1.9041208028793335,762 "logps/chosen": -4.9399566650390625,763 "logps/rejected": -6.410666465759277,764 "loss": 0.6844,765 "rewards/accuracies": 0.6499999761581421,766 "rewards/chosen": 0.0037350181955844164,767 "rewards/margins": 0.023170799016952515,768 "rewards/rejected": -0.01943577639758587,769 "step": 260770 },771 {772 "epoch": 0.07,773 "learning_rate": 1.963014203577896e-07,774 "logits/chosen": -1.5250468254089355,775 "logits/rejected": -1.5195215940475464,776 "logps/chosen": -4.857203483581543,777 "logps/rejected": -6.858325958251953,778 "loss": 0.6829,779 "rewards/accuracies": 0.625,780 "rewards/chosen": -0.007676561363041401,781 "rewards/margins": 0.008902650326490402,782 "rewards/rejected": -0.016579212620854378,783 "step": 265784 },785 {786 "epoch": 0.07,787 "learning_rate": 1.9607536761368482e-07,788 "logits/chosen": -1.398611068725586,789 "logits/rejected": -1.4291143417358398,790 "logps/chosen": -5.944151878356934,791 "logps/rejected": -6.5233049392700195,792 "loss": 0.682,793 "rewards/accuracies": 0.6875,794 "rewards/chosen": 0.004370951093733311,795 "rewards/margins": 0.02115224488079548,796 "rewards/rejected": -0.016781292855739594,797 "step": 270798 },799 {800 "epoch": 0.07,801 "learning_rate": 1.9584274824582527e-07,802 "logits/chosen": -1.3120192289352417,803 "logits/rejected": -1.3574376106262207,804 "logps/chosen": -4.558770179748535,805 "logps/rejected": -5.6866583824157715,806 "loss": 0.6821,807 "rewards/accuracies": 0.625,808 "rewards/chosen": 0.004323200322687626,809 "rewards/margins": 0.01929726079106331,810 "rewards/rejected": -0.014974060468375683,811 "step": 275812 },813 {814 "epoch": 0.07,815 "learning_rate": 1.9560357815343574e-07,816 "logits/chosen": -1.5410289764404297,817 "logits/rejected": -1.5422722101211548,818 "logps/chosen": -7.481714725494385,819 "logps/rejected": -6.8584442138671875,820 "loss": 0.6795,821 "rewards/accuracies": 0.6625000238418579,822 "rewards/chosen": 0.004259726498275995,823 "rewards/margins": 0.026613563299179077,824 "rewards/rejected": -0.02235383726656437,825 "step": 280826 },827 {828 "epoch": 0.07,829 "learning_rate": 1.9535787368347442e-07,830 "logits/chosen": -1.6866031885147095,831 "logits/rejected": -1.573209524154663,832 "logps/chosen": -6.985440731048584,833 "logps/rejected": -6.92074728012085,834 "loss": 0.6813,835 "rewards/accuracies": 0.612500011920929,836 "rewards/chosen": -0.003224983811378479,837 "rewards/margins": 0.03093186393380165,838 "rewards/rejected": -0.03415685147047043,839 "step": 285840 },841 {842 "epoch": 0.07,843 "learning_rate": 1.9510565162951537e-07,844 "logits/chosen": -1.5131175518035889,845 "logits/rejected": -1.5239301919937134,846 "logps/chosen": -5.186364650726318,847 "logps/rejected": -6.2596564292907715,848 "loss": 0.6808,849 "rewards/accuracies": 0.675000011920929,850 "rewards/chosen": 0.012961247935891151,851 "rewards/margins": 0.05755941942334175,852 "rewards/rejected": -0.04459817335009575,853 "step": 290854 },855 {856 "epoch": 0.07,857 "learning_rate": 1.9484692923060094e-07,858 "logits/chosen": -1.540378212928772,859 "logits/rejected": -1.5740854740142822,860 "logps/chosen": -6.078642845153809,861 "logps/rejected": -6.573060512542725,862 "loss": 0.6809,863 "rewards/accuracies": 0.612500011920929,864 "rewards/chosen": -0.011097406968474388,865 "rewards/margins": 0.030860627070069313,866 "rewards/rejected": -0.041958026587963104,867 "step": 295868 },869 {870 "epoch": 0.08,871 "learning_rate": 1.9458172417006346e-07,872 "logits/chosen": -1.5138806104660034,873 "logits/rejected": -1.3892674446105957,874 "logps/chosen": -5.267967224121094,875 "logps/rejected": -6.3208441734313965,876 "loss": 0.6801,877 "rewards/accuracies": 0.637499988079071,878 "rewards/chosen": 0.0035496647469699383,879 "rewards/margins": 0.038710370659828186,880 "rewards/rejected": -0.03516070172190666,881 "step": 300882 },883 {884 "epoch": 0.08,885 "eval_logits/chosen": -3.184242010116577,886 "eval_logits/rejected": -3.1636037826538086,887 "eval_logps/chosen": -6.222862720489502,888 "eval_logps/rejected": -7.056950569152832,889 "eval_loss": 0.6925042867660522,890 "eval_rewards/accuracies": 0.5252403616905212,891 "eval_rewards/chosen": -0.008223854936659336,892 "eval_rewards/margins": 0.0029297315049916506,893 "eval_rewards/rejected": -0.011153585277497768,894 "eval_runtime": 620.1964,895 "eval_samples_per_second": 32.141,896 "eval_steps_per_second": 0.503,897 "step": 300898 },899 {900 "epoch": 0.08,901 "learning_rate": 1.943100545743165e-07,902 "logits/chosen": -1.6512733697891235,903 "logits/rejected": -1.721056342124939,904 "logps/chosen": -5.9783034324646,905 "logps/rejected": -5.989422798156738,906 "loss": 0.6787,907 "rewards/accuracies": 0.612500011920929,908 "rewards/chosen": -0.009316826239228249,909 "rewards/margins": 0.029956454411149025,910 "rewards/rejected": -0.039273280650377274,911 "step": 305912 },913 {914 "epoch": 0.08,915 "learning_rate": 1.9403193901161612e-07,916 "logits/chosen": -1.4372011423110962,917 "logits/rejected": -1.2608239650726318,918 "logps/chosen": -5.216915130615234,919 "logps/rejected": -7.368325233459473,920 "loss": 0.6783,921 "rewards/accuracies": 0.5625,922 "rewards/chosen": -0.00843791477382183,923 "rewards/margins": 0.019914906471967697,924 "rewards/rejected": -0.028352823108434677,925 "step": 310926 },927 {928 "epoch": 0.08,929 "learning_rate": 1.9374739649079154e-07,930 "logits/chosen": -1.4851138591766357,931 "logits/rejected": -1.5565675497055054,932 "logps/chosen": -6.530230522155762,933 "logps/rejected": -5.977784156799316,934 "loss": 0.6795,935 "rewards/accuracies": 0.6000000238418579,936 "rewards/chosen": 0.005143959075212479,937 "rewards/margins": 0.03382311016321182,938 "rewards/rejected": -0.028679147362709045,939 "step": 315940 },941 {942 "epoch": 0.08,943 "learning_rate": 1.9345644645994608e-07,944 "logits/chosen": -1.7324178218841553,945 "logits/rejected": -1.6093097925186157,946 "logps/chosen": -5.668793201446533,947 "logps/rejected": -8.58001708984375,948 "loss": 0.6813,949 "rewards/accuracies": 0.637499988079071,950 "rewards/chosen": -0.007929663173854351,951 "rewards/margins": 0.016542982310056686,952 "rewards/rejected": -0.02447264827787876,953 "step": 320954 },955 {956 "epoch": 0.08,957 "learning_rate": 1.9315910880512788e-07,958 "logits/chosen": -1.664690613746643,959 "logits/rejected": -1.6959164142608643,960 "logps/chosen": -4.798338413238525,961 "logps/rejected": -6.650571346282959,962 "loss": 0.6793,963 "rewards/accuracies": 0.6875,964 "rewards/chosen": -0.002946319757029414,965 "rewards/margins": 0.03725341707468033,966 "rewards/rejected": -0.04019974544644356,967 "step": 325968 },969 {970 "epoch": 0.08,971 "learning_rate": 1.928554038489707e-07,972 "logits/chosen": -1.6366933584213257,973 "logits/rejected": -1.6312482357025146,974 "logps/chosen": -5.757204532623291,975 "logps/rejected": -6.9877214431762695,976 "loss": 0.6771,977 "rewards/accuracies": 0.5874999761581421,978 "rewards/chosen": -0.004410495515912771,979 "rewards/margins": 0.03300872817635536,980 "rewards/rejected": -0.03741922602057457,981 "step": 330982 },983 {984 "epoch": 0.08,985 "learning_rate": 1.9254535234930483e-07,986 "logits/chosen": -1.383345365524292,987 "logits/rejected": -1.4103121757507324,988 "logps/chosen": -7.0129594802856445,989 "logps/rejected": -6.204277038574219,990 "loss": 0.6795,991 "rewards/accuracies": 0.5625,992 "rewards/chosen": -0.0018229834968224168,993 "rewards/margins": 0.018833067268133163,994 "rewards/rejected": -0.020656052976846695,995 "step": 335996 },997 {998 "epoch": 0.09,999 "learning_rate": 1.9222897549773846e-07,1000 "logits/chosen": -1.3323512077331543,1001 "logits/rejected": -1.3201320171356201,1002 "logps/chosen": -5.118114471435547,1003 "logps/rejected": -7.212060451507568,1004 "loss": 0.6776,1005 "rewards/accuracies": 0.612500011920929,1006 "rewards/chosen": -0.009920697659254074,1007 "rewards/margins": 0.029569601640105247,1008 "rewards/rejected": -0.03949030116200447,1009 "step": 3401010 },1011 {1012 "epoch": 0.09,1013 "learning_rate": 1.9190629491820908e-07,1014 "logits/chosen": -1.4638566970825195,1015 "logits/rejected": -1.4451267719268799,1016 "logps/chosen": -5.2605671882629395,1017 "logps/rejected": -6.085940837860107,1018 "loss": 0.6739,1019 "rewards/accuracies": 0.7124999761581421,1020 "rewards/chosen": -0.005447469651699066,1021 "rewards/margins": 0.034440845251083374,1022 "rewards/rejected": -0.03988831490278244,1023 "step": 3451024 },1025 {1026 "epoch": 0.09,1027 "learning_rate": 1.9157733266550572e-07,1028 "logits/chosen": -1.5962337255477905,1029 "logits/rejected": -1.6220163106918335,1030 "logps/chosen": -6.018959045410156,1031 "logps/rejected": -6.889263153076172,1032 "loss": 0.674,1033 "rewards/accuracies": 0.675000011920929,1034 "rewards/chosen": -0.0057447971776127815,1035 "rewards/margins": 0.07764319330453873,1036 "rewards/rejected": -0.08338797837495804,1037 "step": 3501038 },1039 {1040 "epoch": 0.09,1041 "learning_rate": 1.9124211122376135e-07,1042 "logits/chosen": -1.4028089046478271,1043 "logits/rejected": -1.3758150339126587,1044 "logps/chosen": -5.279055595397949,1045 "logps/rejected": -8.15546989440918,1046 "loss": 0.6746,1047 "rewards/accuracies": 0.699999988079071,1048 "rewards/chosen": 0.004057350568473339,1049 "rewards/margins": 0.041755061596632004,1050 "rewards/rejected": -0.03769771382212639,1051 "step": 3551052 },1053 {1054 "epoch": 0.09,1055 "learning_rate": 1.9090065350491624e-07,1056 "logits/chosen": -1.466732144355774,1057 "logits/rejected": -1.505338430404663,1058 "logps/chosen": -6.279904365539551,1059 "logps/rejected": -6.962622165679932,1060 "loss": 0.6722,1061 "rewards/accuracies": 0.699999988079071,1062 "rewards/chosen": 0.0070061227306723595,1063 "rewards/margins": 0.04479537159204483,1064 "rewards/rejected": -0.037789251655340195,1065 "step": 3601066 },1067 {1068 "epoch": 0.09,1069 "learning_rate": 1.905529828471519e-07,1070 "logits/chosen": -1.5488044023513794,1071 "logits/rejected": -1.4669849872589111,1072 "logps/chosen": -6.46566104888916,1073 "logps/rejected": -6.615389823913574,1074 "loss": 0.6752,1075 "rewards/accuracies": 0.6875,1076 "rewards/chosen": -0.009375613182783127,1077 "rewards/margins": 0.0315098911523819,1078 "rewards/rejected": -0.04088550806045532,1079 "step": 3651080 },1081 {1082 "epoch": 0.09,1083 "learning_rate": 1.901991230132959e-07,1084 "logits/chosen": -1.5748860836029053,1085 "logits/rejected": -1.5498300790786743,1086 "logps/chosen": -5.943437099456787,1087 "logps/rejected": -7.429045677185059,1088 "loss": 0.6752,1089 "rewards/accuracies": 0.5874999761581421,1090 "rewards/chosen": -0.012559416703879833,1091 "rewards/margins": 0.03320794180035591,1092 "rewards/rejected": -0.04576735943555832,1093 "step": 3701094 },1095 {1096 "epoch": 0.09,1097 "learning_rate": 1.8983909818919788e-07,1098 "logits/chosen": -1.4835641384124756,1099 "logits/rejected": -1.4734997749328613,1100 "logps/chosen": -5.776877403259277,1101 "logps/rejected": -6.821134090423584,1102 "loss": 0.6746,1103 "rewards/accuracies": 0.675000011920929,1104 "rewards/chosen": 8.074291690718383e-05,1105 "rewards/margins": 0.041906945407390594,1106 "rewards/rejected": -0.04182619974017143,1107 "step": 3751108 },1109 {1110 "epoch": 0.1,1111 "learning_rate": 1.8947293298207635e-07,1112 "logits/chosen": -1.590511441230774,1113 "logits/rejected": -1.5636823177337646,1114 "logps/chosen": -6.252840042114258,1115 "logps/rejected": -7.568482398986816,1116 "loss": 0.6773,1117 "rewards/accuracies": 0.6000000238418579,1118 "rewards/chosen": -0.009611086919903755,1119 "rewards/margins": 0.04731304943561554,1120 "rewards/rejected": -0.056924134492874146,1121 "step": 3801122 },1123 {1124 "epoch": 0.1,1125 "learning_rate": 1.8910065241883678e-07,1126 "logits/chosen": -1.7463115453720093,1127 "logits/rejected": -1.552099347114563,1128 "logps/chosen": -5.755384922027588,1129 "logps/rejected": -9.108583450317383,1130 "loss": 0.6762,1131 "rewards/accuracies": 0.6625000238418579,1132 "rewards/chosen": -0.014299096539616585,1133 "rewards/margins": 0.04487228766083717,1134 "rewards/rejected": -0.05917138606309891,1135 "step": 3851136 },1137 {1138 "epoch": 0.1,1139 "learning_rate": 1.8872228194436116e-07,1140 "logits/chosen": -1.4452903270721436,1141 "logits/rejected": -1.4187732934951782,1142 "logps/chosen": -5.313692092895508,1143 "logps/rejected": -6.416788578033447,1144 "loss": 0.6713,1145 "rewards/accuracies": 0.6625000238418579,1146 "rewards/chosen": -0.017885476350784302,1147 "rewards/margins": 0.03439543396234512,1148 "rewards/rejected": -0.052280914038419724,1149 "step": 3901150 },1151 {1152 "epoch": 0.1,1153 "learning_rate": 1.8833784741976886e-07,1154 "logits/chosen": -1.5387766361236572,1155 "logits/rejected": -1.5287269353866577,1156 "logps/chosen": -5.880291938781738,1157 "logps/rejected": -6.970144748687744,1158 "loss": 0.6682,1159 "rewards/accuracies": 0.6000000238418579,1160 "rewards/chosen": -0.018293332308530807,1161 "rewards/margins": 0.044221844524145126,1162 "rewards/rejected": -0.06251517683267593,1163 "step": 3951164 },1165 {1166 "epoch": 0.1,1167 "learning_rate": 1.8794737512064888e-07,1168 "logits/chosen": -1.640610933303833,1169 "logits/rejected": -1.539394736289978,1170 "logps/chosen": -5.029040336608887,1171 "logps/rejected": -6.50949764251709,1172 "loss": 0.6676,1173 "rewards/accuracies": 0.6875,1174 "rewards/chosen": -0.008673004806041718,1175 "rewards/margins": 0.04148182272911072,1176 "rewards/rejected": -0.050154827535152435,1177 "step": 4001178 },1179 {1180 "epoch": 0.1,1181 "eval_logits/chosen": -3.1871755123138428,1182 "eval_logits/rejected": -3.1664505004882812,1183 "eval_logps/chosen": -6.327807903289795,1184 "eval_logps/rejected": -7.203298091888428,1185 "eval_loss": 0.6918210983276367,1186 "eval_rewards/accuracies": 0.5324519276618958,1187 "eval_rewards/chosen": -0.018718333914875984,1188 "eval_rewards/margins": 0.007069970481097698,1189 "eval_rewards/rejected": -0.025788303464651108,1190 "eval_runtime": 624.3628,1191 "eval_samples_per_second": 31.927,1192 "eval_steps_per_second": 0.5,1193 "step": 4001194 },1195 {1196 "epoch": 0.1,1197 "learning_rate": 1.875508917352643e-07,1198 "logits/chosen": -1.3193857669830322,1199 "logits/rejected": -1.3139396905899048,1200 "logps/chosen": -5.116377830505371,