FormlessAI/single_gpu_30_max_turns_7B
012
1{2 "best_global_step": null,3 "best_metric": null,4 "best_model_checkpoint": null,5 "epoch": 1.0,6 "eval_steps": 500,7 "global_step": 1000,8 "is_hyper_param_search": false,9 "is_local_process_zero": true,10 "is_world_process_zero": true,11 "log_history": [12 {13 "clip_ratio/high_max": 0.0,14 "clip_ratio/high_mean": 0.0,15 "clip_ratio/low_mean": 0.0,16 "clip_ratio/low_min": 0.0,17 "clip_ratio/region_mean": 0.0,18 "completions/clipped_ratio": 0.0,19 "completions/max_length": 58.0,20 "completions/max_terminated_length": 58.0,21 "completions/mean_length": 52.25,22 "completions/mean_terminated_length": 52.25,23 "completions/min_length": 47.0,24 "completions/min_terminated_length": 47.0,25 "entropy": 0.1690794974565506,26 "epoch": 0.001,27 "frac_reward_zero_std": 0.0,28 "grad_norm": 1.7694201469421387,29 "kl": 0.0,30 "learning_rate": 0.0,31 "loss": -0.2838,32 "num_tokens": 2891.0,33 "reward": 0.45499998331069946,34 "reward_std": 0.6123452186584473,35 "rewards/reward_func/mean": 0.45499998331069946,36 "rewards/reward_func/std": 0.612345278263092,37 "sampling/importance_sampling_ratio/max": 1.8600257635116577,38 "sampling/importance_sampling_ratio/mean": 1.3248674869537354,39 "sampling/importance_sampling_ratio/min": 0.6954079270362854,40 "sampling/sampling_logp_difference/max": 0.3488762378692627,41 "sampling/sampling_logp_difference/mean": 0.019007110968232155,42 "step": 1,43 "step_time": 28.52634390495950444 },45 {46 "clip_ratio/high_max": 0.0,47 "clip_ratio/high_mean": 0.0,48 "clip_ratio/low_mean": 0.0,49 "clip_ratio/low_min": 0.0,50 "clip_ratio/region_mean": 0.0,51 "completions/clipped_ratio": 0.0,52 "completions/max_length": 56.0,53 "completions/max_terminated_length": 56.0,54 "completions/mean_length": 53.75,55 "completions/mean_terminated_length": 53.75,56 "completions/min_length": 51.0,57 "completions/min_terminated_length": 51.0,58 "entropy": 0.25211217999458313,59 "epoch": 0.002,60 "frac_reward_zero_std": 0.0,61 "grad_norm": 1.065087080001831,62 "kl": 0.0,63 "learning_rate": 1.6666666666666668e-07,64 "loss": 0.1109,65 "num_tokens": 5506.0,66 "reward": 0.45500001311302185,67 "reward_std": 0.6066575646400452,68 "rewards/reward_func/mean": 0.45500001311302185,69 "rewards/reward_func/std": 0.6066575050354004,70 "sampling/importance_sampling_ratio/max": 1.186666488647461,71 "sampling/importance_sampling_ratio/mean": 0.9661478996276855,72 "sampling/importance_sampling_ratio/min": 0.43538862466812134,73 "sampling/sampling_logp_difference/max": 0.32524752616882324,74 "sampling/sampling_logp_difference/mean": 0.01666935160756111,75 "step": 2,76 "step_time": 23.22447338298661677 },78 {79 "clip_ratio/high_max": 0.0,80 "clip_ratio/high_mean": 0.0,81 "clip_ratio/low_mean": 0.0,82 "clip_ratio/low_min": 0.0,83 "clip_ratio/region_mean": 0.0,84 "completions/clipped_ratio": 0.0,85 "completions/max_length": 57.0,86 "completions/max_terminated_length": 57.0,87 "completions/mean_length": 47.5,88 "completions/mean_terminated_length": 47.5,89 "completions/min_length": 37.0,90 "completions/min_terminated_length": 37.0,91 "entropy": 0.15375655889511108,92 "epoch": 0.003,93 "frac_reward_zero_std": 0.0,94 "grad_norm": 0.7260156869888306,95 "kl": 0.000387201172998175,96 "learning_rate": 3.3333333333333335e-07,97 "loss": -0.0158,98 "num_tokens": 7967.0,99 "reward": 0.7250000238418579,100 "reward_std": 0.4850085973739624,101 "rewards/reward_func/mean": 0.7250000238418579,102 "rewards/reward_func/std": 0.4850085973739624,103 "sampling/importance_sampling_ratio/max": 1.035571575164795,104 "sampling/importance_sampling_ratio/mean": 0.8206162452697754,105 "sampling/importance_sampling_ratio/min": 0.6426183581352234,106 "sampling/sampling_logp_difference/max": 0.2029057741165161,107 "sampling/sampling_logp_difference/mean": 0.011877943761646748,108 "step": 3,109 "step_time": 17.72353470302187110 },111 {112 "clip_ratio/high_max": 0.0,113 "clip_ratio/high_mean": 0.0,114 "clip_ratio/low_mean": 0.0,115 "clip_ratio/low_min": 0.0,116 "clip_ratio/region_mean": 0.0,117 "completions/clipped_ratio": 0.0,118 "completions/max_length": 52.0,119 "completions/max_terminated_length": 52.0,120 "completions/mean_length": 50.25,121 "completions/mean_terminated_length": 50.25,122 "completions/min_length": 48.0,123 "completions/min_terminated_length": 48.0,124 "entropy": 0.2212316393852234,125 "epoch": 0.004,126 "frac_reward_zero_std": 0.0,127 "grad_norm": 0.7334376573562622,128 "kl": 0.0022102410439401865,129 "learning_rate": 5.000000000000001e-07,130 "loss": -0.1995,131 "num_tokens": 10952.0,132 "reward": 0.7325000166893005,133 "reward_std": 0.488629013299942,134 "rewards/reward_func/mean": 0.7325000166893005,135 "rewards/reward_func/std": 0.488629013299942,136 "sampling/importance_sampling_ratio/max": 1.0403285026550293,137 "sampling/importance_sampling_ratio/mean": 0.761037290096283,138 "sampling/importance_sampling_ratio/min": 0.3470890522003174,139 "sampling/sampling_logp_difference/max": 0.9722568988800049,140 "sampling/sampling_logp_difference/mean": 0.023567143827676773,141 "step": 4,142 "step_time": 26.211164197011385143 },144 {145 "clip_ratio/high_max": 0.0,146 "clip_ratio/high_mean": 0.0,147 "clip_ratio/low_mean": 0.0,148 "clip_ratio/low_min": 0.0,149 "clip_ratio/region_mean": 0.0,150 "completions/clipped_ratio": 0.0,151 "completions/max_length": 56.0,152 "completions/max_terminated_length": 56.0,153 "completions/mean_length": 53.5,154 "completions/mean_terminated_length": 53.5,155 "completions/min_length": 51.0,156 "completions/min_terminated_length": 51.0,157 "entropy": 0.14337338507175446,158 "epoch": 0.005,159 "frac_reward_zero_std": 0.0,160 "grad_norm": 0.4706338346004486,161 "kl": 0.00046622363151982427,162 "learning_rate": 6.666666666666667e-07,163 "loss": 0.0025,164 "num_tokens": 14061.0,165 "reward": 0.21000000834465027,166 "reward_std": 0.52801513671875,167 "rewards/reward_func/mean": 0.21000000834465027,168 "rewards/reward_func/std": 0.52801513671875,169 "sampling/importance_sampling_ratio/max": 1.1055026054382324,170 "sampling/importance_sampling_ratio/mean": 0.9579700231552124,171 "sampling/importance_sampling_ratio/min": 0.850264310836792,172 "sampling/sampling_logp_difference/max": 0.21829354763031006,173 "sampling/sampling_logp_difference/mean": 0.00928188394755125,174 "step": 5,175 "step_time": 30.61694531404646176 },177 {178 "clip_ratio/high_max": 0.0,179 "clip_ratio/high_mean": 0.0,180 "clip_ratio/low_mean": 0.0,181 "clip_ratio/low_min": 0.0,182 "clip_ratio/region_mean": 0.0,183 "completions/clipped_ratio": 0.0,184 "completions/max_length": 50.0,185 "completions/max_terminated_length": 50.0,186 "completions/mean_length": 44.5,187 "completions/mean_terminated_length": 44.5,188 "completions/min_length": 37.0,189 "completions/min_terminated_length": 37.0,190 "entropy": 0.19655971229076385,191 "epoch": 0.006,192 "frac_reward_zero_std": 0.0,193 "grad_norm": 4.993420124053955,194 "kl": 0.001883858349174261,195 "learning_rate": 8.333333333333333e-07,196 "loss": -0.3115,197 "num_tokens": 16326.0,198 "reward": 0.9900000095367432,199 "reward_std": 0.008164958097040653,200 "rewards/reward_func/mean": 0.9900000095367432,201 "rewards/reward_func/std": 0.008164958097040653,202 "sampling/importance_sampling_ratio/max": 1.9480223655700684,203 "sampling/importance_sampling_ratio/mean": 1.2064706087112427,204 "sampling/importance_sampling_ratio/min": 0.786892831325531,205 "sampling/sampling_logp_difference/max": 0.552229642868042,206 "sampling/sampling_logp_difference/mean": 0.02012234926223755,207 "step": 6,208 "step_time": 13.275526605022606209 },210 {211 "clip_ratio/high_max": 0.0,212 "clip_ratio/high_mean": 0.0,213 "clip_ratio/low_mean": 0.0,214 "clip_ratio/low_min": 0.0,215 "clip_ratio/region_mean": 0.0,216 "completions/clipped_ratio": 0.0,217 "completions/max_length": 69.0,218 "completions/max_terminated_length": 69.0,219 "completions/mean_length": 57.75,220 "completions/mean_terminated_length": 57.75,221 "completions/min_length": 44.0,222 "completions/min_terminated_length": 44.0,223 "entropy": 0.21491773426532745,224 "epoch": 0.007,225 "frac_reward_zero_std": 0.0,226 "grad_norm": 0.6698219776153564,227 "kl": 0.001308084581978619,228 "learning_rate": 1.0000000000000002e-06,229 "loss": 0.0146,230 "num_tokens": 19580.0,231 "reward": 0.20000000298023224,232 "reward_std": 0.5343531966209412,233 "rewards/reward_func/mean": 0.20000000298023224,234 "rewards/reward_func/std": 0.5343531966209412,235 "sampling/importance_sampling_ratio/max": 0.8031184077262878,236 "sampling/importance_sampling_ratio/mean": 0.6652683019638062,237 "sampling/importance_sampling_ratio/min": 0.511317789554596,238 "sampling/sampling_logp_difference/max": 0.4191169738769531,239 "sampling/sampling_logp_difference/mean": 0.019288605079054832,240 "step": 7,241 "step_time": 33.92142505198717242 },243 {244 "clip_ratio/high_max": 0.0,245 "clip_ratio/high_mean": 0.0,246 "clip_ratio/low_mean": 0.0,247 "clip_ratio/low_min": 0.0,248 "clip_ratio/region_mean": 0.0,249 "completions/clipped_ratio": 0.0,250 "completions/max_length": 55.0,251 "completions/max_terminated_length": 55.0,252 "completions/mean_length": 48.75,253 "completions/mean_terminated_length": 48.75,254 "completions/min_length": 37.0,255 "completions/min_terminated_length": 37.0,256 "entropy": 0.1761166900396347,257 "epoch": 0.008,258 "frac_reward_zero_std": 0.0,259 "grad_norm": 1.2735919952392578,260 "kl": 0.0024672183208167553,261 "learning_rate": 1.1666666666666668e-06,262 "loss": 0.0549,263 "num_tokens": 22411.0,264 "reward": -0.06499999761581421,265 "reward_std": 0.05916079506278038,266 "rewards/reward_func/mean": -0.06499999761581421,267 "rewards/reward_func/std": 0.05916079878807068,268 "sampling/importance_sampling_ratio/max": 1.1058381795883179,269 "sampling/importance_sampling_ratio/mean": 0.9285240173339844,270 "sampling/importance_sampling_ratio/min": 0.6420994997024536,271 "sampling/sampling_logp_difference/max": 0.3237142562866211,272 "sampling/sampling_logp_difference/mean": 0.01667601801455021,273 "step": 8,274 "step_time": 27.820944591017906275 },276 {277 "clip_ratio/high_max": 0.0,278 "clip_ratio/high_mean": 0.0,279 "clip_ratio/low_mean": 0.0,280 "clip_ratio/low_min": 0.0,281 "clip_ratio/region_mean": 0.0,282 "completions/clipped_ratio": 0.0,283 "completions/max_length": 70.0,284 "completions/max_terminated_length": 70.0,285 "completions/mean_length": 55.25,286 "completions/mean_terminated_length": 55.25,287 "completions/min_length": 45.0,288 "completions/min_terminated_length": 45.0,289 "entropy": 0.19499044120311737,290 "epoch": 0.009,291 "frac_reward_zero_std": 0.0,292 "grad_norm": 0.6493493914604187,293 "kl": 0.0014519351534545422,294 "learning_rate": 1.3333333333333334e-06,295 "loss": 0.027,296 "num_tokens": 25058.0,297 "reward": 0.20749999582767487,298 "reward_std": 0.5302436351776123,299 "rewards/reward_func/mean": 0.20749999582767487,300 "rewards/reward_func/std": 0.5302436351776123,301 "sampling/importance_sampling_ratio/max": 1.536926031112671,302 "sampling/importance_sampling_ratio/mean": 0.8416682481765747,303 "sampling/importance_sampling_ratio/min": 0.5120096802711487,304 "sampling/sampling_logp_difference/max": 0.3361164331436157,305 "sampling/sampling_logp_difference/mean": 0.015237247571349144,306 "step": 9,307 "step_time": 23.23614341497887308 },309 {310 "clip_ratio/high_max": 0.0,311 "clip_ratio/high_mean": 0.0,312 "clip_ratio/low_mean": 0.0,313 "clip_ratio/low_min": 0.0,314 "clip_ratio/region_mean": 0.0,315 "completions/clipped_ratio": 0.0,316 "completions/max_length": 54.0,317 "completions/max_terminated_length": 54.0,318 "completions/mean_length": 45.5,319 "completions/mean_terminated_length": 45.5,320 "completions/min_length": 38.0,321 "completions/min_terminated_length": 38.0,322 "entropy": 0.18325170874595642,323 "epoch": 0.01,324 "frac_reward_zero_std": 0.0,325 "grad_norm": 1.2250245809555054,326 "kl": 0.0007307051564566791,327 "learning_rate": 1.5e-06,328 "loss": -0.0388,329 "num_tokens": 27748.0,330 "reward": 0.20499999821186066,331 "reward_std": 0.525642454624176,332 "rewards/reward_func/mean": 0.20499999821186066,333 "rewards/reward_func/std": 0.525642454624176,334 "sampling/importance_sampling_ratio/max": 1.5835230350494385,335 "sampling/importance_sampling_ratio/mean": 1.143486499786377,336 "sampling/importance_sampling_ratio/min": 0.7374402284622192,337 "sampling/sampling_logp_difference/max": 0.3930445909500122,338 "sampling/sampling_logp_difference/mean": 0.01502042543143034,339 "step": 10,340 "step_time": 27.988202619017102341 },342 {343 "clip_ratio/high_max": 0.0,344 "clip_ratio/high_mean": 0.0,345 "clip_ratio/low_mean": 0.0,346 "clip_ratio/low_min": 0.0,347 "clip_ratio/region_mean": 0.0,348 "completions/clipped_ratio": 0.0,349 "completions/max_length": 73.0,350 "completions/max_terminated_length": 73.0,351 "completions/mean_length": 61.25,352 "completions/mean_terminated_length": 61.25,353 "completions/min_length": 52.0,354 "completions/min_terminated_length": 52.0,355 "entropy": 0.21740968525409698,356 "epoch": 0.011,357 "frac_reward_zero_std": 0.0,358 "grad_norm": 1.256651520729065,359 "kl": 0.0032303757034242153,360 "learning_rate": 1.6666666666666667e-06,361 "loss": -0.0885,362 "num_tokens": 30979.0,363 "reward": 0.75,364 "reward_std": 0.5,365 "rewards/reward_func/mean": 0.75,366 "rewards/reward_func/std": 0.5,367 "sampling/importance_sampling_ratio/max": 1.3089040517807007,368 "sampling/importance_sampling_ratio/mean": 1.1207859516143799,369 "sampling/importance_sampling_ratio/min": 0.7530092000961304,370 "sampling/sampling_logp_difference/max": 0.8883767127990723,371 "sampling/sampling_logp_difference/mean": 0.020340321585536003,372 "step": 11,373 "step_time": 19.88501465099398374 },375 {376 "clip_ratio/high_max": 0.0,377 "clip_ratio/high_mean": 0.0,378 "clip_ratio/low_mean": 0.0,379 "clip_ratio/low_min": 0.0,380 "clip_ratio/region_mean": 0.0,381 "completions/clipped_ratio": 0.0,382 "completions/max_length": 51.0,383 "completions/max_terminated_length": 51.0,384 "completions/mean_length": 47.5,385 "completions/mean_terminated_length": 47.5,386 "completions/min_length": 40.0,387 "completions/min_terminated_length": 40.0,388 "entropy": 0.17663566768169403,389 "epoch": 0.012,390 "frac_reward_zero_std": 0.0,391 "grad_norm": 2.204960346221924,392 "kl": 0.0012955483980476856,393 "learning_rate": 1.8333333333333333e-06,394 "loss": -0.1753,395 "num_tokens": 33906.0,396 "reward": 0.4775000214576721,397 "reward_std": 0.5980733036994934,398 "rewards/reward_func/mean": 0.4775000214576721,399 "rewards/reward_func/std": 0.5980733036994934,400 "sampling/importance_sampling_ratio/max": 1.6643222570419312,401 "sampling/importance_sampling_ratio/mean": 0.8751918077468872,402 "sampling/importance_sampling_ratio/min": 0.39312252402305603,403 "sampling/sampling_logp_difference/max": 0.4464702606201172,404 "sampling/sampling_logp_difference/mean": 0.018733447417616844,405 "step": 12,406 "step_time": 23.57167529099388407 },408 {409 "clip_ratio/high_max": 0.0,410 "clip_ratio/high_mean": 0.0,411 "clip_ratio/low_mean": 0.0,412 "clip_ratio/low_min": 0.0,413 "clip_ratio/region_mean": 0.0,414 "completions/clipped_ratio": 0.0,415 "completions/max_length": 58.0,416 "completions/max_terminated_length": 58.0,417 "completions/mean_length": 55.75,418 "completions/mean_terminated_length": 55.75,419 "completions/min_length": 53.0,420 "completions/min_terminated_length": 53.0,421 "entropy": 0.2222597599029541,422 "epoch": 0.013,423 "frac_reward_zero_std": 0.0,424 "grad_norm": 1.3614099025726318,425 "kl": 0.0021189097315073013,426 "learning_rate": 2.0000000000000003e-06,427 "loss": -0.3654,428 "num_tokens": 37185.0,429 "reward": 0.4975000023841858,430 "reward_std": 0.5802513957023621,431 "rewards/reward_func/mean": 0.4975000023841858,432 "rewards/reward_func/std": 0.5802513957023621,433 "sampling/importance_sampling_ratio/max": 2.2725558280944824,434 "sampling/importance_sampling_ratio/mean": 1.10244619846344,435 "sampling/importance_sampling_ratio/min": 0.41275277733802795,436 "sampling/sampling_logp_difference/max": 0.49567699432373047,437 "sampling/sampling_logp_difference/mean": 0.0183317419141531,438 "step": 13,439 "step_time": 27.432764305965975440 },441 {442 "clip_ratio/high_max": 0.0,443 "clip_ratio/high_mean": 0.0,444 "clip_ratio/low_mean": 0.0,445 "clip_ratio/low_min": 0.0,446 "clip_ratio/region_mean": 0.0,447 "completions/clipped_ratio": 0.0,448 "completions/max_length": 69.0,449 "completions/max_terminated_length": 69.0,450 "completions/mean_length": 53.0,451 "completions/mean_terminated_length": 53.0,452 "completions/min_length": 43.0,453 "completions/min_terminated_length": 43.0,454 "entropy": 0.17930717766284943,455 "epoch": 0.014,456 "frac_reward_zero_std": 0.0,457 "grad_norm": 1.4246803522109985,458 "kl": 0.0013900657650083303,459 "learning_rate": 2.166666666666667e-06,460 "loss": 0.0577,461 "num_tokens": 39521.0,462 "reward": 0.9900000095367432,463 "reward_std": 0.019999999552965164,464 "rewards/reward_func/mean": 0.9900000095367432,465 "rewards/reward_func/std": 0.02000001072883606,466 "sampling/importance_sampling_ratio/max": 1.34236478805542,467 "sampling/importance_sampling_ratio/mean": 1.1689629554748535,468 "sampling/importance_sampling_ratio/min": 1.0116561651229858,469 "sampling/sampling_logp_difference/max": 0.3754175901412964,470 "sampling/sampling_logp_difference/mean": 0.01138111762702465,471 "step": 14,472 "step_time": 12.762974106008187473 },474 {475 "clip_ratio/high_max": 0.0,476 "clip_ratio/high_mean": 0.0,477 "clip_ratio/low_mean": 0.0,478 "clip_ratio/low_min": 0.0,479 "clip_ratio/region_mean": 0.0,480 "completions/clipped_ratio": 0.0,481 "completions/max_length": 72.0,482 "completions/max_terminated_length": 72.0,483 "completions/mean_length": 56.75,484 "completions/mean_terminated_length": 56.75,485 "completions/min_length": 47.0,486 "completions/min_terminated_length": 47.0,487 "entropy": 0.181468665599823,488 "epoch": 0.015,489 "frac_reward_zero_std": 0.0,490 "grad_norm": 0.7324386835098267,491 "kl": 0.0012912629172205925,492 "learning_rate": 2.3333333333333336e-06,493 "loss": 0.2737,494 "num_tokens": 42388.0,495 "reward": 0.23750001192092896,496 "reward_std": 0.5084207653999329,497 "rewards/reward_func/mean": 0.23750001192092896,498 "rewards/reward_func/std": 0.5084207653999329,499 "sampling/importance_sampling_ratio/max": 1.5870178937911987,500 "sampling/importance_sampling_ratio/mean": 1.0515062808990479,501 "sampling/importance_sampling_ratio/min": 0.5773301720619202,502 "sampling/sampling_logp_difference/max": 0.6747937202453613,503 "sampling/sampling_logp_difference/mean": 0.015356915071606636,504 "step": 15,505 "step_time": 28.87673272797838506 },507 {508 "clip_ratio/high_max": 0.0,509 "clip_ratio/high_mean": 0.0,510 "clip_ratio/low_mean": 0.0,511 "clip_ratio/low_min": 0.0,512 "clip_ratio/region_mean": 0.0,513 "completions/clipped_ratio": 0.0,514 "completions/max_length": 54.0,515 "completions/max_terminated_length": 54.0,516 "completions/mean_length": 46.75,517 "completions/mean_terminated_length": 46.75,518 "completions/min_length": 36.0,519 "completions/min_terminated_length": 36.0,520 "entropy": 0.17757576704025269,521 "epoch": 0.016,522 "frac_reward_zero_std": 0.0,523 "grad_norm": 0.7791656851768494,524 "kl": 0.0011586352484300733,525 "learning_rate": 2.5e-06,526 "loss": 0.0899,527 "num_tokens": 45137.0,528 "reward": 0.21000000834465027,529 "reward_std": 0.5212165117263794,530 "rewards/reward_func/mean": 0.21000000834465027,531 "rewards/reward_func/std": 0.5212165117263794,532 "sampling/importance_sampling_ratio/max": 0.9811292290687561,533 "sampling/importance_sampling_ratio/mean": 0.8586262464523315,534 "sampling/importance_sampling_ratio/min": 0.6450943946838379,535 "sampling/sampling_logp_difference/max": 0.31294405460357666,536 "sampling/sampling_logp_difference/mean": 0.011509422212839127,537 "step": 16,538 "step_time": 29.512227962026373539 },540 {541 "clip_ratio/high_max": 0.0,542 "clip_ratio/high_mean": 0.0,543 "clip_ratio/low_mean": 0.0,544 "clip_ratio/low_min": 0.0,545 "clip_ratio/region_mean": 0.0,546 "completions/clipped_ratio": 0.0,547 "completions/max_length": 66.0,548 "completions/max_terminated_length": 66.0,549 "completions/mean_length": 54.0,550 "completions/mean_terminated_length": 54.0,551 "completions/min_length": 41.0,552 "completions/min_terminated_length": 41.0,553 "entropy": 0.2108621597290039,554 "epoch": 0.017,555 "frac_reward_zero_std": 1.0,556 "grad_norm": 0.0017789137782528996,557 "kl": 0.0008321161731146276,558 "learning_rate": 2.666666666666667e-06,559 "loss": 0.0,560 "num_tokens": 47503.0,561 "reward": 1.0,562 "reward_std": 0.0,563 "rewards/reward_func/mean": 1.0,564 "rewards/reward_func/std": 0.0,565 "sampling/importance_sampling_ratio/max": 1.2581455707550049,566 "sampling/importance_sampling_ratio/mean": 1.0041334629058838,567 "sampling/importance_sampling_ratio/min": 0.7038960456848145,568 "sampling/sampling_logp_difference/max": 0.33078861236572266,569 "sampling/sampling_logp_difference/mean": 0.012301802635192871,570 "step": 17,571 "step_time": 9.200428290001582572 },573 {574 "clip_ratio/high_max": 0.0,575 "clip_ratio/high_mean": 0.0,576 "clip_ratio/low_mean": 0.0,577 "clip_ratio/low_min": 0.0,578 "clip_ratio/region_mean": 0.0,579 "completions/clipped_ratio": 0.0,580 "completions/max_length": 63.0,581 "completions/max_terminated_length": 63.0,582 "completions/mean_length": 54.75,583 "completions/mean_terminated_length": 54.75,584 "completions/min_length": 49.0,585 "completions/min_terminated_length": 49.0,586 "entropy": 0.2184532880783081,587 "epoch": 0.018,588 "frac_reward_zero_std": 0.0,589 "grad_norm": 1.601157784461975,590 "kl": 0.0019322438165545464,591 "learning_rate": 2.8333333333333335e-06,592 "loss": 0.4322,593 "num_tokens": 50320.0,594 "reward": 0.4775000214576721,595 "reward_std": 0.5804811120033264,596 "rewards/reward_func/mean": 0.4775000214576721,597 "rewards/reward_func/std": 0.5804811120033264,598 "sampling/importance_sampling_ratio/max": 1.7848742008209229,599 "sampling/importance_sampling_ratio/mean": 1.0737558603286743,600 "sampling/importance_sampling_ratio/min": 0.3417838513851166,601 "sampling/sampling_logp_difference/max": 0.6254115104675293,602 "sampling/sampling_logp_difference/mean": 0.023144036531448364,603 "step": 18,604 "step_time": 27.39610268798424605 },606 {607 "clip_ratio/high_max": 0.0,608 "clip_ratio/high_mean": 0.0,609 "clip_ratio/low_mean": 0.0,610 "clip_ratio/low_min": 0.0,611 "clip_ratio/region_mean": 0.0,612 "completions/clipped_ratio": 0.0,613 "completions/max_length": 65.0,614 "completions/max_terminated_length": 65.0,615 "completions/mean_length": 62.75,616 "completions/mean_terminated_length": 62.75,617 "completions/min_length": 59.0,618 "completions/min_terminated_length": 59.0,619 "entropy": 0.23026728630065918,620 "epoch": 0.019,621 "frac_reward_zero_std": 0.0,622 "grad_norm": 0.7251322865486145,623 "kl": 0.001008337247185409,624 "learning_rate": 3e-06,625 "loss": -0.1061,626 "num_tokens": 52849.0,627 "reward": 0.6749999523162842,628 "reward_std": 0.538918673992157,629 "rewards/reward_func/mean": 0.6749999523162842,630 "rewards/reward_func/std": 0.538918673992157,631 "sampling/importance_sampling_ratio/max": 1.2397409677505493,632 "sampling/importance_sampling_ratio/mean": 0.8263118863105774,633 "sampling/importance_sampling_ratio/min": 0.6256706118583679,634 "sampling/sampling_logp_difference/max": 0.4246499538421631,635 "sampling/sampling_logp_difference/mean": 0.01839565299451351,636 "step": 19,637 "step_time": 23.762960355030373638 },639 {640 "clip_ratio/high_max": 0.0,641 "clip_ratio/high_mean": 0.0,642 "clip_ratio/low_mean": 0.0,643 "clip_ratio/low_min": 0.0,644 "clip_ratio/region_mean": 0.0,645 "completions/clipped_ratio": 0.0,646 "completions/max_length": 63.0,647 "completions/max_terminated_length": 63.0,648 "completions/mean_length": 55.5,649 "completions/mean_terminated_length": 55.5,650 "completions/min_length": 49.0,651 "completions/min_terminated_length": 49.0,652 "entropy": 0.20103175938129425,653 "epoch": 0.02,654 "frac_reward_zero_std": 0.0,655 "grad_norm": 0.8409279584884644,656 "kl": 0.0011635932605713606,657 "learning_rate": 3.1666666666666667e-06,658 "loss": 0.02,659 "num_tokens": 55976.0,660 "reward": 0.1899999976158142,661 "reward_std": 0.5212165117263794,662 "rewards/reward_func/mean": 0.1899999976158142,663 "rewards/reward_func/std": 0.5212165713310242,664 "sampling/importance_sampling_ratio/max": 0.8928262591362,665 "sampling/importance_sampling_ratio/mean": 0.8174147009849548,666 "sampling/importance_sampling_ratio/min": 0.6498741507530212,667 "sampling/sampling_logp_difference/max": 0.3282057046890259,668 "sampling/sampling_logp_difference/mean": 0.016716880723834038,669 "step": 20,670 "step_time": 34.79140626901062671 },672 {673 "clip_ratio/high_max": 0.0,674 "clip_ratio/high_mean": 0.0,675 "clip_ratio/low_mean": 0.0,676 "clip_ratio/low_min": 0.0,677 "clip_ratio/region_mean": 0.0,678 "completions/clipped_ratio": 0.0,679 "completions/max_length": 57.0,680 "completions/max_terminated_length": 57.0,681 "completions/mean_length": 54.75,682 "completions/mean_terminated_length": 54.75,683 "completions/min_length": 52.0,684 "completions/min_terminated_length": 52.0,685 "entropy": 0.24307209253311157,686 "epoch": 0.021,687 "frac_reward_zero_std": 0.0,688 "grad_norm": 2.4089884757995605,689 "kl": 0.005192960146814585,690 "learning_rate": 3.3333333333333333e-06,691 "loss": 0.0297,692 "num_tokens": 58947.0,693 "reward": 0.49000000953674316,694 "reward_std": 0.5889538526535034,695 "rewards/reward_func/mean": 0.49000000953674316,696 "rewards/reward_func/std": 0.5889538526535034,697 "sampling/importance_sampling_ratio/max": 1.8641705513000488,698 "sampling/importance_sampling_ratio/mean": 1.017917275428772,699 "sampling/importance_sampling_ratio/min": 0.15175653994083405,700 "sampling/sampling_logp_difference/max": 1.413463830947876,701 "sampling/sampling_logp_difference/mean": 0.038106679916381836,702 "step": 21,703 "step_time": 19.72863965400029704 },705 {706 "clip_ratio/high_max": 0.0,707 "clip_ratio/high_mean": 0.0,708 "clip_ratio/low_mean": 0.0,709 "clip_ratio/low_min": 0.0,710 "clip_ratio/region_mean": 0.0,711 "completions/clipped_ratio": 0.0,712 "completions/max_length": 65.0,713 "completions/max_terminated_length": 65.0,714 "completions/mean_length": 55.5,715 "completions/mean_terminated_length": 55.5,716 "completions/min_length": 46.0,717 "completions/min_terminated_length": 46.0,718 "entropy": 0.16037502884864807,719 "epoch": 0.022,720 "frac_reward_zero_std": 0.0,721 "grad_norm": 1.9025684595108032,722 "kl": 0.0024386204313486814,723 "learning_rate": 3.5e-06,724 "loss": -0.3497,725 "num_tokens": 61866.0,726 "reward": 0.4724999964237213,727 "reward_std": 0.6091181635856628,728 "rewards/reward_func/mean": 0.4724999964237213,729 "rewards/reward_func/std": 0.6091182231903076,730 "sampling/importance_sampling_ratio/max": 2.9466702938079834,731 "sampling/importance_sampling_ratio/mean": 1.4579466581344604,732 "sampling/importance_sampling_ratio/min": 0.7423606514930725,733 "sampling/sampling_logp_difference/max": 0.34435367584228516,734 "sampling/sampling_logp_difference/mean": 0.018071508035063744,735 "step": 22,736 "step_time": 23.186725946026854737 },738 {739 "clip_ratio/high_max": 0.0,740 "clip_ratio/high_mean": 0.0,741 "clip_ratio/low_mean": 0.0,742 "clip_ratio/low_min": 0.0,743 "clip_ratio/region_mean": 0.0,744 "completions/clipped_ratio": 0.0,745 "completions/max_length": 64.0,746 "completions/max_terminated_length": 64.0,747 "completions/mean_length": 56.75,748 "completions/mean_terminated_length": 56.75,749 "completions/min_length": 52.0,750 "completions/min_terminated_length": 52.0,751 "entropy": 0.19720782339572906,752 "epoch": 0.023,753 "frac_reward_zero_std": 0.0,754 "grad_norm": 0.9149781465530396,755 "kl": 0.0011606084881350398,756 "learning_rate": 3.6666666666666666e-06,757 "loss": 0.0361,758 "num_tokens": 64949.0,759 "reward": 0.48750001192092896,760 "reward_std": 0.5921359658241272,761 "rewards/reward_func/mean": 0.48750001192092896,762 "rewards/reward_func/std": 0.592136025428772,763 "sampling/importance_sampling_ratio/max": 1.0435854196548462,764 "sampling/importance_sampling_ratio/mean": 0.862576961517334,765 "sampling/importance_sampling_ratio/min": 0.7559348344802856,766 "sampling/sampling_logp_difference/max": 0.34287071228027344,767 "sampling/sampling_logp_difference/mean": 0.017276717349886894,768 "step": 23,769 "step_time": 22.19894163700519770 },771 {772 "clip_ratio/high_max": 0.0,773 "clip_ratio/high_mean": 0.0,774 "clip_ratio/low_mean": 0.0,775 "clip_ratio/low_min": 0.0,776 "clip_ratio/region_mean": 0.0,777 "completions/clipped_ratio": 0.0,778 "completions/max_length": 67.0,779 "completions/max_terminated_length": 67.0,780 "completions/mean_length": 54.75,781 "completions/mean_terminated_length": 54.75,782 "completions/min_length": 43.0,783 "completions/min_terminated_length": 43.0,784 "entropy": 0.15801791846752167,785 "epoch": 0.024,786 "frac_reward_zero_std": 0.0,787 "grad_norm": 0.7708470821380615,788 "kl": 0.000878384686075151,789 "learning_rate": 3.833333333333334e-06,790 "loss": -0.0433,791 "num_tokens": 67889.0,792 "reward": 0.7124999761581421,793 "reward_std": 0.535062313079834,794 "rewards/reward_func/mean": 0.7124999761581421,795 "rewards/reward_func/std": 0.5350623726844788,796 "sampling/importance_sampling_ratio/max": 1.8083628416061401,797 "sampling/importance_sampling_ratio/mean": 0.807941198348999,798 "sampling/importance_sampling_ratio/min": 0.3332298994064331,799 "sampling/sampling_logp_difference/max": 0.474312424659729,800 "sampling/sampling_logp_difference/mean": 0.021255692467093468,801 "step": 24,802 "step_time": 28.712539033964276803 },804 {805 "clip_ratio/high_max": 0.0,806 "clip_ratio/high_mean": 0.0,807 "clip_ratio/low_mean": 0.0,808 "clip_ratio/low_min": 0.0,809 "clip_ratio/region_mean": 0.0,810 "completions/clipped_ratio": 0.0,811 "completions/max_length": 58.0,812 "completions/max_terminated_length": 58.0,813 "completions/mean_length": 53.25,814 "completions/mean_terminated_length": 53.25,815 "completions/min_length": 44.0,816 "completions/min_terminated_length": 44.0,817 "entropy": 0.19440847635269165,818 "epoch": 0.025,819 "frac_reward_zero_std": 0.0,820 "grad_norm": 3.9135429859161377,821 "kl": 0.001603958779014647,822 "learning_rate": 4.000000000000001e-06,823 "loss": 0.3185,824 "num_tokens": 70486.0,825 "reward": 0.7175000309944153,826 "reward_std": 0.4797481894493103,827 "rewards/reward_func/mean": 0.7175000309944153,828 "rewards/reward_func/std": 0.4797481894493103,829 "sampling/importance_sampling_ratio/max": 2.502864360809326,830 "sampling/importance_sampling_ratio/mean": 1.469748616218567,831 "sampling/importance_sampling_ratio/min": 0.8149593472480774,832 "sampling/sampling_logp_difference/max": 0.3406977653503418,833 "sampling/sampling_logp_difference/mean": 0.017325270920991898,834 "step": 25,835 "step_time": 21.12462861801032836 },837 {838 "clip_ratio/high_max": 0.0,839 "clip_ratio/high_mean": 0.0,840 "clip_ratio/low_mean": 0.0,841 "clip_ratio/low_min": 0.0,842 "clip_ratio/region_mean": 0.0,843 "completions/clipped_ratio": 0.0,844 "completions/max_length": 59.0,845 "completions/max_terminated_length": 59.0,846 "completions/mean_length": 53.25,847 "completions/mean_terminated_length": 53.25,848 "completions/min_length": 47.0,849 "completions/min_terminated_length": 47.0,850 "entropy": 0.18733122944831848,851 "epoch": 0.026,852 "frac_reward_zero_std": 0.0,853 "grad_norm": 1.9654920101165771,854 "kl": 0.0021408493630588055,855 "learning_rate": 4.166666666666667e-06,856 "loss": 0.0049,857 "num_tokens": 73363.0,858 "reward": 0.7325000166893005,859 "reward_std": 0.5283543467521667,860 "rewards/reward_func/mean": 0.7325000166893005,861 "rewards/reward_func/std": 0.5283544063568115,862 "sampling/importance_sampling_ratio/max": 1.5510928630828857,863 "sampling/importance_sampling_ratio/mean": 1.1689436435699463,864 "sampling/importance_sampling_ratio/min": 0.8162763714790344,865 "sampling/sampling_logp_difference/max": 0.7810912132263184,866 "sampling/sampling_logp_difference/mean": 0.018673431128263474,867 "step": 26,868 "step_time": 21.690037710010074869 },870 {871 "clip_ratio/high_max": 0.0,872 "clip_ratio/high_mean": 0.0,873 "clip_ratio/low_mean": 0.0,874 "clip_ratio/low_min": 0.0,875 "clip_ratio/region_mean": 0.0,876 "completions/clipped_ratio": 0.0,877 "completions/max_length": 62.0,878 "completions/max_terminated_length": 62.0,879 "completions/mean_length": 55.75,880 "completions/mean_terminated_length": 55.75,881 "completions/min_length": 48.0,882 "completions/min_terminated_length": 48.0,883 "entropy": 0.24256083369255066,884 "epoch": 0.027,885 "frac_reward_zero_std": 0.0,886 "grad_norm": 0.740149199962616,887 "kl": 0.00063083361601457,888 "learning_rate": 4.333333333333334e-06,889 "loss": -0.0407,890 "num_tokens": 75881.0,891 "reward": 0.9900000095367432,892 "reward_std": 0.019999999552965164,893 "rewards/reward_func/mean": 0.9900000095367432,894 "rewards/reward_func/std": 0.02000001072883606,895 "sampling/importance_sampling_ratio/max": 1.0160430669784546,896 "sampling/importance_sampling_ratio/mean": 0.7664977312088013,897 "sampling/importance_sampling_ratio/min": 0.5705021619796753,898 "sampling/sampling_logp_difference/max": 0.3010873794555664,899 "sampling/sampling_logp_difference/mean": 0.018683861941099167,900 "step": 27,901 "step_time": 12.669248107995372902 },903 {904 "clip_ratio/high_max": 0.0,905 "clip_ratio/high_mean": 0.0,906 "clip_ratio/low_mean": 0.0,907 "clip_ratio/low_min": 0.0,908 "clip_ratio/region_mean": 0.0,909 "completions/clipped_ratio": 0.0,910 "completions/max_length": 62.0,911 "completions/max_terminated_length": 62.0,912 "completions/mean_length": 53.75,913 "completions/mean_terminated_length": 53.75,914 "completions/min_length": 42.0,915 "completions/min_terminated_length": 42.0,916 "entropy": 0.2366439402103424,917 "epoch": 0.028,918 "frac_reward_zero_std": 0.0,919 "grad_norm": 1.0369964838027954,920 "kl": 0.0012403648579493165,921 "learning_rate": 4.5e-06,922 "loss": -0.1528,923 "num_tokens": 78878.0,924 "reward": 0.4925000071525574,925 "reward_std": 0.5860247611999512,926 "rewards/reward_func/mean": 0.4925000071525574,927 "rewards/reward_func/std": 0.5860247015953064,928 "sampling/importance_sampling_ratio/max": 1.1833999156951904,929 "sampling/importance_sampling_ratio/mean": 0.9274792671203613,930 "sampling/importance_sampling_ratio/min": 0.6121615767478943,931 "sampling/sampling_logp_difference/max": 0.3726520538330078,932 "sampling/sampling_logp_difference/mean": 0.016752678900957108,933 "step": 28,934 "step_time": 22.686215320019983935 },936 {937 "clip_ratio/high_max": 0.0,938 "clip_ratio/high_mean": 0.0,939 "clip_ratio/low_mean": 0.0,940 "clip_ratio/low_min": 0.0,941 "clip_ratio/region_mean": 0.0,942 "completions/clipped_ratio": 0.0,943 "completions/max_length": 59.0,944 "completions/max_terminated_length": 59.0,945 "completions/mean_length": 54.25,946 "completions/mean_terminated_length": 54.25,947 "completions/min_length": 51.0,948 "completions/min_terminated_length": 51.0,949 "entropy": 0.17702580988407135,950 "epoch": 0.029,951 "frac_reward_zero_std": 0.0,952 "grad_norm": 0.9103659391403198,953 "kl": 0.0025775833055377007,954 "learning_rate": 4.666666666666667e-06,955 "loss": 0.0843,956 "num_tokens": 82198.0,957 "reward": 0.20000000298023224,958 "reward_std": 0.5358482599258423,959 "rewards/reward_func/mean": 0.20000000298023224,960 "rewards/reward_func/std": 0.5358482599258423,961 "sampling/importance_sampling_ratio/max": 1.0262494087219238,962 "sampling/importance_sampling_ratio/mean": 0.8293523788452148,963 "sampling/importance_sampling_ratio/min": 0.6377174854278564,964 "sampling/sampling_logp_difference/max": 0.5646309852600098,965 "sampling/sampling_logp_difference/mean": 0.020577674731612206,966 "step": 29,967 "step_time": 33.133232187945396968 },969 {970 "clip_ratio/high_max": 0.0,971 "clip_ratio/high_mean": 0.0,972 "clip_ratio/low_mean": 0.0,973 "clip_ratio/low_min": 0.0,974 "clip_ratio/region_mean": 0.0,975 "completions/clipped_ratio": 0.0,976 "completions/max_length": 55.0,977 "completions/max_terminated_length": 55.0,978 "completions/mean_length": 49.25,979 "completions/mean_terminated_length": 49.25,980 "completions/min_length": 40.0,981 "completions/min_terminated_length": 40.0,982 "entropy": 0.19803452491760254,983 "epoch": 0.03,984 "frac_reward_zero_std": 0.0,985 "grad_norm": 1.4241100549697876,986 "kl": 0.0012677285121753812,987 "learning_rate": 4.833333333333333e-06,988 "loss": 0.0696,989 "num_tokens": 85079.0,990 "reward": 0.19749999046325684,991 "reward_std": 0.5361825227737427,992 "rewards/reward_func/mean": 0.19749999046325684,993 "rewards/reward_func/std": 0.5361825227737427,994 "sampling/importance_sampling_ratio/max": 1.7217131853103638,995 "sampling/importance_sampling_ratio/mean": 1.4462528228759766,996 "sampling/importance_sampling_ratio/min": 1.1936447620391846,997 "sampling/sampling_logp_difference/max": 0.3546537160873413,998 "sampling/sampling_logp_difference/mean": 0.015671029686927795,999 "step": 30,1000 "step_time": 32.311564416042531001 },1002 {1003 "clip_ratio/high_max": 0.0,1004 "clip_ratio/high_mean": 0.0,1005 "clip_ratio/low_mean": 0.0,1006 "clip_ratio/low_min": 0.0,1007 "clip_ratio/region_mean": 0.0,1008 "completions/clipped_ratio": 0.0,1009 "completions/max_length": 53.0,1010 "completions/max_terminated_length": 53.0,1011 "completions/mean_length": 47.0,1012 "completions/mean_terminated_length": 47.0,1013 "completions/min_length": 36.0,1014 "completions/min_terminated_length": 36.0,1015 "entropy": 0.2113637775182724,1016 "epoch": 0.031,1017 "frac_reward_zero_std": 0.0,1018 "grad_norm": 1.3625730276107788,1019 "kl": 0.0018731742165982723,1020 "learning_rate": 5e-06,1021 "loss": 0.2336,1022 "num_tokens": 87815.0,1023 "reward": 0.7100000381469727,1024 "reward_std": 0.5600595474243164,1025 "rewards/reward_func/mean": 0.7100000381469727,1026 "rewards/reward_func/std": 0.5600595474243164,1027 "sampling/importance_sampling_ratio/max": 1.1667720079421997,1028 "sampling/importance_sampling_ratio/mean": 0.7437838315963745,1029 "sampling/importance_sampling_ratio/min": 0.4153501093387604,1030 "sampling/sampling_logp_difference/max": 0.4128420352935791,1031 "sampling/sampling_logp_difference/mean": 0.016734153032302856,1032 "step": 31,1033 "step_time": 23.091439365991391034 },1035 {1036 "clip_ratio/high_max": 0.0,1037 "clip_ratio/high_mean": 0.0,1038 "clip_ratio/low_mean": 0.0,1039 "clip_ratio/low_min": 0.0,1040 "clip_ratio/region_mean": 0.0,1041 "completions/clipped_ratio": 0.0,1042 "completions/max_length": 76.0,1043 "completions/max_terminated_length": 76.0,1044 "completions/mean_length": 55.5,1045 "completions/mean_terminated_length": 55.5,1046 "completions/min_length": 40.0,1047 "completions/min_terminated_length": 40.0,1048 "entropy": 0.20192047953605652,1049 "epoch": 0.032,1050 "frac_reward_zero_std": 0.0,1051 "grad_norm": 1.5310053825378418,1052 "kl": 0.0009873858653008938,1053 "learning_rate": 4.99998688809149e-06,1054 "loss": -0.2058,1055 "num_tokens": 90384.0,1056 "reward": 0.7425000071525574,1057 "reward_std": 0.4950000047683716,1058 "rewards/reward_func/mean": 0.7425000071525574,1059 "rewards/reward_func/std": 0.4950000047683716,1060 "sampling/importance_sampling_ratio/max": 1.8033684492111206,1061 "sampling/importance_sampling_ratio/mean": 1.410750389099121,1062 "sampling/importance_sampling_ratio/min": 0.9831962585449219,1063 "sampling/sampling_logp_difference/max": 0.48407530784606934,1064 "sampling/sampling_logp_difference/mean": 0.01803717575967312,1065 "step": 32,1066 "step_time": 16.2323565160040741067 },1068 {1069 "clip_ratio/high_max": 0.0,1070 "clip_ratio/high_mean": 0.0,1071 "clip_ratio/low_mean": 0.0,1072 "clip_ratio/low_min": 0.0,1073 "clip_ratio/region_mean": 0.0,1074 "completions/clipped_ratio": 0.0,1075 "completions/max_length": 55.0,1076 "completions/max_terminated_length": 55.0,1077 "completions/mean_length": 54.0,1078 "completions/mean_terminated_length": 54.0,1079 "completions/min_length": 53.0,1080 "completions/min_terminated_length": 53.0,1081 "entropy": 0.15265171229839325,1082 "epoch": 0.033,1083 "frac_reward_zero_std": 0.0,1084 "grad_norm": 1.2789109945297241,1085 "kl": 0.0014273010892793536,1086 "learning_rate": 4.9999475525034974e-06,1087 "loss": 0.2443,1088 "num_tokens": 93459.0,1089 "reward": 0.21250000596046448,1090 "reward_std": 0.5273439288139343,1091 "rewards/reward_func/mean": 0.21250000596046448,1092 "rewards/reward_func/std": 0.5273439884185791,1093 "sampling/importance_sampling_ratio/max": 1.7152032852172852,1094 "sampling/importance_sampling_ratio/mean": 1.0911431312561035,1095 "sampling/importance_sampling_ratio/min": 0.6519092321395874,1096 "sampling/sampling_logp_difference/max": 0.3830188512802124,1097 "sampling/sampling_logp_difference/mean": 0.015168999321758747,1098 "step": 33,1099 "step_time": 27.381604071008041100 },1101 {1102 "clip_ratio/high_max": 0.0,1103 "clip_ratio/high_mean": 0.0,1104 "clip_ratio/low_mean": 0.0,1105 "clip_ratio/low_min": 0.0,1106 "clip_ratio/region_mean": 0.0,1107 "completions/clipped_ratio": 0.0,1108 "completions/max_length": 60.0,1109 "completions/max_terminated_length": 60.0,1110 "completions/mean_length": 56.0,1111 "completions/mean_terminated_length": 56.0,1112 "completions/min_length": 50.0,1113 "completions/min_terminated_length": 50.0,1114 "entropy": 0.16871626675128937,1115 "epoch": 0.034,1116 "frac_reward_zero_std": 0.0,1117 "grad_norm": 1.5993002653121948,1118 "kl": 0.0012848010519519448,1119 "learning_rate": 4.999881993648633e-06,1120 "loss": 0.6006,1121 "num_tokens": 96452.0,1122 "reward": 0.4700000286102295,1123 "reward_std": 0.5838379263877869,1124 "rewards/reward_func/mean": 0.4700000286102295,1125 "rewards/reward_func/std": 0.5838378667831421,1126 "sampling/importance_sampling_ratio/max": 2.7293713092803955,1127 "sampling/importance_sampling_ratio/mean": 1.2541234493255615,1128 "sampling/importance_sampling_ratio/min": 0.551033079624176,1129 "sampling/sampling_logp_difference/max": 0.4532332420349121,1130 "sampling/sampling_logp_difference/mean": 0.017988380044698715,1131 "step": 34,1132 "step_time": 27.344148867996411133 },1134 {1135 "clip_ratio/high_max": 0.0,1136 "clip_ratio/high_mean": 0.0,1137 "clip_ratio/low_mean": 0.0,1138 "clip_ratio/low_min": 0.0,1139 "clip_ratio/region_mean": 0.0,1140 "completions/clipped_ratio": 0.0,1141 "completions/max_length": 50.0,1142 "completions/max_terminated_length": 50.0,1143 "completions/mean_length": 48.25,1144 "completions/mean_terminated_length": 48.25,1145 "completions/min_length": 45.0,1146 "completions/min_terminated_length": 45.0,1147 "entropy": 0.1804264634847641,1148 "epoch": 0.035,1149 "frac_reward_zero_std": 0.0,1150 "grad_norm": 0.9022880792617798,1151 "kl": 0.0009833512594923377,1152 "learning_rate": 4.99979021221458e-06,1153 "loss": -0.0283,1154 "num_tokens": 99630.0,1155 "reward": 0.20499999821186066,1156 "reward_std": 0.5313190817832947,1157 "rewards/reward_func/mean": 0.20499999821186066,1158 "rewards/reward_func/std": 0.5313190817832947,1159 "sampling/importance_sampling_ratio/max": 0.9818359613418579,1160 "sampling/importance_sampling_ratio/mean": 0.7903547286987305,1161 "sampling/importance_sampling_ratio/min": 0.5214921236038208,1162 "sampling/sampling_logp_difference/max": 0.5723757743835449,1163 "sampling/sampling_logp_difference/mean": 0.020144499838352203,1164 "step": 35,1165 "step_time": 31.4636145040276461166 },1167 {1168 "clip_ratio/high_max": 0.0,1169 "clip_ratio/high_mean": 0.0,1170 "clip_ratio/low_mean": 0.0,1171 "clip_ratio/low_min": 0.0,1172 "clip_ratio/region_mean": 0.0,1173 "completions/clipped_ratio": 0.0,1174 "completions/max_length": 54.0,1175 "completions/max_terminated_length": 54.0,1176 "completions/mean_length": 50.0,1177 "completions/mean_terminated_length": 50.0,1178 "completions/min_length": 42.0,1179 "completions/min_terminated_length": 42.0,1180 "entropy": 0.18060731887817383,1181 "epoch": 0.036,1182 "frac_reward_zero_std": 0.0,1183 "grad_norm": 0.9135256409645081,1184 "kl": 0.001348503283225,1185 "learning_rate": 4.9996722091640805e-06,1186 "loss": 0.1413,1187 "num_tokens": 102328.0,1188 "reward": 0.45499998331069946,1189 "reward_std": 0.6297882795333862,1190 "rewards/reward_func/mean": 0.45499998331069946,1191 "rewards/reward_func/std": 0.629788339138031,1192 "sampling/importance_sampling_ratio/max": 1.2054076194763184,1193 "sampling/importance_sampling_ratio/mean": 0.7335603833198547,1194 "sampling/importance_sampling_ratio/min": 0.3148641586303711,1195 "sampling/sampling_logp_difference/max": 0.4715766906738281,1196 "sampling/sampling_logp_difference/mean": 0.02093822881579399,1197 "step": 36,1198 "step_time": 22.8150227979640481199 },1200 {