RussellNLP/qwen_path_3k
010
1{2 "best_global_step": null,3 "best_metric": null,4 "best_model_checkpoint": null,5 "epoch": 0.25,6 "eval_steps": 500,7 "global_step": 3000,8 "is_hyper_param_search": false,9 "is_local_process_zero": true,10 "is_world_process_zero": true,11 "log_history": [12 {13 "completion_length": 89.1,14 "completions/clipped_ratio": 0.0,15 "completions/max_length": 103.6,16 "completions/max_terminated_length": 103.6,17 "completions/mean_length": 89.1,18 "completions/mean_terminated_length": 89.1,19 "completions/min_length": 79.5,20 "completions/min_terminated_length": 79.5,21 "epoch": 0.0008333333333333334,22 "frac_reward_zero_std": 0.3,23 "grad_norm": 0.7635708451271057,24 "kl": 0.0008940239387698057,25 "learning_rate": 1.8e-07,26 "loss": 0.0004,27 "num_tokens": 27380.0,28 "reward": 0.6977083325386048,29 "reward_std": 0.1806380206719041,30 "rewards/combined_reward_with_logging/mean": 0.6977083325386048,31 "rewards/combined_reward_with_logging/std": 0.1806380210444331,32 "step": 1033 },34 {35 "completion_length": 126.05,36 "completions/clipped_ratio": 0.0,37 "completions/max_length": 152.4,38 "completions/max_terminated_length": 152.4,39 "completions/mean_length": 126.05,40 "completions/mean_terminated_length": 126.05,41 "completions/min_length": 90.7,42 "completions/min_terminated_length": 90.7,43 "epoch": 0.0016666666666666668,44 "frac_reward_zero_std": 0.4,45 "grad_norm": 0.00015988279483281076,46 "kl": 0.0004369220678540842,47 "learning_rate": 3.8e-07,48 "loss": -0.0002,49 "num_tokens": 56458.0,50 "reward": 0.7241423174738884,51 "reward_std": 0.1553278874605894,52 "rewards/combined_reward_with_logging/mean": 0.7241423174738884,53 "rewards/combined_reward_with_logging/std": 0.15532788932323455,54 "step": 2055 },56 {57 "completion_length": 90.825,58 "completions/clipped_ratio": 0.0,59 "completions/max_length": 102.1,60 "completions/max_terminated_length": 102.1,61 "completions/mean_length": 90.825,62 "completions/mean_terminated_length": 90.825,63 "completions/min_length": 80.3,64 "completions/min_terminated_length": 80.3,65 "epoch": 0.0025,66 "frac_reward_zero_std": 0.5,67 "grad_norm": 0.7655800580978394,68 "kl": 0.00038943361223573445,69 "learning_rate": 5.800000000000001e-07,70 "loss": 0.0004,71 "num_tokens": 82759.0,72 "reward": 0.5061011910438538,73 "reward_std": 0.1260129153728485,74 "rewards/combined_reward_with_logging/mean": 0.5061011910438538,75 "rewards/combined_reward_with_logging/std": 0.12601291686296462,76 "step": 3077 },78 {79 "completion_length": 98.15,80 "completions/clipped_ratio": 0.0,81 "completions/max_length": 110.5,82 "completions/max_terminated_length": 110.5,83 "completions/mean_length": 98.15,84 "completions/mean_terminated_length": 98.15,85 "completions/min_length": 87.7,86 "completions/min_terminated_length": 87.7,87 "epoch": 0.0033333333333333335,88 "frac_reward_zero_std": 0.5,89 "grad_norm": 0.5018187165260315,90 "kl": 0.0005384670025160964,91 "learning_rate": 7.8e-07,92 "loss": -0.0,93 "num_tokens": 110681.0,94 "reward": 0.7111607134342194,95 "reward_std": 0.13735095355659724,96 "rewards/combined_reward_with_logging/mean": 0.7111607134342194,97 "rewards/combined_reward_with_logging/std": 0.13735095262527466,98 "step": 4099 },100 {101 "completion_length": 86.4,102 "completions/clipped_ratio": 0.0,103 "completions/max_length": 107.0,104 "completions/max_terminated_length": 107.0,105 "completions/mean_length": 86.4,106 "completions/mean_terminated_length": 86.4,107 "completions/min_length": 74.5,108 "completions/min_terminated_length": 74.5,109 "epoch": 0.004166666666666667,110 "frac_reward_zero_std": 0.5,111 "grad_norm": 0.0003931742685381323,112 "kl": 0.001969781374577906,113 "learning_rate": 9.800000000000001e-07,114 "loss": 0.0004,115 "num_tokens": 137445.0,116 "reward": 0.6103124976158142,117 "reward_std": 0.22312394082546233,118 "rewards/combined_reward_with_logging/mean": 0.6103124976158142,119 "rewards/combined_reward_with_logging/std": 0.22312394082546233,120 "step": 50121 },122 {123 "completion_length": 99.575,124 "completions/clipped_ratio": 0.0,125 "completions/max_length": 137.1,126 "completions/max_terminated_length": 137.1,127 "completions/mean_length": 99.575,128 "completions/mean_terminated_length": 99.575,129 "completions/min_length": 76.8,130 "completions/min_terminated_length": 76.8,131 "epoch": 0.005,132 "frac_reward_zero_std": 0.4,133 "grad_norm": 0.5019232630729675,134 "kl": 0.00047573179743736204,135 "learning_rate": 1.1800000000000001e-06,136 "loss": 0.0005,137 "num_tokens": 164216.0,138 "reward": 0.6642187505960464,139 "reward_std": 0.19941234588623047,140 "rewards/combined_reward_with_logging/mean": 0.6642187505960464,141 "rewards/combined_reward_with_logging/std": 0.19941234290599824,142 "step": 60143 },144 {145 "completion_length": 88.8,146 "completions/clipped_ratio": 0.0,147 "completions/max_length": 103.5,148 "completions/max_terminated_length": 103.5,149 "completions/mean_length": 88.8,150 "completions/mean_terminated_length": 88.8,151 "completions/min_length": 74.2,152 "completions/min_terminated_length": 74.2,153 "epoch": 0.005833333333333334,154 "frac_reward_zero_std": 0.4,155 "grad_norm": 8.643526234664023e-05,156 "kl": 0.0004479940557843065,157 "learning_rate": 1.3800000000000001e-06,158 "loss": -0.0002,159 "num_tokens": 190608.0,160 "reward": 0.5629166722297668,161 "reward_std": 0.20985555201768874,162 "rewards/combined_reward_with_logging/mean": 0.5629166722297668,163 "rewards/combined_reward_with_logging/std": 0.20985555201768874,164 "step": 70165 },166 {167 "completion_length": 93.5,168 "completions/clipped_ratio": 0.0,169 "completions/max_length": 108.2,170 "completions/max_terminated_length": 108.2,171 "completions/mean_length": 93.5,172 "completions/mean_terminated_length": 93.5,173 "completions/min_length": 78.7,174 "completions/min_terminated_length": 78.7,175 "epoch": 0.006666666666666667,176 "frac_reward_zero_std": 0.6,177 "grad_norm": 0.45340266823768616,178 "kl": 0.000687123501369058,179 "learning_rate": 1.5800000000000001e-06,180 "loss": -0.0001,181 "num_tokens": 217608.0,182 "reward": 0.7576041638851165,183 "reward_std": 0.08458152413368225,184 "rewards/combined_reward_with_logging/mean": 0.7576041638851165,185 "rewards/combined_reward_with_logging/std": 0.08458152413368225,186 "step": 80187 },188 {189 "completion_length": 102.025,190 "completions/clipped_ratio": 0.0,191 "completions/max_length": 115.1,192 "completions/max_terminated_length": 115.1,193 "completions/mean_length": 102.025,194 "completions/mean_terminated_length": 102.025,195 "completions/min_length": 90.9,196 "completions/min_terminated_length": 90.9,197 "epoch": 0.0075,198 "frac_reward_zero_std": 0.4,199 "grad_norm": 0.5984262228012085,200 "kl": 0.000671394714228768,201 "learning_rate": 1.7800000000000001e-06,202 "loss": 0.0002,203 "num_tokens": 245397.0,204 "reward": 0.567534726858139,205 "reward_std": 0.17743055671453475,206 "rewards/combined_reward_with_logging/mean": 0.567534726858139,207 "rewards/combined_reward_with_logging/std": 0.17743055522441864,208 "step": 90209 },210 {211 "completion_length": 103.575,212 "completions/clipped_ratio": 0.0,213 "completions/max_length": 113.4,214 "completions/max_terminated_length": 113.4,215 "completions/mean_length": 103.575,216 "completions/mean_terminated_length": 103.575,217 "completions/min_length": 94.9,218 "completions/min_terminated_length": 94.9,219 "epoch": 0.008333333333333333,220 "frac_reward_zero_std": 0.5,221 "grad_norm": 0.41970035433769226,222 "kl": 0.0003494232346241688,223 "learning_rate": 1.98e-06,224 "loss": -0.0001,225 "num_tokens": 273260.0,226 "reward": 0.696875,227 "reward_std": 0.1949523825198412,228 "rewards/combined_reward_with_logging/mean": 0.696875,229 "rewards/combined_reward_with_logging/std": 0.19495238363742828,230 "step": 100231 },232 {233 "completion_length": 109.0,234 "completions/clipped_ratio": 0.0,235 "completions/max_length": 132.9,236 "completions/max_terminated_length": 132.9,237 "completions/mean_length": 109.0,238 "completions/mean_terminated_length": 109.0,239 "completions/min_length": 88.9,240 "completions/min_terminated_length": 88.9,241 "epoch": 0.009166666666666667,242 "frac_reward_zero_std": 0.4,243 "grad_norm": 0.32645201683044434,244 "kl": 0.0006206340746208738,245 "learning_rate": 2.1800000000000003e-06,246 "loss": -0.0001,247 "num_tokens": 301700.0,248 "reward": 0.36286458373069763,249 "reward_std": 0.23074487298727037,250 "rewards/combined_reward_with_logging/mean": 0.36286458373069763,251 "rewards/combined_reward_with_logging/std": 0.2307448774576187,252 "step": 110253 },254 {255 "completion_length": 88.8,256 "completions/clipped_ratio": 0.0,257 "completions/max_length": 99.1,258 "completions/max_terminated_length": 99.1,259 "completions/mean_length": 88.8,260 "completions/mean_terminated_length": 88.8,261 "completions/min_length": 78.4,262 "completions/min_terminated_length": 78.4,263 "epoch": 0.01,264 "frac_reward_zero_std": 0.6,265 "grad_norm": 1.7138996554422192e-05,266 "kl": 0.0005296995641236757,267 "learning_rate": 2.38e-06,268 "loss": -0.0002,269 "num_tokens": 328352.0,270 "reward": 0.7472172617912293,271 "reward_std": 0.06379534467123449,272 "rewards/combined_reward_with_logging/mean": 0.7472172617912293,273 "rewards/combined_reward_with_logging/std": 0.06379534527659417,274 "step": 120275 },276 {277 "completion_length": 102.225,278 "completions/clipped_ratio": 0.0,279 "completions/max_length": 118.0,280 "completions/max_terminated_length": 118.0,281 "completions/mean_length": 102.225,282 "completions/mean_terminated_length": 102.225,283 "completions/min_length": 86.4,284 "completions/min_terminated_length": 86.4,285 "epoch": 0.010833333333333334,286 "frac_reward_zero_std": 0.6,287 "grad_norm": 0.00016481519560329616,288 "kl": 0.0010904741588092294,289 "learning_rate": 2.5800000000000003e-06,290 "loss": 0.0002,291 "num_tokens": 355093.0,292 "reward": 0.7279017865657806,293 "reward_std": 0.09139735829085112,294 "rewards/combined_reward_with_logging/mean": 0.7279017865657806,295 "rewards/combined_reward_with_logging/std": 0.09139735829085112,296 "step": 130297 },298 {299 "completion_length": 109.85,300 "completions/clipped_ratio": 0.0,301 "completions/max_length": 126.7,302 "completions/max_terminated_length": 126.7,303 "completions/mean_length": 109.85,304 "completions/mean_terminated_length": 109.85,305 "completions/min_length": 99.0,306 "completions/min_terminated_length": 99.0,307 "epoch": 0.011666666666666667,308 "frac_reward_zero_std": 0.5,309 "grad_norm": 0.0001452508440706879,310 "kl": 0.0006153774211526298,311 "learning_rate": 2.7800000000000005e-06,312 "loss": 0.0003,313 "num_tokens": 383375.0,314 "reward": 0.520052082836628,315 "reward_std": 0.13143815994262695,316 "rewards/combined_reward_with_logging/mean": 0.520052082836628,317 "rewards/combined_reward_with_logging/std": 0.13143815994262695,318 "step": 140319 },320 {321 "completion_length": 100.425,322 "completions/clipped_ratio": 0.0,323 "completions/max_length": 112.5,324 "completions/max_terminated_length": 112.5,325 "completions/mean_length": 100.425,326 "completions/mean_terminated_length": 100.425,327 "completions/min_length": 90.0,328 "completions/min_terminated_length": 90.0,329 "epoch": 0.0125,330 "frac_reward_zero_std": 0.7,331 "grad_norm": 8.576813706895337e-05,332 "kl": 0.0005368013010325967,333 "learning_rate": 2.9800000000000003e-06,334 "loss": -0.0001,335 "num_tokens": 410452.0,336 "reward": 0.42410714328289034,337 "reward_std": 0.07321428656578063,338 "rewards/combined_reward_with_logging/mean": 0.42410714328289034,339 "rewards/combined_reward_with_logging/std": 0.07321428656578063,340 "step": 150341 },342 {343 "completion_length": 98.65,344 "completions/clipped_ratio": 0.0,345 "completions/max_length": 124.3,346 "completions/max_terminated_length": 124.3,347 "completions/mean_length": 98.65,348 "completions/mean_terminated_length": 98.65,349 "completions/min_length": 76.4,350 "completions/min_terminated_length": 76.4,351 "epoch": 0.013333333333333334,352 "frac_reward_zero_std": 0.5,353 "grad_norm": 0.7682750821113586,354 "kl": 0.0005419679728895233,355 "learning_rate": 3.1800000000000005e-06,356 "loss": 0.0004,357 "num_tokens": 436602.0,358 "reward": 0.6430208325386048,359 "reward_std": 0.1277103504166007,360 "rewards/combined_reward_with_logging/mean": 0.6430208325386048,361 "rewards/combined_reward_with_logging/std": 0.12771035209298134,362 "step": 160363 },364 {365 "completion_length": 82.675,366 "completions/clipped_ratio": 0.0,367 "completions/max_length": 89.5,368 "completions/max_terminated_length": 89.5,369 "completions/mean_length": 82.675,370 "completions/mean_terminated_length": 82.675,371 "completions/min_length": 74.1,372 "completions/min_terminated_length": 74.1,373 "epoch": 0.014166666666666666,374 "frac_reward_zero_std": 0.4,375 "grad_norm": 1.0834494829177856,376 "kl": 0.0005685133873839732,377 "learning_rate": 3.3800000000000007e-06,378 "loss": 0.0004,379 "num_tokens": 462977.0,380 "reward": 0.7405059516429902,381 "reward_std": 0.15567945837974548,382 "rewards/combined_reward_with_logging/mean": 0.7405059516429902,383 "rewards/combined_reward_with_logging/std": 0.15567946135997773,384 "step": 170385 },386 {387 "completion_length": 96.475,388 "completions/clipped_ratio": 0.0,389 "completions/max_length": 105.9,390 "completions/max_terminated_length": 105.9,391 "completions/mean_length": 96.475,392 "completions/mean_terminated_length": 96.475,393 "completions/min_length": 89.1,394 "completions/min_terminated_length": 89.1,395 "epoch": 0.015,396 "frac_reward_zero_std": 0.5,397 "grad_norm": 0.00017970296903513372,398 "kl": 0.0004920295719648582,399 "learning_rate": 3.58e-06,400 "loss": -0.0003,401 "num_tokens": 488736.0,402 "reward": 0.5341666683554649,403 "reward_std": 0.2052350252866745,404 "rewards/combined_reward_with_logging/mean": 0.5341666683554649,405 "rewards/combined_reward_with_logging/std": 0.2052350252866745,406 "step": 180407 },408 {409 "completion_length": 97.9,410 "completions/clipped_ratio": 0.0,411 "completions/max_length": 102.1,412 "completions/max_terminated_length": 102.1,413 "completions/mean_length": 97.9,414 "completions/mean_terminated_length": 97.9,415 "completions/min_length": 92.8,416 "completions/min_terminated_length": 92.8,417 "epoch": 0.015833333333333335,418 "frac_reward_zero_std": 0.5,419 "grad_norm": 0.00019758465350605547,420 "kl": 0.0013767435375257263,421 "learning_rate": 3.7800000000000002e-06,422 "loss": 0.0003,423 "num_tokens": 516176.0,424 "reward": 0.6020833313465118,425 "reward_std": 0.13383883461356164,426 "rewards/combined_reward_with_logging/mean": 0.6020833313465118,427 "rewards/combined_reward_with_logging/std": 0.13383883461356164,428 "step": 190429 },430 {431 "completion_length": 103.425,432 "completions/clipped_ratio": 0.0,433 "completions/max_length": 118.3,434 "completions/max_terminated_length": 118.3,435 "completions/mean_length": 103.425,436 "completions/mean_terminated_length": 103.425,437 "completions/min_length": 84.3,438 "completions/min_terminated_length": 84.3,439 "epoch": 0.016666666666666666,440 "frac_reward_zero_std": 0.1,441 "grad_norm": 1.269307017326355,442 "kl": 0.0010686940103092012,443 "learning_rate": 3.980000000000001e-06,444 "loss": 0.0017,445 "num_tokens": 542733.0,446 "reward": 0.4278422713279724,447 "reward_std": 0.3312844455242157,448 "rewards/combined_reward_with_logging/mean": 0.4278422713279724,449 "rewards/combined_reward_with_logging/std": 0.33128444850444794,450 "step": 200451 },452 {453 "completion_length": 83.675,454 "completions/clipped_ratio": 0.0,455 "completions/max_length": 97.5,456 "completions/max_terminated_length": 97.5,457 "completions/mean_length": 83.675,458 "completions/mean_terminated_length": 83.675,459 "completions/min_length": 74.4,460 "completions/min_terminated_length": 74.4,461 "epoch": 0.0175,462 "frac_reward_zero_std": 0.4,463 "grad_norm": 0.4841049313545227,464 "kl": 0.0013043641187778122,465 "learning_rate": 4.18e-06,466 "loss": 0.0013,467 "num_tokens": 569176.0,468 "reward": 0.6419196426868439,469 "reward_std": 0.217718642950058,470 "rewards/combined_reward_with_logging/mean": 0.6419196426868439,471 "rewards/combined_reward_with_logging/std": 0.217718642950058,472 "step": 210473 },474 {475 "completion_length": 99.5,476 "completions/clipped_ratio": 0.0,477 "completions/max_length": 105.6,478 "completions/max_terminated_length": 105.6,479 "completions/mean_length": 99.5,480 "completions/mean_terminated_length": 99.5,481 "completions/min_length": 90.8,482 "completions/min_terminated_length": 90.8,483 "epoch": 0.018333333333333333,484 "frac_reward_zero_std": 0.7,485 "grad_norm": 0.449802428483963,486 "kl": 0.0008728265635927102,487 "learning_rate": 4.38e-06,488 "loss": 0.0001,489 "num_tokens": 596464.0,490 "reward": 0.5780952379107476,491 "reward_std": 0.10807691514492035,492 "rewards/combined_reward_with_logging/mean": 0.5780952379107476,493 "rewards/combined_reward_with_logging/std": 0.10807691663503646,494 "step": 220495 },496 {497 "completion_length": 85.675,498 "completions/clipped_ratio": 0.0,499 "completions/max_length": 94.7,500 "completions/max_terminated_length": 94.7,501 "completions/mean_length": 85.675,502 "completions/mean_terminated_length": 85.675,503 "completions/min_length": 74.2,504 "completions/min_terminated_length": 74.2,505 "epoch": 0.019166666666666665,506 "frac_reward_zero_std": 0.5,507 "grad_norm": 0.0007017672760412097,508 "kl": 0.0015955587439975716,509 "learning_rate": 4.58e-06,510 "loss": 0.0004,511 "num_tokens": 622271.0,512 "reward": 0.6588541686534881,513 "reward_std": 0.19449251294136047,514 "rewards/combined_reward_with_logging/mean": 0.6588541686534881,515 "rewards/combined_reward_with_logging/std": 0.19449251294136047,516 "step": 230517 },518 {519 "completion_length": 103.175,520 "completions/clipped_ratio": 0.0,521 "completions/max_length": 117.4,522 "completions/max_terminated_length": 117.4,523 "completions/mean_length": 103.175,524 "completions/mean_terminated_length": 103.175,525 "completions/min_length": 87.8,526 "completions/min_terminated_length": 87.8,527 "epoch": 0.02,528 "frac_reward_zero_std": 0.6,529 "grad_norm": 0.00019546717521734536,530 "kl": 0.0008891218908161136,531 "learning_rate": 4.78e-06,532 "loss": -0.0,533 "num_tokens": 649118.0,534 "reward": 0.46623511910438536,535 "reward_std": 0.11170007735490799,536 "rewards/combined_reward_with_logging/mean": 0.46623511910438536,537 "rewards/combined_reward_with_logging/std": 0.11170007735490799,538 "step": 240539 },540 {541 "completion_length": 107.55,542 "completions/clipped_ratio": 0.0,543 "completions/max_length": 138.4,544 "completions/max_terminated_length": 138.4,545 "completions/mean_length": 107.55,546 "completions/mean_terminated_length": 107.55,547 "completions/min_length": 84.5,548 "completions/min_terminated_length": 84.5,549 "epoch": 0.020833333333333332,550 "frac_reward_zero_std": 0.5,551 "grad_norm": 0.00011019164230674505,552 "kl": 0.00142500318014811,553 "learning_rate": 4.980000000000001e-06,554 "loss": 0.0002,555 "num_tokens": 676028.0,556 "reward": 0.578601187467575,557 "reward_std": 0.173603343218565,558 "rewards/combined_reward_with_logging/mean": 0.578601187467575,559 "rewards/combined_reward_with_logging/std": 0.17360334917902948,560 "step": 250561 },562 {563 "completion_length": 100.075,564 "completions/clipped_ratio": 0.0,565 "completions/max_length": 108.4,566 "completions/max_terminated_length": 108.4,567 "completions/mean_length": 100.075,568 "completions/mean_terminated_length": 100.075,569 "completions/min_length": 90.3,570 "completions/min_terminated_length": 90.3,571 "epoch": 0.021666666666666667,572 "frac_reward_zero_std": 0.4,573 "grad_norm": 0.5309756398200989,574 "kl": 0.0020459796819523035,575 "learning_rate": 5.18e-06,576 "loss": 0.0006,577 "num_tokens": 703719.0,578 "reward": 0.6182142794132233,579 "reward_std": 0.11029952131211758,580 "rewards/combined_reward_with_logging/mean": 0.6182142794132233,581 "rewards/combined_reward_with_logging/std": 0.11029952615499497,582 "step": 260583 },584 {585 "completion_length": 115.35,586 "completions/clipped_ratio": 0.0,587 "completions/max_length": 135.1,588 "completions/max_terminated_length": 135.1,589 "completions/mean_length": 115.35,590 "completions/mean_terminated_length": 115.35,591 "completions/min_length": 98.2,592 "completions/min_terminated_length": 98.2,593 "epoch": 0.0225,594 "frac_reward_zero_std": 0.3,595 "grad_norm": 6.668347486993298e-05,596 "kl": 0.0019772483233055026,597 "learning_rate": 5.380000000000001e-06,598 "loss": 0.0004,599 "num_tokens": 732365.0,600 "reward": 0.6336458310484886,601 "reward_std": 0.25892585664987566,602 "rewards/combined_reward_with_logging/mean": 0.6336458310484886,603 "rewards/combined_reward_with_logging/std": 0.2589258670806885,604 "step": 270605 },606 {607 "completion_length": 77.4,608 "completions/clipped_ratio": 0.0,609 "completions/max_length": 81.0,610 "completions/max_terminated_length": 81.0,611 "completions/mean_length": 77.4,612 "completions/mean_terminated_length": 77.4,613 "completions/min_length": 71.9,614 "completions/min_terminated_length": 71.9,615 "epoch": 0.023333333333333334,616 "frac_reward_zero_std": 0.9,617 "grad_norm": 4.819453636173421e-08,618 "kl": 0.0014972172638429981,619 "learning_rate": 5.580000000000001e-06,620 "loss": -0.0001,621 "num_tokens": 757229.0,622 "reward": 0.7631250023841858,623 "reward_std": 0.013559283316135406,624 "rewards/combined_reward_with_logging/mean": 0.7631250023841858,625 "rewards/combined_reward_with_logging/std": 0.013559283316135406,626 "step": 280627 },628 {629 "completion_length": 90.825,630 "completions/clipped_ratio": 0.0,631 "completions/max_length": 104.2,632 "completions/max_terminated_length": 104.2,633 "completions/mean_length": 90.825,634 "completions/mean_terminated_length": 90.825,635 "completions/min_length": 79.7,636 "completions/min_terminated_length": 79.7,637 "epoch": 0.024166666666666666,638 "frac_reward_zero_std": 0.3,639 "grad_norm": 1.083999752998352,640 "kl": 0.002592998363257948,641 "learning_rate": 5.78e-06,642 "loss": 0.0008,643 "num_tokens": 783882.0,644 "reward": 0.6988244116306305,645 "reward_std": 0.17091417890042065,646 "rewards/combined_reward_with_logging/mean": 0.6988244116306305,647 "rewards/combined_reward_with_logging/std": 0.1709141790866852,648 "step": 290649 },650 {651 "completion_length": 94.7,652 "completions/clipped_ratio": 0.0,653 "completions/max_length": 108.7,654 "completions/max_terminated_length": 108.7,655 "completions/mean_length": 94.7,656 "completions/mean_terminated_length": 94.7,657 "completions/min_length": 79.6,658 "completions/min_terminated_length": 79.6,659 "epoch": 0.025,660 "frac_reward_zero_std": 0.4,661 "grad_norm": 0.0003086680080741644,662 "kl": 0.003027042169651395,663 "learning_rate": 5.98e-06,664 "loss": 0.0001,665 "num_tokens": 811178.0,666 "reward": 0.5920535668730735,667 "reward_std": 0.13358704531565307,668 "rewards/combined_reward_with_logging/mean": 0.5920535668730735,669 "rewards/combined_reward_with_logging/std": 0.13358704536221921,670 "step": 300671 },672 {673 "completion_length": 98.8,674 "completions/clipped_ratio": 0.0,675 "completions/max_length": 109.8,676 "completions/max_terminated_length": 109.8,677 "completions/mean_length": 98.8,678 "completions/mean_terminated_length": 98.8,679 "completions/min_length": 88.1,680 "completions/min_terminated_length": 88.1,681 "epoch": 0.025833333333333333,682 "frac_reward_zero_std": 0.7,683 "grad_norm": 0.7857518196105957,684 "kl": 0.0029371274195000296,685 "learning_rate": 6.18e-06,686 "loss": 0.0004,687 "num_tokens": 839086.0,688 "reward": 0.4336458325386047,689 "reward_std": 0.08509890921413898,690 "rewards/combined_reward_with_logging/mean": 0.4336458325386047,691 "rewards/combined_reward_with_logging/std": 0.08509890735149384,692 "step": 310693 },694 {695 "completion_length": 84.4,696 "completions/clipped_ratio": 0.0,697 "completions/max_length": 92.4,698 "completions/max_terminated_length": 92.4,699 "completions/mean_length": 84.4,700 "completions/mean_terminated_length": 84.4,701 "completions/min_length": 80.2,702 "completions/min_terminated_length": 80.2,703 "epoch": 0.02666666666666667,704 "frac_reward_zero_std": 0.7,705 "grad_norm": 0.000772831030189991,706 "kl": 0.003626009412619169,707 "learning_rate": 6.380000000000001e-06,708 "loss": 0.001,709 "num_tokens": 866494.0,710 "reward": 0.5607142865657806,711 "reward_std": 0.08392857164144515,712 "rewards/combined_reward_with_logging/mean": 0.5607142865657806,713 "rewards/combined_reward_with_logging/std": 0.08392857015132904,714 "step": 320715 },716 {717 "completion_length": 84.7,718 "completions/clipped_ratio": 0.0,719 "completions/max_length": 100.2,720 "completions/max_terminated_length": 100.2,721 "completions/mean_length": 84.7,722 "completions/mean_terminated_length": 84.7,723 "completions/min_length": 73.2,724 "completions/min_terminated_length": 73.2,725 "epoch": 0.0275,726 "frac_reward_zero_std": 0.2,727 "grad_norm": 0.5985280871391296,728 "kl": 0.007182306539289129,729 "learning_rate": 6.5800000000000005e-06,730 "loss": 0.0,731 "num_tokens": 893122.0,732 "reward": 0.6427083373069763,733 "reward_std": 0.23081925846636295,734 "rewards/combined_reward_with_logging/mean": 0.6427083373069763,735 "rewards/combined_reward_with_logging/std": 0.2308192580938339,736 "step": 330737 },738 {739 "completion_length": 109.4,740 "completions/clipped_ratio": 0.0,741 "completions/max_length": 119.8,742 "completions/max_terminated_length": 119.8,743 "completions/mean_length": 109.4,744 "completions/mean_terminated_length": 109.4,745 "completions/min_length": 101.7,746 "completions/min_terminated_length": 101.7,747 "epoch": 0.028333333333333332,748 "frac_reward_zero_std": 0.4,749 "grad_norm": 0.3083597421646118,750 "kl": 0.0052545823886703145,751 "learning_rate": 6.780000000000001e-06,752 "loss": 0.0005,753 "num_tokens": 920722.0,754 "reward": 0.720863088965416,755 "reward_std": 0.13782292436808347,756 "rewards/combined_reward_with_logging/mean": 0.720863088965416,757 "rewards/combined_reward_with_logging/std": 0.13782293498516082,758 "step": 340759 },760 {761 "completion_length": 303.375,762 "completions/clipped_ratio": 0.05,763 "completions/max_length": 522.6,764 "completions/max_terminated_length": 122.7,765 "completions/mean_length": 303.375,766 "completions/mean_terminated_length": 103.325,767 "completions/min_length": 89.0,768 "completions/min_terminated_length": 89.0,769 "epoch": 0.029166666666666667,770 "frac_reward_zero_std": 0.5,771 "grad_norm": 0.0007893677102401853,772 "kl": 0.008326682686720233,773 "learning_rate": 6.98e-06,774 "loss": 0.0007,775 "num_tokens": 956441.0,776 "reward": 0.5645312428474426,777 "reward_std": 0.10732038840651512,778 "rewards/combined_reward_with_logging/mean": 0.5645312428474426,779 "rewards/combined_reward_with_logging/std": 0.10732039138674736,780 "step": 350781 },782 {783 "completion_length": 107.1,784 "completions/clipped_ratio": 0.0,785 "completions/max_length": 118.9,786 "completions/max_terminated_length": 118.9,787 "completions/mean_length": 107.1,788 "completions/mean_terminated_length": 107.1,789 "completions/min_length": 96.3,790 "completions/min_terminated_length": 96.3,791 "epoch": 0.03,792 "frac_reward_zero_std": 0.3,793 "grad_norm": 0.578423261642456,794 "kl": 0.0070406923041673505,795 "learning_rate": 7.180000000000001e-06,796 "loss": 0.0008,797 "num_tokens": 983729.0,798 "reward": 0.5854166626930237,799 "reward_std": 0.20108191017061472,800 "rewards/combined_reward_with_logging/mean": 0.5854166626930237,801 "rewards/combined_reward_with_logging/std": 0.20108190923929214,802 "step": 360803 },804 {805 "completion_length": 109.95,806 "completions/clipped_ratio": 0.0,807 "completions/max_length": 126.5,808 "completions/max_terminated_length": 126.5,809 "completions/mean_length": 109.95,810 "completions/mean_terminated_length": 109.95,811 "completions/min_length": 93.7,812 "completions/min_terminated_length": 93.7,813 "epoch": 0.030833333333333334,814 "frac_reward_zero_std": 0.4,815 "grad_norm": 0.00046158453915268183,816 "kl": 0.010318851130026374,817 "learning_rate": 7.3800000000000005e-06,818 "loss": -0.0003,819 "num_tokens": 1010483.0,820 "reward": 0.6307812497019768,821 "reward_std": 0.22450854554772376,822 "rewards/combined_reward_with_logging/mean": 0.6307812497019768,823 "rewards/combined_reward_with_logging/std": 0.22450854554772376,824 "step": 370825 },826 {827 "completion_length": 115.575,828 "completions/clipped_ratio": 0.0,829 "completions/max_length": 128.1,830 "completions/max_terminated_length": 128.1,831 "completions/mean_length": 115.575,832 "completions/mean_terminated_length": 115.575,833 "completions/min_length": 102.4,834 "completions/min_terminated_length": 102.4,835 "epoch": 0.03166666666666667,836 "frac_reward_zero_std": 0.6,837 "grad_norm": 0.8582760691642761,838 "kl": 0.014512627789008548,839 "learning_rate": 7.58e-06,840 "loss": 0.0,841 "num_tokens": 1037198.0,842 "reward": 0.5278348252177238,843 "reward_std": 0.13807350248098374,844 "rewards/combined_reward_with_logging/mean": 0.5278348252177238,845 "rewards/combined_reward_with_logging/std": 0.13807350248098374,846 "step": 380847 },848 {849 "completion_length": 104.175,850 "completions/clipped_ratio": 0.0,851 "completions/max_length": 129.5,852 "completions/max_terminated_length": 129.5,853 "completions/mean_length": 104.175,854 "completions/mean_terminated_length": 104.175,855 "completions/min_length": 89.9,856 "completions/min_terminated_length": 89.9,857 "epoch": 0.0325,858 "frac_reward_zero_std": 0.7,859 "grad_norm": 0.4426611065864563,860 "kl": 0.013596086359757464,861 "learning_rate": 7.78e-06,862 "loss": 0.0002,863 "num_tokens": 1063385.0,864 "reward": 0.521875,865 "reward_std": 0.10077722072601318,866 "rewards/combined_reward_with_logging/mean": 0.521875,867 "rewards/combined_reward_with_logging/std": 0.10077722370624542,868 "step": 390869 },870 {871 "completion_length": 112.5,872 "completions/clipped_ratio": 0.0,873 "completions/max_length": 131.4,874 "completions/max_terminated_length": 131.4,875 "completions/mean_length": 112.5,876 "completions/mean_terminated_length": 112.5,877 "completions/min_length": 94.8,878 "completions/min_terminated_length": 94.8,879 "epoch": 0.03333333333333333,880 "frac_reward_zero_std": 0.7,881 "grad_norm": 0.0004901818465441465,882 "kl": 0.005264811985034612,883 "learning_rate": 7.980000000000002e-06,884 "loss": 0.0001,885 "num_tokens": 1091357.0,886 "reward": 0.5884548604488373,887 "reward_std": 0.133970183134079,888 "rewards/combined_reward_with_logging/mean": 0.5884548604488373,889 "rewards/combined_reward_with_logging/std": 0.133970183134079,890 "step": 400891 },892 {893 "completion_length": 105.275,894 "completions/clipped_ratio": 0.0,895 "completions/max_length": 132.6,896 "completions/max_terminated_length": 132.6,897 "completions/mean_length": 105.275,898 "completions/mean_terminated_length": 105.275,899 "completions/min_length": 78.0,900 "completions/min_terminated_length": 78.0,901 "epoch": 0.034166666666666665,902 "frac_reward_zero_std": 0.6,903 "grad_norm": 0.00017372587171848863,904 "kl": 0.011939602546044625,905 "learning_rate": 8.18e-06,906 "loss": 0.0002,907 "num_tokens": 1118012.0,908 "reward": 0.3421875,909 "reward_std": 0.14841381907463075,910 "rewards/combined_reward_with_logging/mean": 0.3421875,911 "rewards/combined_reward_with_logging/std": 0.14841381907463075,912 "step": 410913 },914 {915 "completion_length": 99.125,916 "completions/clipped_ratio": 0.0,917 "completions/max_length": 121.1,918 "completions/max_terminated_length": 121.1,919 "completions/mean_length": 99.125,920 "completions/mean_terminated_length": 99.125,921 "completions/min_length": 86.5,922 "completions/min_terminated_length": 86.5,923 "epoch": 0.035,924 "frac_reward_zero_std": 0.5,925 "grad_norm": 0.2943379282951355,926 "kl": 0.014834571303435951,927 "learning_rate": 8.380000000000001e-06,928 "loss": 0.0002,929 "num_tokens": 1143909.0,930 "reward": 0.5171875,931 "reward_std": 0.13657718598842622,932 "rewards/combined_reward_with_logging/mean": 0.5171875,933 "rewards/combined_reward_with_logging/std": 0.13657718598842622,934 "step": 420935 },936 {937 "completion_length": 100.125,938 "completions/clipped_ratio": 0.0,939 "completions/max_length": 106.9,940 "completions/max_terminated_length": 106.9,941 "completions/mean_length": 100.125,942 "completions/mean_terminated_length": 100.125,943 "completions/min_length": 93.6,944 "completions/min_terminated_length": 93.6,945 "epoch": 0.035833333333333335,946 "frac_reward_zero_std": 0.6,947 "grad_norm": 0.00022998801432549953,948 "kl": 0.017320550897966313,949 "learning_rate": 8.580000000000001e-06,950 "loss": 0.0001,951 "num_tokens": 1171230.0,952 "reward": 0.4604166686534882,953 "reward_std": 0.1354166646488011,954 "rewards/combined_reward_with_logging/mean": 0.4604166686534882,955 "rewards/combined_reward_with_logging/std": 0.13541666567325591,956 "step": 430957 },958 {959 "completion_length": 99.55,960 "completions/clipped_ratio": 0.0,961 "completions/max_length": 115.6,962 "completions/max_terminated_length": 115.6,963 "completions/mean_length": 99.55,964 "completions/mean_terminated_length": 99.55,965 "completions/min_length": 87.6,966 "completions/min_terminated_length": 87.6,967 "epoch": 0.03666666666666667,968 "frac_reward_zero_std": 0.3,969 "grad_norm": 0.8323773145675659,970 "kl": 0.04196918269153684,971 "learning_rate": 8.78e-06,972 "loss": 0.0011,973 "num_tokens": 1199144.0,974 "reward": 0.7206845223903656,975 "reward_std": 0.16198024153709412,976 "rewards/combined_reward_with_logging/mean": 0.7206845223903656,977 "rewards/combined_reward_with_logging/std": 0.16198024302721023,978 "step": 440979 },980 {981 "completion_length": 113.3,982 "completions/clipped_ratio": 0.0,983 "completions/max_length": 131.9,984 "completions/max_terminated_length": 131.9,985 "completions/mean_length": 113.3,986 "completions/mean_terminated_length": 113.3,987 "completions/min_length": 99.0,988 "completions/min_terminated_length": 99.0,989 "epoch": 0.0375,990 "frac_reward_zero_std": 0.6,991 "grad_norm": 0.0013609821908175945,992 "kl": 0.034836143156860545,993 "learning_rate": 8.98e-06,994 "loss": 0.0001,995 "num_tokens": 1227792.0,996 "reward": 0.5891666680574417,997 "reward_std": 0.1449999988079071,998 "rewards/combined_reward_with_logging/mean": 0.5891666680574417,999 "rewards/combined_reward_with_logging/std": 0.1449999988079071,1000 "step": 4501001 },1002 {1003 "completion_length": 99.55,1004 "completions/clipped_ratio": 0.0,1005 "completions/max_length": 116.1,1006 "completions/max_terminated_length": 116.1,1007 "completions/mean_length": 99.55,1008 "completions/mean_terminated_length": 99.55,1009 "completions/min_length": 89.1,1010 "completions/min_terminated_length": 89.1,1011 "epoch": 0.03833333333333333,1012 "frac_reward_zero_std": 0.4,1013 "grad_norm": 8.405782523368543e-07,1014 "kl": 0.024362141279186744,1015 "learning_rate": 9.180000000000002e-06,1016 "loss": 0.0001,1017 "num_tokens": 1255982.0,1018 "reward": 0.6626488089561462,1019 "reward_std": 0.10946768671274185,1020 "rewards/combined_reward_with_logging/mean": 0.6626488089561462,1021 "rewards/combined_reward_with_logging/std": 0.10946768894791603,1022 "step": 4601023 },1024 {1025 "completion_length": 89.775,1026 "completions/clipped_ratio": 0.0,1027 "completions/max_length": 99.6,1028 "completions/max_terminated_length": 99.6,1029 "completions/mean_length": 89.775,1030 "completions/mean_terminated_length": 89.775,1031 "completions/min_length": 80.9,1032 "completions/min_terminated_length": 80.9,1033 "epoch": 0.03916666666666667,1034 "frac_reward_zero_std": 0.5,1035 "grad_norm": 0.0007129553705453873,1036 "kl": 0.06300802943878807,1037 "learning_rate": 9.38e-06,1038 "loss": 0.0003,1039 "num_tokens": 1283721.0,1040 "reward": 0.4604910731315613,1041 "reward_std": 0.2531973868608475,1042 "rewards/combined_reward_with_logging/mean": 0.4604910731315613,1043 "rewards/combined_reward_with_logging/std": 0.25319739282131193,1044 "step": 4701045 },1046 {1047 "completion_length": 89.575,1048 "completions/clipped_ratio": 0.0,1049 "completions/max_length": 99.5,1050 "completions/max_terminated_length": 99.5,1051 "completions/mean_length": 89.575,1052 "completions/mean_terminated_length": 89.575,1053 "completions/min_length": 80.1,1054 "completions/min_terminated_length": 80.1,1055 "epoch": 0.04,1056 "frac_reward_zero_std": 0.7,1057 "grad_norm": 4.1089137084782124e-05,1058 "kl": 0.04395798201439902,1059 "learning_rate": 9.58e-06,1060 "loss": 0.0002,1061 "num_tokens": 1310724.0,1062 "reward": 0.8129613101482391,1063 "reward_std": 0.1172309011220932,1064 "rewards/combined_reward_with_logging/mean": 0.8129613101482391,1065 "rewards/combined_reward_with_logging/std": 0.11723090857267379,1066 "step": 4801067 },1068 {1069 "completion_length": 101.05,1070 "completions/clipped_ratio": 0.0,1071 "completions/max_length": 113.8,1072 "completions/max_terminated_length": 113.8,1073 "completions/mean_length": 101.05,1074 "completions/mean_terminated_length": 101.05,1075 "completions/min_length": 84.2,1076 "completions/min_terminated_length": 84.2,1077 "epoch": 0.04083333333333333,1078 "frac_reward_zero_std": 0.7,1079 "grad_norm": 5.114754458190873e-05,1080 "kl": 0.061332970845978704,1081 "learning_rate": 9.780000000000001e-06,1082 "loss": -0.0,1083 "num_tokens": 1337110.0,1084 "reward": 0.7387152791023255,1085 "reward_std": 0.08299357444047928,1086 "rewards/combined_reward_with_logging/mean": 0.7387152791023255,1087 "rewards/combined_reward_with_logging/std": 0.08299357593059539,1088 "step": 4901089 },1090 {1091 "completion_length": 91.325,1092 "completions/clipped_ratio": 0.0,1093 "completions/max_length": 101.3,1094 "completions/max_terminated_length": 101.3,1095 "completions/mean_length": 91.325,1096 "completions/mean_terminated_length": 91.325,1097 "completions/min_length": 83.9,1098 "completions/min_terminated_length": 83.9,1099 "epoch": 0.041666666666666664,1100 "frac_reward_zero_std": 0.5,1101 "grad_norm": 2.608182430267334,1102 "kl": 0.07197198094800114,1103 "learning_rate": 9.980000000000001e-06,1104 "loss": 0.0007,1105 "num_tokens": 1363511.0,1106 "reward": 0.49447916746139525,1107 "reward_std": 0.1318429384380579,1108 "rewards/combined_reward_with_logging/mean": 0.49447916746139525,1109 "rewards/combined_reward_with_logging/std": 0.13184293806552888,1110 "step": 5001111 },1112 {1113 "completion_length": 111.875,1114 "completions/clipped_ratio": 0.0,1115 "completions/max_length": 120.4,1116 "completions/max_terminated_length": 120.4,1117 "completions/mean_length": 111.875,1118 "completions/mean_terminated_length": 111.875,1119 "completions/min_length": 104.3,1120 "completions/min_terminated_length": 104.3,1121 "epoch": 0.0425,1122 "frac_reward_zero_std": 0.7,1123 "grad_norm": 0.0009244419634342194,1124 "kl": 0.04722539982176386,1125 "learning_rate": 9.999901304280686e-06,1126 "loss": 0.0,1127 "num_tokens": 1392034.0,1128 "reward": 0.765625,1129 "reward_std": 0.1366025388240814,1130 "rewards/combined_reward_with_logging/mean": 0.765625,1131 "rewards/combined_reward_with_logging/std": 0.1366025388240814,1132 "step": 5101133 },1134 {1135 "completion_length": 91.25,1136 "completions/clipped_ratio": 0.0,1137 "completions/max_length": 97.7,1138 "completions/max_terminated_length": 97.7,1139 "completions/mean_length": 91.25,1140 "completions/mean_terminated_length": 91.25,1141 "completions/min_length": 84.7,1142 "completions/min_terminated_length": 84.7,1143 "epoch": 0.043333333333333335,1144 "frac_reward_zero_std": 0.6,1145 "grad_norm": 0.0018858942203223705,1146 "kl": 0.048604192933999,1147 "learning_rate": 9.999560138895238e-06,1148 "loss": 0.0006,1149 "num_tokens": 1417536.0,1150 "reward": 0.6291666686534881,1151 "reward_std": 0.15386751294136047,1152 "rewards/combined_reward_with_logging/mean": 0.6291666686534881,1153 "rewards/combined_reward_with_logging/std": 0.15386751294136047,1154 "step": 5201155 },1156 {1157 "completion_length": 105.875,1158 "completions/clipped_ratio": 0.0,1159 "completions/max_length": 119.7,1160 "completions/max_terminated_length": 119.7,1161 "completions/mean_length": 105.875,1162 "completions/mean_terminated_length": 105.875,1163 "completions/min_length": 90.3,1164 "completions/min_terminated_length": 90.3,1165 "epoch": 0.04416666666666667,1166 "frac_reward_zero_std": 0.4,1167 "grad_norm": 1.4831848602625541e-05,1168 "kl": 0.027789575696806422,1169 "learning_rate": 9.99897530200195e-06,1170 "loss": 0.0005,1171 "num_tokens": 1445459.0,1172 "reward": 0.6507837355136872,1173 "reward_std": 0.15184815023094417,1174 "rewards/combined_reward_with_logging/mean": 0.6507837355136872,1175 "rewards/combined_reward_with_logging/std": 0.1518481496721506,1176 "step": 5301177 },1178 {1179 "completion_length": 103.05,1180 "completions/clipped_ratio": 0.0,1181 "completions/max_length": 117.9,1182 "completions/max_terminated_length": 117.9,1183 "completions/mean_length": 103.05,1184 "completions/mean_terminated_length": 103.05,1185 "completions/min_length": 89.8,1186 "completions/min_terminated_length": 89.8,1187 "epoch": 0.045,1188 "frac_reward_zero_std": 0.6,1189 "grad_norm": 0.4804590344429016,1190 "kl": 0.04985751159256324,1191 "learning_rate": 9.998146822104943e-06,1192 "loss": 0.0001,1193 "num_tokens": 1472141.0,1194 "reward": 0.532604168355465,1195 "reward_std": 0.10224849600344896,1196 "rewards/combined_reward_with_logging/mean": 0.532604168355465,1197 "rewards/combined_reward_with_logging/std": 0.10224850047379733,1198 "step": 5401199 },1200 {