diff --git "a/trainer_state.json" "b/trainer_state.json" --- "a/trainer_state.json" +++ "b/trainer_state.json" @@ -2,9 +2,9 @@ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, - "epoch": 0.22857142857142856, + "epoch": 0.5714285714285714, "eval_steps": 500, - "global_step": 200, + "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, @@ -4610,17 +4610,6917 @@ "step": 200 }, { - "epoch": 0.22857142857142856, - "step": 200, + "clip_fraction": 0.0, + "completion_length": 2760.7084197998047, + "epoch": 0.2297142857142857, + "grad_norm": 0.06477575749158859, + "kl": 1.7711427062749863e-05, + "lambda_div_used": 0.6429438292980194, + "learning_rate": 1e-07, + "loss": -0.0022, + "reward": 0.15019571036100388, + "reward_after_mean": 0.15019571036100388, + "reward_after_std": 0.7328120246529579, + "reward_before_mean": 0.5535202682949603, + "reward_before_std": 0.6601617820560932, + "reward_change_max": 0.0, + "reward_change_mean": -0.4033245462924242, + "reward_change_min": -0.6303750872612, + "reward_change_std": 0.24585676938295364, + "reward_std": 0.7328120358288288, + "rewards/accuracy_reward": 0.41666667349636555, + "rewards/cosine_scaled_reward": 0.13685358315706253, + "step": 201 + }, + { + "clip_fraction": 0.0, + "completion_length": 2511.937511444092, + "epoch": 0.23085714285714284, + "grad_norm": 0.08345558494329453, + "kl": 4.0203332901000977e-05, + "lambda_div_used": 0.5811712816357613, + "learning_rate": 7.72273839962904e-07, + "loss": 0.0587, + "reward": 0.03902309015393257, + "reward_after_mean": 0.03902309015393257, + "reward_after_std": 0.5094954669475555, + "reward_before_mean": 0.5408022310584784, + "reward_before_std": 0.37269798293709755, + "reward_change_max": 0.0, + "reward_change_mean": -0.5017791502177715, + "reward_change_min": -0.7236880213022232, + "reward_change_std": 0.28125342447310686, + "reward_std": 0.5094954781234264, + "rewards/accuracy_reward": 0.3958333395421505, + "rewards/cosine_scaled_reward": 0.14496888127177954, + "step": 202 + }, + { + "clip_fraction": 0.0, + "completion_length": 3065.3333435058594, + "epoch": 0.232, + "grad_norm": 0.05931547284126282, + "kl": 5.182623863220215e-05, + "lambda_div_used": 0.5537669658660889, + "learning_rate": 7.695368466124296e-07, + "loss": 0.0447, + "reward": -0.3224771413952112, + "reward_after_mean": -0.3224771413952112, + "reward_after_std": 0.36307925172150135, + "reward_before_mean": 0.029698201455175877, + "reward_before_std": 0.23980092909187078, + "reward_change_max": 0.0, + "reward_change_mean": -0.3521753493696451, + "reward_change_min": -0.49568963050842285, + "reward_change_std": 0.18385440576821566, + "reward_std": 0.36307926289737225, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.1161351166665554, + "step": 203 + }, + { + "clip_fraction": 0.0, + "completion_length": 2511.833366394043, + "epoch": 0.23314285714285715, + "grad_norm": 0.07267706841230392, + "kl": 5.4508447647094727e-05, + "lambda_div_used": 0.5887176766991615, + "learning_rate": 7.667891533457718e-07, + "loss": 0.0023, + "reward": -0.23160922899842262, + "reward_after_mean": -0.23160922899842262, + "reward_after_std": 0.430733734741807, + "reward_before_mean": 0.06992286071181297, + "reward_before_std": 0.4116028640419245, + "reward_change_max": 0.0, + "reward_change_mean": -0.301532082259655, + "reward_change_min": -0.49793890863657, + "reward_change_std": 0.19511268101632595, + "reward_std": 0.43073374405503273, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.07591048628091812, + "step": 204 + }, + { + "clip_fraction": 0.0, + "completion_length": 2967.1875915527344, + "epoch": 0.2342857142857143, + "grad_norm": 0.06776981800794601, + "kl": 4.439055919647217e-05, + "lambda_div_used": 0.653583250939846, + "learning_rate": 7.640308940816239e-07, + "loss": 0.1014, + "reward": -0.08090712130069733, + "reward_after_mean": -0.08090712130069733, + "reward_after_std": 0.7262772191315889, + "reward_before_mean": 0.17835846357047558, + "reward_before_std": 0.7181071005761623, + "reward_change_max": 0.0, + "reward_change_mean": -0.2592655848711729, + "reward_change_min": -0.4743462074548006, + "reward_change_std": 0.18475584778934717, + "reward_std": 0.7262772284448147, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.05080821271985769, + "step": 205 + }, + { + "clip_fraction": 0.0, + "completion_length": 3085.5208892822266, + "epoch": 0.23542857142857143, + "grad_norm": 0.06689873337745667, + "kl": 3.569573163986206e-05, + "lambda_div_used": 0.5862618833780289, + "learning_rate": 7.612622032536507e-07, + "loss": 0.0353, + "reward": -0.28365214727818966, + "reward_after_mean": -0.28365214727818966, + "reward_after_std": 0.44012872874736786, + "reward_before_mean": 0.004427256062626839, + "reward_before_std": 0.3988625044003129, + "reward_change_max": 0.0, + "reward_change_mean": -0.2880793921649456, + "reward_change_min": -0.49235254526138306, + "reward_change_std": 0.18395678792148829, + "reward_std": 0.44012873619794846, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.12057275045663118, + "step": 206 + }, + { + "clip_fraction": 0.0, + "completion_length": 3146.6458740234375, + "epoch": 0.23657142857142857, + "grad_norm": 0.06122450903058052, + "kl": 3.725104033946991e-05, + "lambda_div_used": 0.579835832118988, + "learning_rate": 7.584832158039378e-07, + "loss": 0.0182, + "reward": -0.28688727831467986, + "reward_after_mean": -0.28688727831467986, + "reward_after_std": 0.4150819983333349, + "reward_before_mean": 0.009919969365000725, + "reward_before_std": 0.3629662115126848, + "reward_change_max": 0.0, + "reward_change_mean": -0.29680725932121277, + "reward_change_min": -0.45919613167643547, + "reward_change_std": 0.17279880121350288, + "reward_std": 0.41508200392127037, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.11508002015762031, + "step": 207 + }, + { + "clip_fraction": 0.0, + "completion_length": 2780.291717529297, + "epoch": 0.2377142857142857, + "grad_norm": 0.06370716542005539, + "kl": 3.692507743835449e-05, + "lambda_div_used": 0.5807016789913177, + "learning_rate": 7.556940671764124e-07, + "loss": 0.0403, + "reward": -0.17094913870096207, + "reward_after_mean": -0.17094913870096207, + "reward_after_std": 0.46442117914557457, + "reward_before_mean": 0.20349296741187572, + "reward_before_std": 0.3671952560544014, + "reward_change_max": 0.0, + "reward_change_mean": -0.3744421415030956, + "reward_change_min": -0.5505415536463261, + "reward_change_std": 0.21210224367678165, + "reward_std": 0.46442119032144547, + "rewards/accuracy_reward": 0.20833333395421505, + "rewards/cosine_scaled_reward": -0.004840357229113579, + "step": 208 + }, + { + "clip_fraction": 0.0, + "completion_length": 2574.6042079925537, + "epoch": 0.23885714285714285, + "grad_norm": 0.08756011724472046, + "kl": 1.8547754734754562e-05, + "lambda_div_used": 0.6246868968009949, + "learning_rate": 7.528948933102438e-07, + "loss": 0.0837, + "reward": 0.028150439262390137, + "reward_after_mean": 0.028150439262390137, + "reward_after_std": 0.5636487938463688, + "reward_before_mean": 0.3838757500052452, + "reward_before_std": 0.576031070202589, + "reward_change_max": 0.0, + "reward_change_mean": -0.3557253200560808, + "reward_change_min": -0.5899170190095901, + "reward_change_std": 0.23946599010378122, + "reward_std": 0.5636488180607557, + "rewards/accuracy_reward": 0.3333333469927311, + "rewards/cosine_scaled_reward": 0.05054241791367531, + "step": 209 + }, + { + "clip_fraction": 0.0, + "completion_length": 2779.3333435058594, + "epoch": 0.24, + "grad_norm": 0.06111512333154678, + "kl": 3.0368566513061523e-05, + "lambda_div_used": 0.5897415727376938, + "learning_rate": 7.500858306332172e-07, + "loss": 0.0277, + "reward": -0.25500940857455134, + "reward_after_mean": -0.25500940857455134, + "reward_after_std": 0.47190882451832294, + "reward_before_mean": 0.03478116978658363, + "reward_before_std": 0.4098346810787916, + "reward_change_max": 0.0, + "reward_change_mean": -0.2897905595600605, + "reward_change_min": -0.42270801588892937, + "reward_change_std": 0.1620404813438654, + "reward_std": 0.4719088301062584, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.11105216934811324, + "step": 210 + }, + { + "clip_fraction": 0.0, + "completion_length": 2558.8541946411133, + "epoch": 0.24114285714285713, + "grad_norm": 0.06923027336597443, + "kl": 3.6522746086120605e-05, + "lambda_div_used": 0.5664671063423157, + "learning_rate": 7.472670160550848e-07, + "loss": 0.0729, + "reward": -0.06898702308535576, + "reward_after_mean": -0.06898702308535576, + "reward_after_std": 0.4354093614965677, + "reward_before_mean": 0.4075740482658148, + "reward_before_std": 0.30287738889455795, + "reward_change_max": 0.0, + "reward_change_mean": -0.4765610620379448, + "reward_change_min": -0.6883851811289787, + "reward_change_std": 0.262312775477767, + "reward_std": 0.4354093801230192, + "rewards/accuracy_reward": 0.3125, + "rewards/cosine_scaled_reward": 0.09507404454052448, + "step": 211 + }, + { + "clip_fraction": 0.0, + "completion_length": 2327.750015258789, + "epoch": 0.2422857142857143, + "grad_norm": 0.08028864860534668, + "kl": 2.8003007173538208e-05, + "lambda_div_used": 0.5712975114583969, + "learning_rate": 7.444385869608921e-07, + "loss": -0.0113, + "reward": -0.09517045877873898, + "reward_after_mean": -0.09517045877873898, + "reward_after_std": 0.49211281538009644, + "reward_before_mean": 0.3566615767776966, + "reward_before_std": 0.32028379291296005, + "reward_change_max": 0.0, + "reward_change_mean": -0.4518320318311453, + "reward_change_min": -0.6186705827713013, + "reward_change_std": 0.23167487233877182, + "reward_std": 0.4921128321439028, + "rewards/accuracy_reward": 0.2916666679084301, + "rewards/cosine_scaled_reward": 0.06499490264104679, + "step": 212 + }, + { + "clip_fraction": 0.0, + "completion_length": 2145.6041946411133, + "epoch": 0.24342857142857144, + "grad_norm": 0.09073800593614578, + "kl": 4.177819937467575e-05, + "lambda_div_used": 0.6168783828616142, + "learning_rate": 7.416006812042827e-07, + "loss": 0.027, + "reward": -0.08367926510982215, + "reward_after_mean": -0.08367926510982215, + "reward_after_std": 0.5790203902870417, + "reward_before_mean": 0.24448610469698906, + "reward_before_std": 0.5364782512187958, + "reward_change_max": 0.0, + "reward_change_mean": -0.32816537097096443, + "reward_change_min": -0.5068525895476341, + "reward_change_std": 0.19713077135384083, + "reward_std": 0.579020407050848, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": 0.036152773071080446, + "step": 213 + }, + { + "clip_fraction": 0.0, + "completion_length": 2982.6458587646484, + "epoch": 0.24457142857142858, + "grad_norm": 0.06497277319431305, + "kl": 3.772228956222534e-05, + "lambda_div_used": 0.6168569102883339, + "learning_rate": 7.387534371007797e-07, + "loss": 0.0247, + "reward": -0.10639690980315208, + "reward_after_mean": -0.10639690980315208, + "reward_after_std": 0.563305439427495, + "reward_before_mean": 0.21648684330284595, + "reward_before_std": 0.5375220291316509, + "reward_change_max": 0.0, + "reward_change_mean": -0.3228837437927723, + "reward_change_min": -0.5641361065208912, + "reward_change_std": 0.20987980626523495, + "reward_std": 0.5633054543286562, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": 0.00815348862670362, + "step": 214 + }, + { + "clip_fraction": 0.0, + "completion_length": 2382.2500610351562, + "epoch": 0.24571428571428572, + "grad_norm": 0.07877103984355927, + "kl": 4.485622048377991e-05, + "lambda_div_used": 0.5695274397730827, + "learning_rate": 7.358969934210438e-07, + "loss": 0.0351, + "reward": -0.39196348818950355, + "reward_after_mean": -0.39196348818950355, + "reward_after_std": 0.39862883277237415, + "reward_before_mean": -0.128106027841568, + "reward_before_std": 0.31178954988718033, + "reward_change_max": 0.0, + "reward_change_mean": -0.263857439160347, + "reward_change_min": -0.38021961972117424, + "reward_change_std": 0.13994430005550385, + "reward_std": 0.39862884022295475, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.1697727027349174, + "step": 215 + }, + { + "clip_fraction": 0.0, + "completion_length": 2156.3958740234375, + "epoch": 0.24685714285714286, + "grad_norm": 0.11318857222795486, + "kl": 4.297494888305664e-05, + "lambda_div_used": 0.5965342745184898, + "learning_rate": 7.330314893841101e-07, + "loss": 0.0186, + "reward": -0.22045887634158134, + "reward_after_mean": -0.22045887634158134, + "reward_after_std": 0.4805217906832695, + "reward_before_mean": 0.08258800266776234, + "reward_before_std": 0.4415160808712244, + "reward_change_max": 0.0, + "reward_change_mean": -0.3030468635261059, + "reward_change_min": -0.4999500457197428, + "reward_change_std": 0.185057258233428, + "reward_std": 0.4805217981338501, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.08407866954803467, + "step": 216 + }, + { + "clip_fraction": 0.0, + "completion_length": 2763.520866394043, + "epoch": 0.248, + "grad_norm": 0.067295141518116, + "kl": 3.0644237995147705e-05, + "lambda_div_used": 0.6201670467853546, + "learning_rate": 7.301570646506027e-07, + "loss": 0.0139, + "reward": -0.19884846359491348, + "reward_after_mean": -0.19884846359491348, + "reward_after_std": 0.5856006741523743, + "reward_before_mean": 0.06158541014883667, + "reward_before_std": 0.5545480605214834, + "reward_change_max": 0.0, + "reward_change_mean": -0.2604338899254799, + "reward_change_min": -0.503634799271822, + "reward_change_std": 0.1784855630248785, + "reward_std": 0.5856006946414709, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.10508125275373459, + "step": 217 + }, + { + "clip_fraction": 0.0, + "completion_length": 2880.7083587646484, + "epoch": 0.24914285714285714, + "grad_norm": 0.07706139236688614, + "kl": 3.7044286727905273e-05, + "lambda_div_used": 0.6074161231517792, + "learning_rate": 7.27273859315928e-07, + "loss": -0.0223, + "reward": -0.19500153325498104, + "reward_after_mean": -0.19500153325498104, + "reward_after_std": 0.5332945492118597, + "reward_before_mean": 0.08328226953744888, + "reward_before_std": 0.5006468072533607, + "reward_change_max": 0.0, + "reward_change_mean": -0.27828381210565567, + "reward_change_min": -0.5113113783299923, + "reward_change_std": 0.18696410488337278, + "reward_std": 0.5332945715636015, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.08338439278304577, + "step": 218 + }, + { + "clip_fraction": 0.0, + "completion_length": 2352.645854949951, + "epoch": 0.2502857142857143, + "grad_norm": 0.10948827862739563, + "kl": 3.1441450119018555e-05, + "lambda_div_used": 0.5561031624674797, + "learning_rate": 7.243820139034464e-07, + "loss": -0.0096, + "reward": -0.029067307710647583, + "reward_after_mean": -0.029067307710647583, + "reward_after_std": 0.41141366213560104, + "reward_before_mean": 0.4911606255918741, + "reward_before_std": 0.2519808644428849, + "reward_change_max": 0.0, + "reward_change_mean": -0.5202279426157475, + "reward_change_min": -0.7093416638672352, + "reward_change_std": 0.27748389169573784, + "reward_std": 0.41141367703676224, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.13699393905699253, + "step": 219 + }, + { + "clip_fraction": 0.0, + "completion_length": 2635.5833435058594, + "epoch": 0.25142857142857145, + "grad_norm": 0.08914919197559357, + "kl": 3.5672448575496674e-05, + "lambda_div_used": 0.5308618620038033, + "learning_rate": 7.214816693576234e-07, + "loss": -0.0362, + "reward": -0.5146691724658012, + "reward_after_mean": -0.5146691724658012, + "reward_after_std": 0.23347610421478748, + "reward_before_mean": -0.2369950506836176, + "reward_before_std": 0.13545648753643036, + "reward_change_max": 0.0, + "reward_change_mean": -0.27767411433160305, + "reward_change_min": -0.3908136747777462, + "reward_change_std": 0.14405533485114574, + "reward_std": 0.23347610607743263, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2369950469583273, + "step": 220 + }, + { + "clip_fraction": 0.0, + "completion_length": 2279.562530517578, + "epoch": 0.25257142857142856, + "grad_norm": 0.0954662561416626, + "kl": 2.4788081645965576e-05, + "lambda_div_used": 0.5764833092689514, + "learning_rate": 7.185729670371604e-07, + "loss": -0.0168, + "reward": 0.023251693695783615, + "reward_after_mean": 0.023251693695783615, + "reward_after_std": 0.4863298423588276, + "reward_before_mean": 0.5212067291140556, + "reward_before_std": 0.34889572812244296, + "reward_change_max": 0.0, + "reward_change_mean": -0.49795500561594963, + "reward_change_min": -0.7016883753240108, + "reward_change_std": 0.27911104913800955, + "reward_std": 0.48632985167205334, + "rewards/accuracy_reward": 0.4166666716337204, + "rewards/cosine_scaled_reward": 0.10454002395272255, + "step": 221 + }, + { + "clip_fraction": 0.0, + "completion_length": 2104.9583740234375, + "epoch": 0.2537142857142857, + "grad_norm": 0.0776790902018547, + "kl": 2.4726614356040955e-05, + "lambda_div_used": 0.5997953563928604, + "learning_rate": 7.156560487081051e-07, + "loss": -0.0148, + "reward": -0.04329463094472885, + "reward_after_mean": -0.04329463094472885, + "reward_after_std": 0.48106030002236366, + "reward_before_mean": 0.33003126084804535, + "reward_before_std": 0.46051184553653, + "reward_change_max": 0.0, + "reward_change_mean": -0.3733258917927742, + "reward_change_min": -0.5728014186024666, + "reward_change_std": 0.23287776950746775, + "reward_std": 0.48106031119823456, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.0591979268938303, + "step": 222 + }, + { + "clip_fraction": 0.0, + "completion_length": 2521.3333587646484, + "epoch": 0.25485714285714284, + "grad_norm": 0.07036174088716507, + "kl": 2.9304996132850647e-05, + "lambda_div_used": 0.5750112235546112, + "learning_rate": 7.127310565369415e-07, + "loss": 0.0216, + "reward": -0.1473498847335577, + "reward_after_mean": -0.1473498847335577, + "reward_after_std": 0.4359878208488226, + "reward_before_mean": 0.24288302287459373, + "reward_before_std": 0.3438769578933716, + "reward_change_max": 0.0, + "reward_change_mean": -0.39023288898169994, + "reward_change_min": -0.588653527200222, + "reward_change_std": 0.22452317085117102, + "reward_std": 0.43598783388733864, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.027950339019298553, + "step": 223 + }, + { + "clip_fraction": 0.0, + "completion_length": 3317.729217529297, + "epoch": 0.256, + "grad_norm": 0.05309538170695305, + "kl": 2.8050970286130905e-05, + "lambda_div_used": 0.5725216493010521, + "learning_rate": 7.097981330836616e-07, + "loss": 0.069, + "reward": -0.30242439545691013, + "reward_after_mean": -0.30242439545691013, + "reward_after_std": 0.38869454339146614, + "reward_before_mean": -0.000308917835354805, + "reward_before_std": 0.3251040354371071, + "reward_change_max": 0.0, + "reward_change_mean": -0.30211549811065197, + "reward_change_min": -0.4553787522017956, + "reward_change_std": 0.171081081032753, + "reward_std": 0.3886945564299822, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.12530891119968146, + "step": 224 + }, + { + "clip_fraction": 0.0, + "completion_length": 3037.208335876465, + "epoch": 0.2571428571428571, + "grad_norm": 0.0768936425447464, + "kl": 4.080682992935181e-05, + "lambda_div_used": 0.5907088667154312, + "learning_rate": 7.068574212948169e-07, + "loss": 0.0169, + "reward": -0.28160014655441046, + "reward_after_mean": -0.28160014655441046, + "reward_after_std": 0.48472702503204346, + "reward_before_mean": -0.008234186680056155, + "reward_before_std": 0.41540220472961664, + "reward_change_max": 0.0, + "reward_change_mean": -0.27336596697568893, + "reward_change_min": -0.4061685614287853, + "reward_change_std": 0.15673903841525316, + "reward_std": 0.48472702503204346, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.13323417864739895, + "step": 225 + }, + { + "clip_fraction": 0.0, + "completion_length": 2861.416702270508, + "epoch": 0.2582857142857143, + "grad_norm": 0.07499676197767258, + "kl": 3.580749034881592e-05, + "lambda_div_used": 0.6208535805344582, + "learning_rate": 7.039090644965509e-07, + "loss": 0.0702, + "reward": -0.08291551470756531, + "reward_after_mean": -0.08291551470756531, + "reward_after_std": 0.5764818880707026, + "reward_before_mean": 0.23111886344850063, + "reward_before_std": 0.5593543313443661, + "reward_change_max": 0.0, + "reward_change_mean": -0.3140343725681305, + "reward_change_min": -0.5283008627593517, + "reward_change_std": 0.20683539099991322, + "reward_std": 0.5764818955212831, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": -0.03971448230731767, + "step": 226 + }, + { + "clip_fraction": 0.0, + "completion_length": 1926.6875305175781, + "epoch": 0.25942857142857145, + "grad_norm": 0.10923109948635101, + "kl": 4.176795482635498e-05, + "lambda_div_used": 0.5834182798862457, + "learning_rate": 7.009532063876148e-07, + "loss": -0.0447, + "reward": -0.2778073139488697, + "reward_after_mean": -0.2778073139488697, + "reward_after_std": 0.4395454227924347, + "reward_before_mean": 0.009000460617244244, + "reward_before_std": 0.3901812704280019, + "reward_change_max": 0.0, + "reward_change_mean": -0.28680778108537197, + "reward_change_min": -0.45677450299263, + "reward_change_std": 0.18111994117498398, + "reward_std": 0.43954543210566044, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.1159995449706912, + "step": 227 + }, + { + "clip_fraction": 0.0, + "completion_length": 2502.562545776367, + "epoch": 0.26057142857142856, + "grad_norm": 0.08182154595851898, + "kl": 1.8984079360961914e-05, + "lambda_div_used": 0.6025907471776009, + "learning_rate": 6.979899910323624e-07, + "loss": 0.074, + "reward": 0.03976001590490341, + "reward_after_mean": 0.03976001590490341, + "reward_after_std": 0.5421929359436035, + "reward_before_mean": 0.4711545445024967, + "reward_before_std": 0.4750876808539033, + "reward_change_max": 0.0, + "reward_change_mean": -0.4313945174217224, + "reward_change_min": -0.6774051003158092, + "reward_change_std": 0.2637105621397495, + "reward_std": 0.5421929433941841, + "rewards/accuracy_reward": 0.3541666753590107, + "rewards/cosine_scaled_reward": 0.11698784306645393, + "step": 228 + }, + { + "clip_fraction": 0.0, + "completion_length": 3196.3125228881836, + "epoch": 0.26171428571428573, + "grad_norm": 0.06398586928844452, + "kl": 4.0434300899505615e-05, + "lambda_div_used": 0.5669709742069244, + "learning_rate": 6.950195628537299e-07, + "loss": 0.0135, + "reward": -0.15348458290100098, + "reward_after_mean": -0.15348458290100098, + "reward_after_std": 0.42672324273735285, + "reward_before_mean": 0.26130594592541456, + "reward_before_std": 0.3047938751988113, + "reward_change_max": 0.0, + "reward_change_mean": -0.41479056514799595, + "reward_change_min": -0.5674791261553764, + "reward_change_std": 0.22689451277256012, + "reward_std": 0.4267232706770301, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": 0.03213928057812154, + "step": 229 + }, + { + "clip_fraction": 0.0, + "completion_length": 3100.5000228881836, + "epoch": 0.26285714285714284, + "grad_norm": 0.05996527522802353, + "kl": 2.22623348236084e-05, + "lambda_div_used": 0.5811162814497948, + "learning_rate": 6.920420666261961e-07, + "loss": 0.0391, + "reward": -0.36068666726350784, + "reward_after_mean": -0.36068666726350784, + "reward_after_std": 0.43734684213995934, + "reward_before_mean": -0.08804995659738779, + "reward_before_std": 0.37305002473294735, + "reward_change_max": 0.0, + "reward_change_mean": -0.2726367134600878, + "reward_change_min": -0.4353605732321739, + "reward_change_std": 0.16089125256985426, + "reward_std": 0.43734684586524963, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.15054996183607727, + "step": 230 + }, + { + "clip_fraction": 0.0, + "completion_length": 2678.7083587646484, + "epoch": 0.264, + "grad_norm": 0.06770680844783783, + "kl": 2.5890767574310303e-05, + "lambda_div_used": 0.5817164853215218, + "learning_rate": 6.890576474687263e-07, + "loss": 0.0722, + "reward": -0.13625335786491632, + "reward_after_mean": -0.13625335786491632, + "reward_after_std": 0.46202925965189934, + "reward_before_mean": 0.25235490314662457, + "reward_before_std": 0.37311020120978355, + "reward_change_max": 0.0, + "reward_change_mean": -0.3886082824319601, + "reward_change_min": -0.6004670634865761, + "reward_change_std": 0.2275555245578289, + "reward_std": 0.4620292726904154, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.018478410318493843, + "step": 231 + }, + { + "clip_fraction": 0.0, + "completion_length": 3258.625030517578, + "epoch": 0.2651428571428571, + "grad_norm": 0.05468503385782242, + "kl": 3.515183925628662e-05, + "lambda_div_used": 0.5800251811742783, + "learning_rate": 6.860664508377001e-07, + "loss": 0.0159, + "reward": -0.3428461756557226, + "reward_after_mean": -0.3428461756557226, + "reward_after_std": 0.4266065489500761, + "reward_before_mean": -0.07336848601698875, + "reward_before_std": 0.36755594704300165, + "reward_change_max": 0.0, + "reward_change_mean": -0.26947769708931446, + "reward_change_min": -0.43084757775068283, + "reward_change_std": 0.1597052849829197, + "reward_std": 0.4266065787523985, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.1567018236964941, + "step": 232 + }, + { + "clip_fraction": 0.0, + "completion_length": 2661.2292251586914, + "epoch": 0.2662857142857143, + "grad_norm": 0.08775883167982101, + "kl": 3.3989548683166504e-05, + "lambda_div_used": 0.6155472174286842, + "learning_rate": 6.83068622519821e-07, + "loss": -0.016, + "reward": -0.22876367531716824, + "reward_after_mean": -0.22876367531716824, + "reward_after_std": 0.5766322333365679, + "reward_before_mean": 0.027781556826084852, + "reward_before_std": 0.5290827043354511, + "reward_change_max": 0.0, + "reward_change_mean": -0.2565452288836241, + "reward_change_min": -0.4581022933125496, + "reward_change_std": 0.16627201065421104, + "reward_std": 0.5766322445124388, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.11805178504437208, + "step": 233 + }, + { + "clip_fraction": 0.0, + "completion_length": 2694.3125228881836, + "epoch": 0.2674285714285714, + "grad_norm": 0.11633959412574768, + "kl": 3.643333911895752e-05, + "lambda_div_used": 0.5539941042661667, + "learning_rate": 6.800643086250121e-07, + "loss": -0.0056, + "reward": -0.22800572216510773, + "reward_after_mean": -0.22800572216510773, + "reward_after_std": 0.3484720904380083, + "reward_before_mean": 0.16009100899100304, + "reward_before_std": 0.24206165876239538, + "reward_change_max": 0.0, + "reward_change_mean": -0.3880967255681753, + "reward_change_min": -0.5548957660794258, + "reward_change_std": 0.21049270872026682, + "reward_std": 0.34847209975123405, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.06907567009329796, + "step": 234 + }, + { + "clip_fraction": 0.0, + "completion_length": 2490.4375, + "epoch": 0.26857142857142857, + "grad_norm": 0.10378725826740265, + "kl": 2.8094742447137833e-05, + "lambda_div_used": 0.5782083421945572, + "learning_rate": 6.770536555792944e-07, + "loss": -0.0425, + "reward": 0.01590564101934433, + "reward_after_mean": 0.01590564101934433, + "reward_after_std": 0.5053408965468407, + "reward_before_mean": 0.49857149738818407, + "reward_before_std": 0.3586234971880913, + "reward_change_max": 0.0, + "reward_change_mean": -0.48266585171222687, + "reward_change_min": -0.7199156694114208, + "reward_change_std": 0.2697788691148162, + "reward_std": 0.505340900272131, + "rewards/accuracy_reward": 0.3750000037252903, + "rewards/cosine_scaled_reward": 0.12357146013528109, + "step": 235 + }, + { + "clip_fraction": 0.0, + "completion_length": 2824.7917098999023, + "epoch": 0.26971428571428574, + "grad_norm": 0.06474865972995758, + "kl": 2.1502375602722168e-05, + "lambda_div_used": 0.5763456672430038, + "learning_rate": 6.740368101176495e-07, + "loss": 0.0011, + "reward": -0.3977004513144493, + "reward_after_mean": -0.3977004513144493, + "reward_after_std": 0.41489073634147644, + "reward_before_mean": -0.1460555698722601, + "reward_before_std": 0.347917802631855, + "reward_change_max": 0.0, + "reward_change_mean": -0.2516448702663183, + "reward_change_min": -0.42327918112277985, + "reward_change_std": 0.14836463797837496, + "reward_std": 0.4148907568305731, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.20855557406321168, + "step": 236 + }, + { + "clip_fraction": 0.0, + "completion_length": 2764.9583740234375, + "epoch": 0.27085714285714285, + "grad_norm": 0.06224860996007919, + "kl": 2.5756657123565674e-05, + "lambda_div_used": 0.5886275842785835, + "learning_rate": 6.710139192768694e-07, + "loss": -0.0228, + "reward": -0.1175742200575769, + "reward_after_mean": -0.1175742200575769, + "reward_after_std": 0.4582329224795103, + "reward_before_mean": 0.24255024455487728, + "reward_before_std": 0.40142686292529106, + "reward_change_max": 0.0, + "reward_change_mean": -0.36012447625398636, + "reward_change_min": -0.5257687419652939, + "reward_change_std": 0.20567627251148224, + "reward_std": 0.4582329299300909, + "rewards/accuracy_reward": 0.22916667722165585, + "rewards/cosine_scaled_reward": 0.013383567042183131, + "step": 237 + }, + { + "clip_fraction": 0.0, + "completion_length": 3222.479217529297, + "epoch": 0.272, + "grad_norm": 0.05841728672385216, + "kl": 3.546103835105896e-05, + "lambda_div_used": 0.6281886473298073, + "learning_rate": 6.679851303883891e-07, + "loss": 0.0655, + "reward": 0.044486068189144135, + "reward_after_mean": 0.044486068189144135, + "reward_after_std": 0.6025157757103443, + "reward_before_mean": 0.4036002438515425, + "reward_before_std": 0.6015807576477528, + "reward_change_max": 0.0, + "reward_change_mean": -0.359114158898592, + "reward_change_min": -0.609935961663723, + "reward_change_std": 0.24452881701290607, + "reward_std": 0.6025157924741507, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.09110023453831673, + "step": 238 + }, + { + "clip_fraction": 0.0, + "completion_length": 1740.1666793823242, + "epoch": 0.27314285714285713, + "grad_norm": 0.09357193857431412, + "kl": 1.8790364265441895e-05, + "lambda_div_used": 0.5962883979082108, + "learning_rate": 6.649505910711058e-07, + "loss": 0.0055, + "reward": 0.07376761082559824, + "reward_after_mean": 0.07376761082559824, + "reward_after_std": 0.5891099888831377, + "reward_before_mean": 0.5742647312581539, + "reward_before_std": 0.44304153323173523, + "reward_change_max": 0.0, + "reward_change_mean": -0.5004971195012331, + "reward_change_min": -0.7436067499220371, + "reward_change_std": 0.2840570341795683, + "reward_std": 0.5891100075095892, + "rewards/accuracy_reward": 0.37500000186264515, + "rewards/cosine_scaled_reward": 0.19926471984945238, + "step": 239 + }, + { + "clip_fraction": 0.0, + "completion_length": 3087.416702270508, + "epoch": 0.2742857142857143, + "grad_norm": 0.06324295699596405, + "kl": 3.217160701751709e-05, + "lambda_div_used": 0.5339473709464073, + "learning_rate": 6.619104492241847e-07, + "loss": 0.0166, + "reward": -0.5358126908540726, + "reward_after_mean": -0.5358126908540726, + "reward_after_std": 0.23053276538848877, + "reward_before_mean": -0.2753155492246151, + "reward_before_std": 0.1490377252921462, + "reward_change_max": 0.0, + "reward_change_mean": -0.26049717888236046, + "reward_change_min": -0.3783254958689213, + "reward_change_std": 0.1383643699809909, + "reward_std": 0.23053277097642422, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2753155454993248, + "step": 240 + }, + { + "clip_fraction": 0.0, + "completion_length": 3419.625, + "epoch": 0.2754285714285714, + "grad_norm": 0.047396283596754074, + "kl": 3.555417060852051e-05, + "lambda_div_used": 0.5585425272583961, + "learning_rate": 6.588648530198504e-07, + "loss": 0.0027, + "reward": -0.4805446630343795, + "reward_after_mean": -0.4805446630343795, + "reward_after_std": 0.3355217073112726, + "reward_before_mean": -0.23604051489382982, + "reward_before_std": 0.2620125887915492, + "reward_change_max": 0.0, + "reward_change_mean": -0.2445041425526142, + "reward_change_min": -0.37724288925528526, + "reward_change_std": 0.13484715577214956, + "reward_std": 0.3355217222124338, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.2568738413974643, + "step": 241 + }, + { + "clip_fraction": 0.0, + "completion_length": 2400.8750228881836, + "epoch": 0.2765714285714286, + "grad_norm": 0.08974741399288177, + "kl": 4.5746564865112305e-05, + "lambda_div_used": 0.5887154564261436, + "learning_rate": 6.558139508961654e-07, + "loss": -0.0452, + "reward": -0.2555653927847743, + "reward_after_mean": -0.2555653927847743, + "reward_after_std": 0.47001883387565613, + "reward_before_mean": 0.03157716616988182, + "reward_before_std": 0.4061046461574733, + "reward_change_max": 0.0, + "reward_change_mean": -0.28714255429804325, + "reward_change_min": -0.41172971203923225, + "reward_change_std": 0.16218022629618645, + "reward_std": 0.47001886926591396, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.1142561559099704, + "step": 242 + }, + { + "clip_fraction": 0.0, + "completion_length": 2828.541679382324, + "epoch": 0.2777142857142857, + "grad_norm": 0.07105688005685806, + "kl": 1.8571503460407257e-05, + "lambda_div_used": 0.5722818151116371, + "learning_rate": 6.527578915497951e-07, + "loss": 0.0039, + "reward": -0.1431608721613884, + "reward_after_mean": -0.1431608721613884, + "reward_after_std": 0.42974008433520794, + "reward_before_mean": 0.2529455106705427, + "reward_before_std": 0.3276430475525558, + "reward_change_max": 0.0, + "reward_change_mean": -0.39610639959573746, + "reward_change_min": -0.5491434335708618, + "reward_change_std": 0.21527612209320068, + "reward_std": 0.42974009923636913, + "rewards/accuracy_reward": 0.25000000558793545, + "rewards/cosine_scaled_reward": 0.0029455198673531413, + "step": 243 + }, + { + "clip_fraction": 0.0, + "completion_length": 2882.291679382324, + "epoch": 0.27885714285714286, + "grad_norm": 0.0700126439332962, + "kl": 3.098323941230774e-05, + "lambda_div_used": 0.6161807402968407, + "learning_rate": 6.496968239287603e-07, + "loss": 0.0, + "reward": -0.06458889320492744, + "reward_after_mean": -0.06458889320492744, + "reward_after_std": 0.5576508566737175, + "reward_before_mean": 0.2605547234416008, + "reward_before_std": 0.5402053641155362, + "reward_change_max": 0.0, + "reward_change_mean": -0.32514358684420586, + "reward_change_min": -0.5207333639264107, + "reward_change_std": 0.21059911139309406, + "reward_std": 0.5576508603990078, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": -0.01027863984927535, + "step": 244 + }, + { + "clip_fraction": 0.0, + "completion_length": 2907.4583587646484, + "epoch": 0.28, + "grad_norm": 0.05816182866692543, + "kl": 3.5434961318969727e-05, + "lambda_div_used": 0.5615689232945442, + "learning_rate": 6.466308972251785e-07, + "loss": -0.0133, + "reward": -0.42641448229551315, + "reward_after_mean": -0.42641448229551315, + "reward_after_std": 0.3361923936754465, + "reward_before_mean": -0.15409247018396854, + "reward_before_std": 0.2813150165602565, + "reward_change_max": 0.0, + "reward_change_mean": -0.2723220158368349, + "reward_change_min": -0.445806298404932, + "reward_change_std": 0.1635848032310605, + "reward_std": 0.3361923974007368, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.19575914042070508, + "step": 245 + }, + { + "clip_fraction": 0.0, + "completion_length": 2680.7083587646484, + "epoch": 0.28114285714285714, + "grad_norm": 0.07198330760002136, + "kl": 1.764507032930851e-05, + "lambda_div_used": 0.5865297466516495, + "learning_rate": 6.435602608679916e-07, + "loss": 0.0347, + "reward": -0.19568787328898907, + "reward_after_mean": -0.19568787328898907, + "reward_after_std": 0.4485197216272354, + "reward_before_mean": 0.129365224391222, + "reward_before_std": 0.393932550214231, + "reward_change_max": 0.0, + "reward_change_mean": -0.32505310885608196, + "reward_change_min": -0.4816659241914749, + "reward_change_std": 0.1897038472816348, + "reward_std": 0.44851974956691265, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.0789681114256382, + "step": 246 + }, + { + "clip_fraction": 0.0, + "completion_length": 3076.0208435058594, + "epoch": 0.2822857142857143, + "grad_norm": 0.08006418496370316, + "kl": 4.15705144405365e-05, + "lambda_div_used": 0.5735789015889168, + "learning_rate": 6.404850645156841e-07, + "loss": 0.0199, + "reward": -0.3997113136574626, + "reward_after_mean": -0.3997113136574626, + "reward_after_std": 0.41638931445777416, + "reward_before_mean": -0.1426885835826397, + "reward_before_std": 0.3330965582281351, + "reward_change_max": 0.0, + "reward_change_mean": -0.2570227347314358, + "reward_change_min": -0.38052143156528473, + "reward_change_std": 0.13911327440291643, + "reward_std": 0.4163893237709999, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.18435525218956172, + "step": 247 + }, + { + "clip_fraction": 0.0, + "completion_length": 2145.0000228881836, + "epoch": 0.2834285714285714, + "grad_norm": 0.10052043944597244, + "kl": 3.3289194107055664e-05, + "lambda_div_used": 0.6171368733048439, + "learning_rate": 6.374054580489873e-07, + "loss": 0.0514, + "reward": 0.28496517799794674, + "reward_after_mean": 0.28496517799794674, + "reward_after_std": 0.6654367055743933, + "reward_before_mean": 0.83577667362988, + "reward_before_std": 0.5420532608404756, + "reward_change_max": 0.0, + "reward_change_mean": -0.5508115068078041, + "reward_change_min": -0.7994196638464928, + "reward_change_std": 0.3166588256135583, + "reward_std": 0.6654367428272963, + "rewards/accuracy_reward": 0.5000000093132257, + "rewards/cosine_scaled_reward": 0.33577666338533163, + "step": 248 + }, + { + "clip_fraction": 0.0, + "completion_length": 2076.937515258789, + "epoch": 0.2845714285714286, + "grad_norm": 0.0908605083823204, + "kl": 2.4942681193351746e-05, + "lambda_div_used": 0.5948201194405556, + "learning_rate": 6.343215915635761e-07, + "loss": 0.0219, + "reward": -0.003894178196787834, + "reward_after_mean": -0.003894178196787834, + "reward_after_std": 0.5242901761084795, + "reward_before_mean": 0.4153097262606025, + "reward_before_std": 0.43534869560971856, + "reward_change_max": 0.0, + "reward_change_mean": -0.41920389235019684, + "reward_change_min": -0.5903440341353416, + "reward_change_std": 0.2394579891115427, + "reward_std": 0.524290194734931, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.08197638019919395, + "step": 249 + }, + { + "clip_fraction": 0.0, + "completion_length": 2605.0208587646484, + "epoch": 0.2857142857142857, + "grad_norm": 0.09027790278196335, + "kl": 3.804638981819153e-05, + "lambda_div_used": 0.5779460370540619, + "learning_rate": 6.31233615362752e-07, + "loss": -0.0589, + "reward": -0.2859988175332546, + "reward_after_mean": -0.2859988175332546, + "reward_after_std": 0.4080943390727043, + "reward_before_mean": 0.008792944252490997, + "reward_before_std": 0.35337654035538435, + "reward_change_max": 0.0, + "reward_change_mean": -0.2947917561978102, + "reward_change_min": -0.4365835040807724, + "reward_change_std": 0.16840537451207638, + "reward_std": 0.40809434466063976, + "rewards/accuracy_reward": 0.12500000558793545, + "rewards/cosine_scaled_reward": -0.11620705761015415, + "step": 250 + }, + { + "clip_fraction": 0.0, + "completion_length": 2368.9167137145996, + "epoch": 0.28685714285714287, + "grad_norm": 0.09640161693096161, + "kl": 2.7060508728027344e-05, + "lambda_div_used": 0.5987462773919106, + "learning_rate": 6.281416799501187e-07, + "loss": -0.0435, + "reward": -0.028017258271574974, + "reward_after_mean": -0.028017258271574974, + "reward_after_std": 0.5376799181103706, + "reward_before_mean": 0.38438196340575814, + "reward_before_std": 0.4528377316892147, + "reward_change_max": 0.0, + "reward_change_mean": -0.41239920631051064, + "reward_change_min": -0.6119380593299866, + "reward_change_std": 0.23853347077965736, + "reward_std": 0.5376799292862415, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.05104860384017229, + "step": 251 + }, + { + "clip_fraction": 0.0, + "completion_length": 2798.5625228881836, + "epoch": 0.288, + "grad_norm": 0.06784425675868988, + "kl": 3.535952419042587e-05, + "lambda_div_used": 0.5580189973115921, + "learning_rate": 6.25045936022246e-07, + "loss": 0.0335, + "reward": -0.2931139934808016, + "reward_after_mean": -0.2931139934808016, + "reward_after_std": 0.38731373474001884, + "reward_before_mean": 0.06481979880481958, + "reward_before_std": 0.2658566879108548, + "reward_change_max": 0.0, + "reward_change_mean": -0.3579337988048792, + "reward_change_min": -0.5358714908361435, + "reward_change_std": 0.19932966493070126, + "reward_std": 0.3873137477785349, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.10184688493609428, + "step": 252 + }, + { + "clip_fraction": 0.0, + "completion_length": 2946.416679382324, + "epoch": 0.28914285714285715, + "grad_norm": 0.09991417825222015, + "kl": 3.4242868423461914e-05, + "lambda_div_used": 0.6016674339771271, + "learning_rate": 6.219465344613258e-07, + "loss": -0.0465, + "reward": -0.0356330550275743, + "reward_after_mean": -0.0356330550275743, + "reward_after_std": 0.5561915785074234, + "reward_before_mean": 0.353804474696517, + "reward_before_std": 0.4667428769171238, + "reward_change_max": 0.0, + "reward_change_mean": -0.389437522739172, + "reward_change_min": -0.592089481651783, + "reward_change_std": 0.22914788126945496, + "reward_std": 0.5561915840953588, + "rewards/accuracy_reward": 0.31250000558793545, + "rewards/cosine_scaled_reward": 0.041304459096863866, + "step": 253 + }, + { + "clip_fraction": 0.0, + "completion_length": 2754.3541946411133, + "epoch": 0.29028571428571426, + "grad_norm": 0.08061102032661438, + "kl": 3.886967897415161e-05, + "lambda_div_used": 0.6251527816057205, + "learning_rate": 6.188436263278172e-07, + "loss": 0.0857, + "reward": -0.1664750911295414, + "reward_after_mean": -0.1664750911295414, + "reward_after_std": 0.599596256390214, + "reward_before_mean": 0.1026190984994173, + "reward_before_std": 0.5836509419605136, + "reward_change_max": 0.0, + "reward_change_mean": -0.2690941859036684, + "reward_change_min": -0.49960801005363464, + "reward_change_std": 0.19118957500904799, + "reward_std": 0.5995962955057621, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.08488089527236298, + "step": 254 + }, + { + "clip_fraction": 0.0, + "completion_length": 3190.083335876465, + "epoch": 0.2914285714285714, + "grad_norm": 0.08434556424617767, + "kl": 2.3322179913520813e-05, + "lambda_div_used": 0.5955284982919693, + "learning_rate": 6.157373628530852e-07, + "loss": -0.0125, + "reward": -0.3585695568472147, + "reward_after_mean": -0.3585695568472147, + "reward_after_std": 0.4996061436831951, + "reward_before_mean": -0.12086338270455599, + "reward_before_std": 0.4362535886466503, + "reward_change_max": 0.0, + "reward_change_mean": -0.2377061638981104, + "reward_change_min": -0.39885040931403637, + "reward_change_std": 0.1406589960679412, + "reward_std": 0.4996061585843563, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.20419671526178718, + "step": 255 + }, + { + "clip_fraction": 0.0, + "completion_length": 3160.8125610351562, + "epoch": 0.2925714285714286, + "grad_norm": 0.0576457642018795, + "kl": 3.143772482872009e-05, + "lambda_div_used": 0.639025017619133, + "learning_rate": 6.126278954320294e-07, + "loss": 0.0243, + "reward": -0.03833652473986149, + "reward_after_mean": -0.03833652473986149, + "reward_after_std": 0.6670235451310873, + "reward_before_mean": 0.2675688254312263, + "reward_before_std": 0.6473761759698391, + "reward_change_max": 0.0, + "reward_change_mean": -0.30590534023940563, + "reward_change_min": -0.511355496942997, + "reward_change_std": 0.2077637044712901, + "reward_std": 0.6670235693454742, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": 0.01756881456822157, + "step": 256 + }, + { + "clip_fraction": 0.0, + "completion_length": 3138.5000228881836, + "epoch": 0.2937142857142857, + "grad_norm": 0.05595054104924202, + "kl": 3.0346214771270752e-05, + "lambda_div_used": 0.6474356725811958, + "learning_rate": 6.095153756157051e-07, + "loss": 0.025, + "reward": -0.0769930558744818, + "reward_after_mean": -0.0769930558744818, + "reward_after_std": 0.7138958293944597, + "reward_before_mean": 0.18525638710707426, + "reward_before_std": 0.685269920155406, + "reward_change_max": 0.0, + "reward_change_mean": -0.26224944926798344, + "reward_change_min": -0.4810149297118187, + "reward_change_std": 0.18088674824684858, + "reward_std": 0.7138958312571049, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.04391028080135584, + "step": 257 + }, + { + "clip_fraction": 0.0, + "completion_length": 3473.187530517578, + "epoch": 0.2948571428571429, + "grad_norm": 0.05202889442443848, + "kl": 3.561750054359436e-05, + "lambda_div_used": 0.6007983982563019, + "learning_rate": 6.06399955103937e-07, + "loss": 0.0058, + "reward": -0.23316247668117285, + "reward_after_mean": -0.23316247668117285, + "reward_after_std": 0.5119184870272875, + "reward_before_mean": 0.04610642418265343, + "reward_before_std": 0.46757086645811796, + "reward_change_max": 0.0, + "reward_change_mean": -0.2792689222842455, + "reward_change_min": -0.4602624364197254, + "reward_change_std": 0.17828952055424452, + "reward_std": 0.5119185000658035, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.09972689533606172, + "step": 258 + }, + { + "clip_fraction": 0.0, + "completion_length": 2996.7500534057617, + "epoch": 0.296, + "grad_norm": 0.11353152990341187, + "kl": 4.0959566831588745e-05, + "lambda_div_used": 0.605031318962574, + "learning_rate": 6.032817857379256e-07, + "loss": -0.0572, + "reward": -0.24131755530834198, + "reward_after_mean": -0.24131755530834198, + "reward_after_std": 0.5208989772945642, + "reward_before_mean": 0.03226012596860528, + "reward_before_std": 0.4831458665430546, + "reward_change_max": 0.0, + "reward_change_mean": -0.27357766777276993, + "reward_change_min": -0.4753991588950157, + "reward_change_std": 0.1725512887351215, + "reward_std": 0.5208989884704351, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.1135732214897871, + "step": 259 + }, + { + "clip_fraction": 0.0, + "completion_length": 2184.8542098999023, + "epoch": 0.29714285714285715, + "grad_norm": 0.1352783739566803, + "kl": 2.3871660232543945e-05, + "lambda_div_used": 0.6604775562882423, + "learning_rate": 6.001610194928464e-07, + "loss": 0.0496, + "reward": 0.1305767484009266, + "reward_after_mean": 0.1305767484009266, + "reward_after_std": 0.7162795849144459, + "reward_before_mean": 0.48802967881783843, + "reward_before_std": 0.7462537074461579, + "reward_change_max": 0.0, + "reward_change_mean": -0.3574528992176056, + "reward_change_min": -0.6257632970809937, + "reward_change_std": 0.25755990110337734, + "reward_std": 0.7162796072661877, + "rewards/accuracy_reward": 0.39583334513008595, + "rewards/cosine_scaled_reward": 0.0921963145956397, + "step": 260 + }, + { + "clip_fraction": 0.0, + "completion_length": 3070.3125228881836, + "epoch": 0.29828571428571427, + "grad_norm": 0.06917808949947357, + "kl": 3.3020973205566406e-05, + "lambda_div_used": 0.5845921337604523, + "learning_rate": 5.97037808470444e-07, + "loss": 0.0106, + "reward": -0.3141371374949813, + "reward_after_mean": -0.3141371374949813, + "reward_after_std": 0.43580205366015434, + "reward_before_mean": -0.03358305338770151, + "reward_before_std": 0.38949463702738285, + "reward_change_max": 0.0, + "reward_change_mean": -0.28055410645902157, + "reward_change_min": -0.44674795493483543, + "reward_change_std": 0.16879158467054367, + "reward_std": 0.4358020592480898, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.11691637896001339, + "step": 261 + }, + { + "clip_fraction": 0.0, + "completion_length": 3258.3125, + "epoch": 0.29942857142857143, + "grad_norm": 0.06239396706223488, + "kl": 4.535168409347534e-05, + "lambda_div_used": 0.5396690741181374, + "learning_rate": 5.939123048916173e-07, + "loss": 0.0153, + "reward": -0.49852147325873375, + "reward_after_mean": -0.49852147325873375, + "reward_after_std": 0.23993146419525146, + "reward_before_mean": -0.21930878423154354, + "reward_before_std": 0.17422470543533564, + "reward_change_max": 0.0, + "reward_change_mean": -0.2792126890271902, + "reward_change_min": -0.4082174263894558, + "reward_change_std": 0.15261581167578697, + "reward_std": 0.2399314697831869, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.21930878423154354, + "step": 262 + }, + { + "clip_fraction": 0.0, + "completion_length": 2837.6666717529297, + "epoch": 0.30057142857142854, + "grad_norm": 0.07131204754114151, + "kl": 2.771243453025818e-05, + "lambda_div_used": 0.5753477811813354, + "learning_rate": 5.907846610890011e-07, + "loss": 0.0159, + "reward": -0.32574891671538353, + "reward_after_mean": -0.32574891671538353, + "reward_after_std": 0.40547293052077293, + "reward_before_mean": -0.04349888768047094, + "reward_before_std": 0.3437284992542118, + "reward_change_max": 0.0, + "reward_change_mean": -0.28225001133978367, + "reward_change_min": -0.423517182469368, + "reward_change_std": 0.16225541010499, + "reward_std": 0.4054729398339987, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.14766556210815907, + "step": 263 + }, + { + "clip_fraction": 0.0, + "completion_length": 2913.812515258789, + "epoch": 0.3017142857142857, + "grad_norm": 0.07276218384504318, + "kl": 3.5703182220458984e-05, + "lambda_div_used": 0.5857644975185394, + "learning_rate": 5.87655029499542e-07, + "loss": 0.039, + "reward": -0.19073306024074554, + "reward_after_mean": -0.19073306024074554, + "reward_after_std": 0.4183583725243807, + "reward_before_mean": 0.13579276762902737, + "reward_before_std": 0.39237749949097633, + "reward_change_max": 0.0, + "reward_change_mean": -0.32652581483125687, + "reward_change_min": -0.512863963842392, + "reward_change_std": 0.20149299688637257, + "reward_std": 0.4183583725243807, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.051707250997424126, + "step": 264 + }, + { + "clip_fraction": 0.0, + "completion_length": 2092.333351135254, + "epoch": 0.3028571428571429, + "grad_norm": 0.09631343185901642, + "kl": 2.018176019191742e-05, + "lambda_div_used": 0.5927653685212135, + "learning_rate": 5.845235626570683e-07, + "loss": 0.0409, + "reward": -0.05978839658200741, + "reward_after_mean": -0.05978839658200741, + "reward_after_std": 0.5236263573169708, + "reward_before_mean": 0.3295632619410753, + "reward_before_std": 0.4253385625779629, + "reward_change_max": 0.0, + "reward_change_mean": -0.38935166224837303, + "reward_change_min": -0.5855174511671066, + "reward_change_std": 0.22452097665518522, + "reward_std": 0.5236263833940029, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.037896597757935524, + "step": 265 + }, + { + "clip_fraction": 0.0, + "completion_length": 3210.8541717529297, + "epoch": 0.304, + "grad_norm": 0.05451129376888275, + "kl": 3.138929605484009e-05, + "lambda_div_used": 0.557657316327095, + "learning_rate": 5.813904131848564e-07, + "loss": -0.0024, + "reward": -0.3950183019042015, + "reward_after_mean": -0.3950183019042015, + "reward_after_std": 0.32154187746345997, + "reward_before_mean": -0.12019951082766056, + "reward_before_std": 0.2612606221809983, + "reward_change_max": 0.0, + "reward_change_mean": -0.2748187892138958, + "reward_change_min": -0.42080841958522797, + "reward_change_std": 0.158494733273983, + "reward_std": 0.321541890501976, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.20353284664452076, + "step": 266 + }, + { + "clip_fraction": 0.0, + "completion_length": 3039.916679382324, + "epoch": 0.30514285714285716, + "grad_norm": 0.06252676248550415, + "kl": 3.5075165214948356e-05, + "lambda_div_used": 0.5528770685195923, + "learning_rate": 5.78255733788191e-07, + "loss": -0.037, + "reward": -0.4460434205830097, + "reward_after_mean": -0.4460434205830097, + "reward_after_std": 0.3107005339115858, + "reward_before_mean": -0.1744655454531312, + "reward_before_std": 0.2377403611317277, + "reward_change_max": 0.0, + "reward_change_mean": -0.27157786674797535, + "reward_change_min": -0.42647456377744675, + "reward_change_std": 0.1541104121133685, + "reward_std": 0.3107005413621664, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.21613221243023872, + "step": 267 + }, + { + "clip_fraction": 0.0, + "completion_length": 2704.8333587646484, + "epoch": 0.3062857142857143, + "grad_norm": 0.08417963981628418, + "kl": 4.7713518142700195e-05, + "lambda_div_used": 0.6233388632535934, + "learning_rate": 5.751196772469237e-07, + "loss": 0.0822, + "reward": -0.21941478177905083, + "reward_after_mean": -0.21941478177905083, + "reward_after_std": 0.6098091676831245, + "reward_before_mean": 0.03345827816519886, + "reward_before_std": 0.5743174999952316, + "reward_change_max": 0.0, + "reward_change_mean": -0.25287305377423763, + "reward_change_min": -0.45724478363990784, + "reward_change_std": 0.1690685572102666, + "reward_std": 0.6098091900348663, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.11237506754696369, + "step": 268 + }, + { + "clip_fraction": 0.0, + "completion_length": 3165.9791717529297, + "epoch": 0.30742857142857144, + "grad_norm": 0.06371315568685532, + "kl": 2.181902527809143e-05, + "lambda_div_used": 0.5798910111188889, + "learning_rate": 5.71982396408026e-07, + "loss": 0.0477, + "reward": -0.38025568798184395, + "reward_after_mean": -0.38025568798184395, + "reward_after_std": 0.4126162938773632, + "reward_before_mean": -0.13023450784385204, + "reward_before_std": 0.3629760518670082, + "reward_change_max": 0.0, + "reward_change_mean": -0.2500211838632822, + "reward_change_min": -0.40024495497345924, + "reward_change_std": 0.15232266392558813, + "reward_std": 0.4126163087785244, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.2135678417980671, + "step": 269 + }, + { + "clip_fraction": 0.0, + "completion_length": 3056.875045776367, + "epoch": 0.30857142857142855, + "grad_norm": 0.05920446664094925, + "kl": 2.387911081314087e-05, + "lambda_div_used": 0.6503699943423271, + "learning_rate": 5.688440441781398e-07, + "loss": 0.0, + "reward": -0.004786740057170391, + "reward_after_mean": -0.004786740057170391, + "reward_after_std": 0.6943319551646709, + "reward_before_mean": 0.292231020051986, + "reward_before_std": 0.7036092299968004, + "reward_change_max": 0.0, + "reward_change_mean": -0.2970177587121725, + "reward_change_min": -0.5634518079459667, + "reward_change_std": 0.21872046310454607, + "reward_std": 0.6943319924175739, + "rewards/accuracy_reward": 0.27083333767950535, + "rewards/cosine_scaled_reward": 0.021397670439910144, + "step": 270 + }, + { + "clip_fraction": 0.0, + "completion_length": 2221.3958702087402, + "epoch": 0.3097142857142857, + "grad_norm": 0.09025074541568756, + "kl": 6.723217666149139e-06, + "lambda_div_used": 0.640851192176342, + "learning_rate": 5.657047735161255e-07, + "loss": 0.0064, + "reward": 0.11755780689418316, + "reward_after_mean": 0.11755780689418316, + "reward_after_std": 0.6356876939535141, + "reward_before_mean": 0.4748641401529312, + "reward_before_std": 0.654548792168498, + "reward_change_max": 0.0, + "reward_change_mean": -0.35730634443461895, + "reward_change_min": -0.5644437614828348, + "reward_change_std": 0.23989163804799318, + "reward_std": 0.6356877088546753, + "rewards/accuracy_reward": 0.39583334885537624, + "rewards/cosine_scaled_reward": 0.07903079688549042, + "step": 271 + }, + { + "clip_fraction": 0.0, + "completion_length": 2814.5208740234375, + "epoch": 0.31085714285714283, + "grad_norm": 0.07214022427797318, + "kl": 2.553686499595642e-05, + "lambda_div_used": 0.5532237812876701, + "learning_rate": 5.625647374256061e-07, + "loss": -0.023, + "reward": -0.33643968403339386, + "reward_after_mean": -0.33643968403339386, + "reward_after_std": 0.35168860107660294, + "reward_before_mean": 0.01186647079885006, + "reward_before_std": 0.2353415172547102, + "reward_change_max": 0.0, + "reward_change_mean": -0.34830615669488907, + "reward_change_min": -0.5079392194747925, + "reward_change_std": 0.1846030419692397, + "reward_std": 0.351688614115119, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.13396686874330044, + "step": 272 + }, + { + "clip_fraction": 0.0, + "completion_length": 2673.937515258789, + "epoch": 0.312, + "grad_norm": 0.067450150847435, + "kl": 2.347538247704506e-05, + "lambda_div_used": 0.5976931154727936, + "learning_rate": 5.594240889475106e-07, + "loss": 0.0439, + "reward": 0.060379184782505035, + "reward_after_mean": 0.060379184782505035, + "reward_after_std": 0.5129444599151611, + "reward_before_mean": 0.4946505483239889, + "reward_before_std": 0.44454328902065754, + "reward_change_max": 0.0, + "reward_change_mean": -0.4342713989317417, + "reward_change_min": -0.6668536812067032, + "reward_change_std": 0.25412870943546295, + "reward_std": 0.5129444822669029, + "rewards/accuracy_reward": 0.3750000111758709, + "rewards/cosine_scaled_reward": 0.1196505706757307, + "step": 273 + }, + { + "clip_fraction": 0.0, + "completion_length": 1866.4375076293945, + "epoch": 0.31314285714285717, + "grad_norm": 0.11823767423629761, + "kl": 2.6108697056770325e-05, + "lambda_div_used": 0.6173145174980164, + "learning_rate": 5.562829811526154e-07, + "loss": -0.0666, + "reward": 0.20245935022830963, + "reward_after_mean": 0.20245935022830963, + "reward_after_std": 0.6078098546713591, + "reward_before_mean": 0.6919816123554483, + "reward_before_std": 0.5468513960950077, + "reward_change_max": 0.0, + "reward_change_mean": -0.4895222559571266, + "reward_change_min": -0.7379350513219833, + "reward_change_std": 0.3024911228567362, + "reward_std": 0.6078098695725203, + "rewards/accuracy_reward": 0.4583333432674408, + "rewards/cosine_scaled_reward": 0.2336482839891687, + "step": 274 + }, + { + "clip_fraction": 0.0, + "completion_length": 2334.0416870117188, + "epoch": 0.3142857142857143, + "grad_norm": 0.07390157133340836, + "kl": 2.975761890411377e-05, + "lambda_div_used": 0.5953021869063377, + "learning_rate": 5.531415671340826e-07, + "loss": 0.0067, + "reward": -0.02193348854780197, + "reward_after_mean": -0.02193348854780197, + "reward_after_std": 0.5273136273026466, + "reward_before_mean": 0.3926870714276447, + "reward_before_std": 0.4364564120769501, + "reward_change_max": 0.0, + "reward_change_mean": -0.4146205447614193, + "reward_change_min": -0.6028474234044552, + "reward_change_std": 0.23770872876048088, + "reward_std": 0.5273136328905821, + "rewards/accuracy_reward": 0.31250000558793545, + "rewards/cosine_scaled_reward": 0.08018706925213337, + "step": 275 + }, + { + "clip_fraction": 0.0, + "completion_length": 2735.6458740234375, + "epoch": 0.31542857142857145, + "grad_norm": 0.07891591638326645, + "kl": 3.522634506225586e-05, + "lambda_div_used": 0.6019129753112793, + "learning_rate": 5.5e-07, + "loss": -0.0433, + "reward": 0.016597013920545578, + "reward_after_mean": 0.016597013920545578, + "reward_after_std": 0.5496951006352901, + "reward_before_mean": 0.43198024667799473, + "reward_before_std": 0.4716393407434225, + "reward_change_max": 0.0, + "reward_change_mean": -0.4153832569718361, + "reward_change_min": -0.6486029289662838, + "reward_change_std": 0.25036295782774687, + "reward_std": 0.5496951248496771, + "rewards/accuracy_reward": 0.3333333395421505, + "rewards/cosine_scaled_reward": 0.09864690899848938, + "step": 276 + }, + { + "clip_fraction": 0.0, + "completion_length": 2435.437530517578, + "epoch": 0.31657142857142856, + "grad_norm": 0.08397295325994492, + "kl": 2.7611851692199707e-05, + "lambda_div_used": 0.6150463595986366, + "learning_rate": 5.468584328659172e-07, + "loss": 0.0536, + "reward": -0.049143560230731964, + "reward_after_mean": -0.049143560230731964, + "reward_after_std": 0.5724204778671265, + "reward_before_mean": 0.29686027206480503, + "reward_before_std": 0.5315626971423626, + "reward_change_max": 0.0, + "reward_change_mean": -0.34600381925702095, + "reward_change_min": -0.5762605480849743, + "reward_change_std": 0.21639508474618196, + "reward_std": 0.5724204815924168, + "rewards/accuracy_reward": 0.27083333767950535, + "rewards/cosine_scaled_reward": 0.02602693811058998, + "step": 277 + }, + { + "clip_fraction": 0.0, + "completion_length": 2029.708381652832, + "epoch": 0.3177142857142857, + "grad_norm": 0.11247767508029938, + "kl": 3.094971179962158e-05, + "lambda_div_used": 0.6170216798782349, + "learning_rate": 5.437170188473847e-07, + "loss": 0.002, + "reward": 0.056462954729795456, + "reward_after_mean": 0.056462954729795456, + "reward_after_std": 0.6216260213404894, + "reward_before_mean": 0.48691817931830883, + "reward_before_std": 0.5461275167763233, + "reward_change_max": 0.0, + "reward_change_mean": -0.4304552264511585, + "reward_change_min": -0.7186719551682472, + "reward_change_std": 0.27375681325793266, + "reward_std": 0.6216260306537151, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.13275149278342724, + "step": 278 + }, + { + "clip_fraction": 0.0, + "completion_length": 3158.6875228881836, + "epoch": 0.31885714285714284, + "grad_norm": 0.06945524364709854, + "kl": 3.522634506225586e-05, + "lambda_div_used": 0.5564287528395653, + "learning_rate": 5.405759110524894e-07, + "loss": -0.0272, + "reward": -0.43442236818373203, + "reward_after_mean": -0.43442236818373203, + "reward_after_std": 0.32187592424452305, + "reward_before_mean": -0.1544840056449175, + "reward_before_std": 0.25075172632932663, + "reward_change_max": 0.0, + "reward_change_mean": -0.27993838116526604, + "reward_change_min": -0.41151439025998116, + "reward_change_std": 0.15283891931176186, + "reward_std": 0.32187593914568424, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.17531732900533825, + "step": 279 + }, + { + "clip_fraction": 0.0, + "completion_length": 2157.833335876465, + "epoch": 0.32, + "grad_norm": 0.14424921572208405, + "kl": 4.202499985694885e-05, + "lambda_div_used": 0.630903884768486, + "learning_rate": 5.37435262574394e-07, + "loss": -0.0339, + "reward": -0.14438428170979023, + "reward_after_mean": -0.14438428170979023, + "reward_after_std": 0.6414528246968985, + "reward_before_mean": 0.1313652544049546, + "reward_before_std": 0.6011563409119844, + "reward_change_max": 0.0, + "reward_change_mean": -0.275749534368515, + "reward_change_min": -0.4570390097796917, + "reward_change_std": 0.1709576854482293, + "reward_std": 0.6414528302848339, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.076968085631961, + "step": 280 + }, + { + "clip_fraction": 0.0, + "completion_length": 3559.000030517578, + "epoch": 0.3211428571428571, + "grad_norm": 0.0494488850235939, + "kl": 3.0197203159332275e-05, + "lambda_div_used": 0.5301951244473457, + "learning_rate": 5.342952264838747e-07, + "loss": -0.0035, + "reward": -0.5142169296741486, + "reward_after_mean": -0.5142169296741486, + "reward_after_std": 0.22926145792007446, + "reward_before_mean": -0.2300790660083294, + "reward_before_std": 0.1324998252093792, + "reward_change_max": 0.0, + "reward_change_mean": -0.2841378580778837, + "reward_change_min": -0.4100039228796959, + "reward_change_std": 0.14742697216570377, + "reward_std": 0.22926146537065506, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2300790660083294, + "step": 281 + }, + { + "clip_fraction": 0.0, + "completion_length": 2757.7708587646484, + "epoch": 0.3222857142857143, + "grad_norm": 0.07136296480894089, + "kl": 1.5236437320709229e-05, + "lambda_div_used": 0.6041858941316605, + "learning_rate": 5.311559558218603e-07, + "loss": 0.0304, + "reward": 0.016035709530115128, + "reward_after_mean": 0.016035709530115128, + "reward_after_std": 0.5573885068297386, + "reward_before_mean": 0.4244079850614071, + "reward_before_std": 0.48719789227470756, + "reward_change_max": 0.0, + "reward_change_mean": -0.40837226063013077, + "reward_change_min": -0.630510251969099, + "reward_change_std": 0.253003865480423, + "reward_std": 0.5573885291814804, + "rewards/accuracy_reward": 0.3541666716337204, + "rewards/cosine_scaled_reward": 0.07024132460355759, + "step": 282 + }, + { + "clip_fraction": 0.0, + "completion_length": 2882.5416946411133, + "epoch": 0.32342857142857145, + "grad_norm": 0.05982055515050888, + "kl": 1.1865049600601196e-05, + "lambda_div_used": 0.6191478446125984, + "learning_rate": 5.28017603591974e-07, + "loss": -0.015, + "reward": 0.10000502690672874, + "reward_after_mean": 0.10000502690672874, + "reward_after_std": 0.6287181153893471, + "reward_before_mean": 0.5279214177280664, + "reward_before_std": 0.5531216450035572, + "reward_change_max": 0.0, + "reward_change_mean": -0.4279164057224989, + "reward_change_min": -0.6583010666072369, + "reward_change_std": 0.2602456407621503, + "reward_std": 0.6287181228399277, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/cosine_scaled_reward": 0.1529214084148407, + "step": 283 + }, + { + "clip_fraction": 0.0, + "completion_length": 2187.083381652832, + "epoch": 0.32457142857142857, + "grad_norm": 0.20831985771656036, + "kl": 5.378853529691696e-05, + "lambda_div_used": 0.617803268134594, + "learning_rate": 5.248803227530763e-07, + "loss": 0.0301, + "reward": -0.09672576747834682, + "reward_after_mean": -0.09672576747834682, + "reward_after_std": 0.5820088647305965, + "reward_before_mean": 0.21344758570194244, + "reward_before_std": 0.544846129603684, + "reward_change_max": 0.0, + "reward_change_mean": -0.3101733736693859, + "reward_change_min": -0.5270103476941586, + "reward_change_std": 0.20029443874955177, + "reward_std": 0.5820088759064674, + "rewards/accuracy_reward": 0.22916667349636555, + "rewards/cosine_scaled_reward": -0.0157190952450037, + "step": 284 + }, + { + "clip_fraction": 0.0, + "completion_length": 2571.4583435058594, + "epoch": 0.32571428571428573, + "grad_norm": 0.05750874802470207, + "kl": 2.5823712348937988e-05, + "lambda_div_used": 0.5412982106208801, + "learning_rate": 5.21744266211809e-07, + "loss": 0.0429, + "reward": -0.3610886335372925, + "reward_after_mean": -0.3610886335372925, + "reward_after_std": 0.2963402010500431, + "reward_before_mean": 0.004524916410446167, + "reward_before_std": 0.18141429405659437, + "reward_change_max": 0.0, + "reward_change_mean": -0.36561354249715805, + "reward_change_min": -0.5212092585861683, + "reward_change_std": 0.19485185854136944, + "reward_std": 0.29634021408855915, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.12047509290277958, + "step": 285 + }, + { + "clip_fraction": 0.0, + "completion_length": 2607.625030517578, + "epoch": 0.32685714285714285, + "grad_norm": 0.06422320753335953, + "kl": 2.036895602941513e-05, + "lambda_div_used": 0.593349277973175, + "learning_rate": 5.186095868151436e-07, + "loss": 0.0069, + "reward": -0.1434864792972803, + "reward_after_mean": -0.1434864792972803, + "reward_after_std": 0.4716298431158066, + "reward_before_mean": 0.18680993653833866, + "reward_before_std": 0.4280705275014043, + "reward_change_max": 0.0, + "reward_change_mean": -0.3302964009344578, + "reward_change_min": -0.5444469898939133, + "reward_change_std": 0.20489494875073433, + "reward_std": 0.4716298636049032, + "rewards/accuracy_reward": 0.2291666753590107, + "rewards/cosine_scaled_reward": -0.04235674999654293, + "step": 286 + }, + { + "clip_fraction": 0.0, + "completion_length": 2068.4375343322754, + "epoch": 0.328, + "grad_norm": 0.11217836290597916, + "kl": 3.282725811004639e-05, + "lambda_div_used": 0.576636016368866, + "learning_rate": 5.154764373429315e-07, + "loss": -0.0127, + "reward": -0.015125550329685211, + "reward_after_mean": -0.015125550329685211, + "reward_after_std": 0.44022079929709435, + "reward_before_mean": 0.4338516741991043, + "reward_before_std": 0.349846001714468, + "reward_change_max": 0.0, + "reward_change_mean": -0.4489772208034992, + "reward_change_min": -0.6400703266263008, + "reward_change_std": 0.25500839948654175, + "reward_std": 0.44022080302238464, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.12135165929794312, + "step": 287 + }, + { + "clip_fraction": 0.0, + "completion_length": 3021.1666870117188, + "epoch": 0.3291428571428571, + "grad_norm": 0.05326732248067856, + "kl": 2.1582163753919303e-05, + "lambda_div_used": 0.5814583152532578, + "learning_rate": 5.123449705004581e-07, + "loss": 0.0215, + "reward": -0.23482287488877773, + "reward_after_mean": -0.23482287488877773, + "reward_after_std": 0.4797380156815052, + "reward_before_mean": 0.10618079453706741, + "reward_before_std": 0.37177785113453865, + "reward_change_max": 0.0, + "reward_change_mean": -0.34100368432700634, + "reward_change_min": -0.5246531553566456, + "reward_change_std": 0.190623770467937, + "reward_std": 0.47973802499473095, + "rewards/accuracy_reward": 0.18750000186264515, + "rewards/cosine_scaled_reward": -0.08131920825690031, + "step": 288 + }, + { + "clip_fraction": 0.0, + "completion_length": 2210.0208435058594, + "epoch": 0.3302857142857143, + "grad_norm": 0.0918356254696846, + "kl": 3.3445656299591064e-05, + "lambda_div_used": 0.5579611882567406, + "learning_rate": 5.09215338910999e-07, + "loss": 0.0308, + "reward": -0.1890019178390503, + "reward_after_mean": -0.1890019178390503, + "reward_after_std": 0.38851393200457096, + "reward_before_mean": 0.2269407268613577, + "reward_before_std": 0.2580757178366184, + "reward_change_max": 0.0, + "reward_change_mean": -0.4159426633268595, + "reward_change_min": -0.5701811872422695, + "reward_change_std": 0.21641669981181622, + "reward_std": 0.38851393945515156, + "rewards/accuracy_reward": 0.27083333395421505, + "rewards/cosine_scaled_reward": -0.04389259871095419, + "step": 289 + }, + { + "clip_fraction": 0.0, + "completion_length": 1820.6666870117188, + "epoch": 0.3314285714285714, + "grad_norm": 0.10334110260009766, + "kl": 1.7639249563217163e-05, + "lambda_div_used": 0.649879202246666, + "learning_rate": 5.060876951083828e-07, + "loss": -0.0389, + "reward": 0.04819735325872898, + "reward_after_mean": 0.04819735325872898, + "reward_after_std": 0.752808591350913, + "reward_before_mean": 0.39133079699240625, + "reward_before_std": 0.7000208692625165, + "reward_change_max": 0.0, + "reward_change_mean": -0.3431334514170885, + "reward_change_min": -0.5988858677446842, + "reward_change_std": 0.2255254928022623, + "reward_std": 0.7528086155653, + "rewards/accuracy_reward": 0.3333333358168602, + "rewards/cosine_scaled_reward": 0.05799746699631214, + "step": 290 + }, + { + "clip_fraction": 0.0, + "completion_length": 2717.166732788086, + "epoch": 0.3325714285714286, + "grad_norm": 0.07149659842252731, + "kl": 9.842216968536377e-06, + "lambda_div_used": 0.6230586618185043, + "learning_rate": 5.02962191529556e-07, + "loss": 0.0634, + "reward": 0.04544975981116295, + "reward_after_mean": 0.04544975981116295, + "reward_after_std": 0.6337935384362936, + "reward_before_mean": 0.4299341347068548, + "reward_before_std": 0.5707205794751644, + "reward_change_max": 0.0, + "reward_change_mean": -0.38448438234627247, + "reward_change_min": -0.6367709413170815, + "reward_change_std": 0.2398422248661518, + "reward_std": 0.6337935607880354, + "rewards/accuracy_reward": 0.33333333767950535, + "rewards/cosine_scaled_reward": 0.09660080214962363, + "step": 291 + }, + { + "clip_fraction": 0.0, + "completion_length": 3176.979202270508, + "epoch": 0.33371428571428574, + "grad_norm": 0.05150657147169113, + "kl": 2.0240433514118195e-05, + "lambda_div_used": 0.5560602247714996, + "learning_rate": 4.998389805071536e-07, + "loss": 0.062, + "reward": -0.4294139966368675, + "reward_after_mean": -0.4294139966368675, + "reward_after_std": 0.3452510107308626, + "reward_before_mean": -0.15922172274440527, + "reward_before_std": 0.2578093442134559, + "reward_change_max": 0.0, + "reward_change_mean": -0.27019229158759117, + "reward_change_min": -0.42562897875905037, + "reward_change_std": 0.15438843425363302, + "reward_std": 0.3452510181814432, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.20088838692754507, + "step": 292 + }, + { + "clip_fraction": 0.0, + "completion_length": 2630.9791946411133, + "epoch": 0.33485714285714285, + "grad_norm": 0.05954331159591675, + "kl": 1.2833625078201294e-05, + "lambda_div_used": 0.5555919855833054, + "learning_rate": 4.967182142620745e-07, + "loss": 0.0009, + "reward": -0.19852645695209503, + "reward_after_mean": -0.19852645695209503, + "reward_after_std": 0.3515181578695774, + "reward_before_mean": 0.20652466267347336, + "reward_before_std": 0.24862384609878063, + "reward_change_max": 0.0, + "reward_change_mean": -0.40505112148821354, + "reward_change_min": -0.5747470110654831, + "reward_change_std": 0.22049889154732227, + "reward_std": 0.35151816345751286, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.022642023861408234, + "step": 293 + }, + { + "clip_fraction": 0.0, + "completion_length": 3005.5, + "epoch": 0.336, + "grad_norm": 0.06993379443883896, + "kl": 2.8274953365325928e-05, + "lambda_div_used": 0.5608418732881546, + "learning_rate": 4.93600044896063e-07, + "loss": -0.0248, + "reward": -0.3667301833629608, + "reward_after_mean": -0.3667301833629608, + "reward_after_std": 0.32350156269967556, + "reward_before_mean": -0.07280110754072666, + "reward_before_std": 0.2737845163792372, + "reward_change_max": 0.0, + "reward_change_mean": -0.29392906464636326, + "reward_change_min": -0.4593726359307766, + "reward_change_std": 0.17168805841356516, + "reward_std": 0.3235015720129013, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.15613443590700626, + "step": 294 + }, + { + "clip_fraction": 0.0, + "completion_length": 3221.2291717529297, + "epoch": 0.33714285714285713, + "grad_norm": 0.06364311277866364, + "kl": 1.817569136619568e-05, + "lambda_div_used": 0.5979067236185074, + "learning_rate": 4.904846243842949e-07, + "loss": 0.0107, + "reward": -0.259306401014328, + "reward_after_mean": -0.259306401014328, + "reward_after_std": 0.5024331342428923, + "reward_before_mean": 0.018260781886056066, + "reward_before_std": 0.45087322127074003, + "reward_change_max": 0.0, + "reward_change_mean": -0.277567183598876, + "reward_change_min": -0.41658853366971016, + "reward_change_std": 0.1643626783043146, + "reward_std": 0.5024331398308277, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.10673921927809715, + "step": 295 + }, + { + "clip_fraction": 0.0, + "completion_length": 3282.8750610351562, + "epoch": 0.3382857142857143, + "grad_norm": 0.0698896273970604, + "kl": 2.3877248167991638e-05, + "lambda_div_used": 0.5625706240534782, + "learning_rate": 4.873721045679706e-07, + "loss": 0.0079, + "reward": -0.39721857756376266, + "reward_after_mean": -0.39721857756376266, + "reward_after_std": 0.34284412302076817, + "reward_before_mean": -0.1091517936438322, + "reward_before_std": 0.28544116113334894, + "reward_change_max": 0.0, + "reward_change_mean": -0.28806679882109165, + "reward_change_min": -0.4639837518334389, + "reward_change_std": 0.1706175785511732, + "reward_std": 0.3428441286087036, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.15081845596432686, + "step": 296 + }, + { + "clip_fraction": 0.0, + "completion_length": 3569.8541870117188, + "epoch": 0.3394285714285714, + "grad_norm": 0.04541197791695595, + "kl": 2.245139330625534e-05, + "lambda_div_used": 0.5649159774184227, + "learning_rate": 4.842626371469149e-07, + "loss": 0.0071, + "reward": -0.43140796944499016, + "reward_after_mean": -0.43140796944499016, + "reward_after_std": 0.3433863054960966, + "reward_before_mean": -0.17089181207120419, + "reward_before_std": 0.29400468710809946, + "reward_change_max": 0.0, + "reward_change_mean": -0.26051616482436657, + "reward_change_min": -0.4413977116346359, + "reward_change_std": 0.15714262332767248, + "reward_std": 0.3433863129466772, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2125584715977311, + "step": 297 + }, + { + "clip_fraction": 0.0, + "completion_length": 2834.479202270508, + "epoch": 0.3405714285714286, + "grad_norm": 0.07559877634048462, + "kl": 1.9371509552001953e-05, + "lambda_div_used": 0.5950812250375748, + "learning_rate": 4.811563736721829e-07, + "loss": -0.0217, + "reward": -0.13096300419420004, + "reward_after_mean": -0.13096300419420004, + "reward_after_std": 0.505608232691884, + "reward_before_mean": 0.22305661533027887, + "reward_before_std": 0.43375879526138306, + "reward_change_max": 0.0, + "reward_change_mean": -0.354019645601511, + "reward_change_min": -0.5659407489001751, + "reward_change_std": 0.20670694950968027, + "reward_std": 0.5056082457304001, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": -0.0686100497841835, + "step": 298 + }, + { + "clip_fraction": 0.0, + "completion_length": 3288.5833587646484, + "epoch": 0.3417142857142857, + "grad_norm": 0.05182049795985222, + "kl": 3.013480454683304e-05, + "lambda_div_used": 0.5971293970942497, + "learning_rate": 4.780534655386743e-07, + "loss": -0.0035, + "reward": -0.1649590004235506, + "reward_after_mean": -0.1649590004235506, + "reward_after_std": 0.476910138502717, + "reward_before_mean": 0.15552489459514618, + "reward_before_std": 0.44229450821876526, + "reward_change_max": 0.0, + "reward_change_mean": -0.3204838838428259, + "reward_change_min": -0.5031619034707546, + "reward_change_std": 0.19439208041876554, + "reward_std": 0.47691015154123306, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.052808452397584915, + "step": 299 + }, + { + "clip_fraction": 0.0, + "completion_length": 3455.3958435058594, + "epoch": 0.34285714285714286, + "grad_norm": 0.059302717447280884, + "kl": 3.039836883544922e-05, + "lambda_div_used": 0.5659952610731125, + "learning_rate": 4.749540639777539e-07, + "loss": 0.0631, + "reward": -0.4138263203203678, + "reward_after_mean": -0.4138263203203678, + "reward_after_std": 0.35950295627117157, + "reward_before_mean": -0.14833886176347733, + "reward_before_std": 0.30170741491019726, + "reward_change_max": 0.0, + "reward_change_mean": -0.26548744924366474, + "reward_change_min": -0.45029690116643906, + "reward_change_std": 0.16043098457157612, + "reward_std": 0.35950295627117157, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.19000552780926228, + "step": 300 + }, + { + "clip_fraction": 0.0, + "completion_length": 2581.2917404174805, + "epoch": 0.344, + "grad_norm": 0.0829557478427887, + "kl": 1.7702579498291016e-05, + "lambda_div_used": 0.6020706444978714, + "learning_rate": 4.7185832004988133e-07, + "loss": 0.0033, + "reward": -0.2905968725681305, + "reward_after_mean": -0.2905968725681305, + "reward_after_std": 0.522302333265543, + "reward_before_mean": -0.03232934419065714, + "reward_before_std": 0.47291796933859587, + "reward_change_max": 0.0, + "reward_change_mean": -0.2582675274461508, + "reward_change_min": -0.4681916609406471, + "reward_change_std": 0.16694430727511644, + "reward_std": 0.5223023407161236, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.13649601582437754, + "step": 301 + }, + { + "clip_fraction": 0.0, + "completion_length": 2299.250026702881, + "epoch": 0.34514285714285714, + "grad_norm": 0.09259702265262604, + "kl": 2.473965287208557e-05, + "lambda_div_used": 0.6064160838723183, + "learning_rate": 4.68766384637248e-07, + "loss": 0.0049, + "reward": 0.1023632986471057, + "reward_after_mean": 0.1023632986471057, + "reward_after_std": 0.5881610047072172, + "reward_before_mean": 0.5643375236541033, + "reward_before_std": 0.48729276517406106, + "reward_change_max": 0.0, + "reward_change_mean": -0.46197423338890076, + "reward_change_min": -0.6793596595525742, + "reward_change_std": 0.2643775464966893, + "reward_std": 0.588161014020443, + "rewards/accuracy_reward": 0.39583334513008595, + "rewards/cosine_scaled_reward": 0.16850417526438832, + "step": 302 + }, + { + "clip_fraction": 0.0, + "completion_length": 2496.604202270508, + "epoch": 0.3462857142857143, + "grad_norm": 0.10342813283205032, + "kl": 2.822279930114746e-05, + "lambda_div_used": 0.6173663139343262, + "learning_rate": 4.656784084364238e-07, + "loss": -0.0508, + "reward": -0.21569269057363272, + "reward_after_mean": -0.21569269057363272, + "reward_after_std": 0.5887450613081455, + "reward_before_mean": 0.048040480352938175, + "reward_before_std": 0.5433634500950575, + "reward_change_max": 0.0, + "reward_change_mean": -0.2637331634759903, + "reward_change_min": -0.4554433934390545, + "reward_change_std": 0.17205023765563965, + "reward_std": 0.5887450724840164, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.09779285965487361, + "step": 303 + }, + { + "clip_fraction": 0.0, + "completion_length": 2828.375030517578, + "epoch": 0.3474285714285714, + "grad_norm": 0.06273287534713745, + "kl": 3.515370190143585e-05, + "lambda_div_used": 0.5850896239280701, + "learning_rate": 4.6259454195101267e-07, + "loss": 0.0395, + "reward": -0.351226020604372, + "reward_after_mean": -0.351226020604372, + "reward_after_std": 0.4407376032322645, + "reward_before_mean": -0.091004628688097, + "reward_before_std": 0.39247662480920553, + "reward_change_max": 0.0, + "reward_change_mean": -0.2602214030921459, + "reward_change_min": -0.4692201763391495, + "reward_change_std": 0.16711712814867496, + "reward_std": 0.4407376106828451, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.1743379645049572, + "step": 304 + }, + { + "clip_fraction": 0.0, + "completion_length": 2963.270835876465, + "epoch": 0.3485714285714286, + "grad_norm": 0.07068406045436859, + "kl": 3.521144390106201e-05, + "lambda_div_used": 0.5555498078465462, + "learning_rate": 4.59514935484316e-07, + "loss": 0.0077, + "reward": -0.44899775832891464, + "reward_after_mean": -0.44899775832891464, + "reward_after_std": 0.32930242642760277, + "reward_before_mean": -0.17746215965598822, + "reward_before_std": 0.2488851365633309, + "reward_change_max": 0.0, + "reward_change_mean": -0.27153559774160385, + "reward_change_min": -0.41173194721341133, + "reward_change_std": 0.14773181919008493, + "reward_std": 0.32930243387818336, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.19829548941925168, + "step": 305 + }, + { + "clip_fraction": 0.0, + "completion_length": 2542.8750228881836, + "epoch": 0.3497142857142857, + "grad_norm": 0.09160702675580978, + "kl": 2.777576446533203e-05, + "lambda_div_used": 0.5710241869091988, + "learning_rate": 4.5643973913200837e-07, + "loss": 0.001, + "reward": -0.28438286297023296, + "reward_after_mean": -0.28438286297023296, + "reward_after_std": 0.45469350554049015, + "reward_before_mean": 0.05048683221684769, + "reward_before_std": 0.3197702756151557, + "reward_change_max": 0.0, + "reward_change_mean": -0.33486970886588097, + "reward_change_min": -0.46306542679667473, + "reward_change_std": 0.1733616916462779, + "reward_std": 0.45469350926578045, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.11617984622716904, + "step": 306 + }, + { + "clip_fraction": 0.0, + "completion_length": 2358.9167098999023, + "epoch": 0.35085714285714287, + "grad_norm": 0.08308543264865875, + "kl": 2.5270506739616394e-05, + "lambda_div_used": 0.6283661872148514, + "learning_rate": 4.5336910277482155e-07, + "loss": 0.0233, + "reward": -0.023870151489973068, + "reward_after_mean": -0.023870151489973068, + "reward_after_std": 0.5968872811645269, + "reward_before_mean": 0.2986760139465332, + "reward_before_std": 0.602552474476397, + "reward_change_max": 0.0, + "reward_change_mean": -0.3225461672991514, + "reward_change_min": -0.5788566246628761, + "reward_change_std": 0.22817020770162344, + "reward_std": 0.5968873165547848, + "rewards/accuracy_reward": 0.2916666716337204, + "rewards/cosine_scaled_reward": 0.007009351626038551, + "step": 307 + }, + { + "clip_fraction": 0.0, + "completion_length": 3364.187530517578, + "epoch": 0.352, + "grad_norm": 0.04729427769780159, + "kl": 1.2226402759552002e-05, + "lambda_div_used": 0.5575956255197525, + "learning_rate": 4.503031760712397e-07, + "loss": -0.0147, + "reward": -0.364587739109993, + "reward_after_mean": -0.364587739109993, + "reward_after_std": 0.3172164801508188, + "reward_before_mean": -0.06630371138453484, + "reward_before_std": 0.25998193118721247, + "reward_change_max": 0.0, + "reward_change_mean": -0.29828402772545815, + "reward_change_min": -0.4606154337525368, + "reward_change_std": 0.17100436985492706, + "reward_std": 0.31721648946404457, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.14963705092668533, + "step": 308 + }, + { + "clip_fraction": 0.0, + "completion_length": 3246.750030517578, + "epoch": 0.35314285714285715, + "grad_norm": 0.054204076528549194, + "kl": 1.4858320355415344e-05, + "lambda_div_used": 0.6364770829677582, + "learning_rate": 4.4724210845020494e-07, + "loss": 0.0936, + "reward": -0.09930397570133209, + "reward_after_mean": -0.09930397570133209, + "reward_after_std": 0.6563027147203684, + "reward_before_mean": 0.18065341375768185, + "reward_before_std": 0.6305609010159969, + "reward_change_max": 0.0, + "reward_change_mean": -0.27995740808546543, + "reward_change_min": -0.5127242244780064, + "reward_change_std": 0.19019456766545773, + "reward_std": 0.6563027296215296, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.027679930441081524, + "step": 309 + }, + { + "clip_fraction": 0.0, + "completion_length": 2454.750015258789, + "epoch": 0.35428571428571426, + "grad_norm": 0.10188445448875427, + "kl": 3.9263395592570305e-05, + "lambda_div_used": 0.5863905549049377, + "learning_rate": 4.441860491038345e-07, + "loss": -0.0091, + "reward": -0.358197920024395, + "reward_after_mean": -0.358197920024395, + "reward_after_std": 0.4752412661910057, + "reward_before_mean": -0.10783447185531259, + "reward_before_std": 0.39106855262070894, + "reward_change_max": 0.0, + "reward_change_mean": -0.25036344304680824, + "reward_change_min": -0.36833222955465317, + "reward_change_std": 0.13517011515796185, + "reward_std": 0.47524126805365086, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.17033447965513915, + "step": 310 + }, + { + "clip_fraction": 0.0, + "completion_length": 2549.5000228881836, + "epoch": 0.3554285714285714, + "grad_norm": 0.07070475816726685, + "kl": 1.1476688086986542e-05, + "lambda_div_used": 0.6028245091438293, + "learning_rate": 4.4113514698014953e-07, + "loss": -0.0234, + "reward": -0.07257996033877134, + "reward_after_mean": -0.07257996033877134, + "reward_after_std": 0.5427698846906424, + "reward_before_mean": 0.32260693423449993, + "reward_before_std": 0.4735551681369543, + "reward_change_max": 0.0, + "reward_change_mean": -0.3951868824660778, + "reward_change_min": -0.6274620294570923, + "reward_change_std": 0.2447280865162611, + "reward_std": 0.5427699014544487, + "rewards/accuracy_reward": 0.3125000037252903, + "rewards/cosine_scaled_reward": 0.010106915608048439, + "step": 311 + }, + { + "clip_fraction": 0.0, + "completion_length": 2062.020851135254, + "epoch": 0.3565714285714286, + "grad_norm": 0.0864594504237175, + "kl": 2.2163614630699158e-05, + "lambda_div_used": 0.5457132831215858, + "learning_rate": 4.3808955077581546e-07, + "loss": 0.0122, + "reward": -0.03406492620706558, + "reward_after_mean": -0.03406492620706558, + "reward_after_std": 0.3851391561329365, + "reward_before_mean": 0.5170513242483139, + "reward_before_std": 0.20092127658426762, + "reward_change_max": 0.0, + "reward_change_mean": -0.5511162541806698, + "reward_change_min": -0.7572819888591766, + "reward_change_std": 0.2855915669351816, + "reward_std": 0.38513917103409767, + "rewards/accuracy_reward": 0.375, + "rewards/cosine_scaled_reward": 0.1420513167977333, + "step": 312 + }, + { + "clip_fraction": 0.0, + "completion_length": 2939.958351135254, + "epoch": 0.3577142857142857, + "grad_norm": 0.06731264293193817, + "kl": 2.4283304810523987e-05, + "lambda_div_used": 0.5877447426319122, + "learning_rate": 4.350494089288943e-07, + "loss": -0.0439, + "reward": -0.15467195864766836, + "reward_after_mean": -0.15467195864766836, + "reward_after_std": 0.522945849224925, + "reward_before_mean": 0.20540180057287216, + "reward_before_std": 0.40051606576889753, + "reward_change_max": 0.0, + "reward_change_mean": -0.3600737862288952, + "reward_change_min": -0.5124829597771168, + "reward_change_std": 0.19857864920049906, + "reward_std": 0.5229458846151829, + "rewards/accuracy_reward": 0.2291666679084301, + "rewards/cosine_scaled_reward": -0.02376485476270318, + "step": 313 + }, + { + "clip_fraction": 0.0, + "completion_length": 2541.18754196167, + "epoch": 0.3588571428571429, + "grad_norm": 0.08012691885232925, + "kl": 3.474205732345581e-05, + "lambda_div_used": 0.607881672680378, + "learning_rate": 4.3201486961161093e-07, + "loss": 0.0168, + "reward": -0.20861644856631756, + "reward_after_mean": -0.20861644856631756, + "reward_after_std": 0.5089061111211777, + "reward_before_mean": 0.08002243563532829, + "reward_before_std": 0.49900877848267555, + "reward_change_max": 0.0, + "reward_change_mean": -0.28863888792693615, + "reward_change_min": -0.548176895827055, + "reward_change_std": 0.2009973768144846, + "reward_std": 0.5089061167091131, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.08664422971196473, + "step": 314 + }, + { + "clip_fraction": 0.0, + "completion_length": 3124.3333740234375, + "epoch": 0.36, + "grad_norm": 0.0558871328830719, + "kl": 1.9735191017389297e-05, + "lambda_div_used": 0.5734824016690254, + "learning_rate": 4.2898608072313045e-07, + "loss": 0.0494, + "reward": -0.1729298224672675, + "reward_after_mean": -0.1729298224672675, + "reward_after_std": 0.45045475475490093, + "reward_before_mean": 0.20758753083646297, + "reward_before_std": 0.33522335812449455, + "reward_change_max": 0.0, + "reward_change_mean": -0.3805173486471176, + "reward_change_min": -0.5440769977867603, + "reward_change_std": 0.20876472163945436, + "reward_std": 0.4504547640681267, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.02157914894632995, + "step": 315 + }, + { + "clip_fraction": 0.0, + "completion_length": 3509.5416870117188, + "epoch": 0.36114285714285715, + "grad_norm": 0.05331319198012352, + "kl": 2.981536090373993e-05, + "lambda_div_used": 0.5893311724066734, + "learning_rate": 4.2596318988235037e-07, + "loss": 0.0135, + "reward": -0.22396450489759445, + "reward_after_mean": -0.22396450489759445, + "reward_after_std": 0.43868801929056644, + "reward_before_mean": 0.07456053979694843, + "reward_before_std": 0.414919788017869, + "reward_change_max": 0.0, + "reward_change_mean": -0.29852502048015594, + "reward_change_min": -0.4823254942893982, + "reward_change_std": 0.1912739286199212, + "reward_std": 0.43868803791701794, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.09210613742470741, + "step": 316 + }, + { + "clip_fraction": 0.0, + "completion_length": 3099.5833435058594, + "epoch": 0.36228571428571427, + "grad_norm": 0.06200157478451729, + "kl": 3.091990947723389e-05, + "lambda_div_used": 0.5825675800442696, + "learning_rate": 4.2294634442070553e-07, + "loss": 0.0437, + "reward": -0.35854507237672806, + "reward_after_mean": -0.35854507237672806, + "reward_after_std": 0.43180895783007145, + "reward_before_mean": -0.102005859836936, + "reward_before_std": 0.3769839182496071, + "reward_change_max": 0.0, + "reward_change_mean": -0.25653921999037266, + "reward_change_min": -0.4170425795018673, + "reward_change_std": 0.15367608424276114, + "reward_std": 0.4318089634180069, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.18533920124173164, + "step": 317 + }, + { + "clip_fraction": 0.0, + "completion_length": 2281.000068664551, + "epoch": 0.36342857142857143, + "grad_norm": 0.0932127833366394, + "kl": 3.602728247642517e-05, + "lambda_div_used": 0.6223095878958702, + "learning_rate": 4.1993569137498776e-07, + "loss": 0.0631, + "reward": -0.07363937655463815, + "reward_after_mean": -0.07363937655463815, + "reward_after_std": 0.5691333152353764, + "reward_before_mean": 0.24711408838629723, + "reward_before_std": 0.5656038168817759, + "reward_change_max": 0.0, + "reward_change_mean": -0.3207534924149513, + "reward_change_min": -0.5814453661441803, + "reward_change_std": 0.2219030074775219, + "reward_std": 0.569133322685957, + "rewards/accuracy_reward": 0.2500000074505806, + "rewards/cosine_scaled_reward": -0.0028859074227511883, + "step": 318 + }, + { + "clip_fraction": 0.0, + "completion_length": 2854.5625076293945, + "epoch": 0.36457142857142855, + "grad_norm": 0.08494079858064651, + "kl": 2.7507543563842773e-05, + "lambda_div_used": 0.5380512997508049, + "learning_rate": 4.1693137748017915e-07, + "loss": -0.0389, + "reward": -0.4965253435075283, + "reward_after_mean": -0.4965253435075283, + "reward_after_std": 0.2333353590220213, + "reward_before_mean": -0.2120634987950325, + "reward_before_std": 0.16711975168436766, + "reward_change_max": 0.0, + "reward_change_mean": -0.28446184657514095, + "reward_change_min": -0.4124374948441982, + "reward_change_std": 0.15511877462267876, + "reward_std": 0.2333353627473116, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2120634950697422, + "step": 319 + }, + { + "clip_fraction": 0.0, + "completion_length": 1964.8958740234375, + "epoch": 0.3657142857142857, + "grad_norm": 0.09068436175584793, + "kl": 3.33394855260849e-05, + "lambda_div_used": 0.6160280704498291, + "learning_rate": 4.1393354916230005e-07, + "loss": 0.0117, + "reward": -0.19740011962130666, + "reward_after_mean": -0.19740011962130666, + "reward_after_std": 0.5963947810232639, + "reward_before_mean": 0.0693174353800714, + "reward_before_std": 0.5336382519453764, + "reward_change_max": 0.0, + "reward_change_mean": -0.26671755872666836, + "reward_change_min": -0.4143567681312561, + "reward_change_std": 0.15817437414079905, + "reward_std": 0.5963947977870703, + "rewards/accuracy_reward": 0.12500000186264515, + "rewards/cosine_scaled_reward": -0.05568256159313023, + "step": 320 + }, + { + "clip_fraction": 0.0, + "completion_length": 1974.8958778381348, + "epoch": 0.3668571428571429, + "grad_norm": 0.0693729817867279, + "kl": 1.574307680130005e-05, + "lambda_div_used": 0.6290735602378845, + "learning_rate": 4.1094235253127374e-07, + "loss": 0.0387, + "reward": 0.11023577488958836, + "reward_after_mean": 0.11023577488958836, + "reward_after_std": 0.6442763805389404, + "reward_before_mean": 0.5218545235693455, + "reward_before_std": 0.6013733670115471, + "reward_change_max": 0.0, + "reward_change_mean": -0.4116187300533056, + "reward_change_min": -0.6836260408163071, + "reward_change_std": 0.2705372450873256, + "reward_std": 0.6442763898521662, + "rewards/accuracy_reward": 0.3958333395421505, + "rewards/cosine_scaled_reward": 0.12602117005735636, + "step": 321 + }, + { + "clip_fraction": 0.0, + "completion_length": 2837.833381652832, + "epoch": 0.368, + "grad_norm": 0.10218925029039383, + "kl": 4.9501657485961914e-05, + "lambda_div_used": 0.6059064492583275, + "learning_rate": 4.079579333738039e-07, + "loss": -0.0178, + "reward": -0.2954628551378846, + "reward_after_mean": -0.2954628551378846, + "reward_after_std": 0.5665331184864044, + "reward_before_mean": -0.04816420469433069, + "reward_before_std": 0.48093545995652676, + "reward_change_max": 0.0, + "reward_change_mean": -0.24729863554239273, + "reward_change_min": -0.3516420107334852, + "reward_change_std": 0.13184416200965643, + "reward_std": 0.5665331222116947, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.13149754563346505, + "step": 322 + }, + { + "clip_fraction": 0.0, + "completion_length": 3015.666717529297, + "epoch": 0.36914285714285716, + "grad_norm": 0.0596209317445755, + "kl": 1.7877668142318726e-05, + "lambda_div_used": 0.5565166473388672, + "learning_rate": 4.0498043714627006e-07, + "loss": 0.0465, + "reward": -0.16177266091108322, + "reward_after_mean": -0.16177266091108322, + "reward_after_std": 0.42436218820512295, + "reward_before_mean": 0.2816486116498709, + "reward_before_std": 0.2578566027805209, + "reward_change_max": 0.0, + "reward_change_mean": -0.4434212874621153, + "reward_change_min": -0.6335049495100975, + "reward_change_std": 0.23841418512165546, + "reward_std": 0.4243622049689293, + "rewards/accuracy_reward": 0.2916666679084301, + "rewards/cosine_scaled_reward": -0.010018057189881802, + "step": 323 + }, + { + "clip_fraction": 0.0, + "completion_length": 2751.6458435058594, + "epoch": 0.3702857142857143, + "grad_norm": 0.10376841574907303, + "kl": 2.1306797862052917e-05, + "lambda_div_used": 0.5670785158872604, + "learning_rate": 4.020100089676376e-07, + "loss": -0.0038, + "reward": -0.10797288408502936, + "reward_after_mean": -0.10797288408502936, + "reward_after_std": 0.4300071895122528, + "reward_before_mean": 0.3295288155786693, + "reward_before_std": 0.3011532872915268, + "reward_change_max": 0.0, + "reward_change_mean": -0.43750171549618244, + "reward_change_min": -0.6185614429414272, + "reward_change_std": 0.2333526872098446, + "reward_std": 0.43000719882547855, + "rewards/accuracy_reward": 0.25000000558793545, + "rewards/cosine_scaled_reward": 0.07952881557866931, + "step": 324 + }, + { + "clip_fraction": 0.0, + "completion_length": 2943.8958587646484, + "epoch": 0.37142857142857144, + "grad_norm": 0.07239633053541183, + "kl": 9.275972843170166e-06, + "lambda_div_used": 0.6356078162789345, + "learning_rate": 3.9904679361238526e-07, + "loss": 0.0667, + "reward": -0.050798285752534866, + "reward_after_mean": -0.050798285752534866, + "reward_after_std": 0.6440786644816399, + "reward_before_mean": 0.24087253957986832, + "reward_before_std": 0.6331603992730379, + "reward_change_max": 0.0, + "reward_change_mean": -0.2916708290576935, + "reward_change_min": -0.4932599440217018, + "reward_change_std": 0.20057004038244486, + "reward_std": 0.6440786886960268, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.02996080555021763, + "step": 325 + }, + { + "clip_fraction": 0.0, + "completion_length": 2375.645851135254, + "epoch": 0.37257142857142855, + "grad_norm": 0.09299691766500473, + "kl": 2.9705464839935303e-05, + "lambda_div_used": 0.5330315083265305, + "learning_rate": 3.9609093550344907e-07, + "loss": -0.0979, + "reward": -0.20862240344285965, + "reward_after_mean": -0.20862240344285965, + "reward_after_std": 0.3296848703175783, + "reward_before_mean": 0.2517512815538794, + "reward_before_std": 0.1450797226279974, + "reward_change_max": 0.0, + "reward_change_mean": -0.460373692214489, + "reward_change_min": -0.6264396719634533, + "reward_change_std": 0.23280893173068762, + "reward_std": 0.3296848740428686, + "rewards/accuracy_reward": 0.25, + "rewards/cosine_scaled_reward": 0.001751287141814828, + "step": 326 + }, + { + "clip_fraction": 0.0, + "completion_length": 2810.187511444092, + "epoch": 0.3737142857142857, + "grad_norm": 0.06210591271519661, + "kl": 8.400529623031616e-06, + "lambda_div_used": 0.5783005133271217, + "learning_rate": 3.931425787051832e-07, + "loss": -0.0483, + "reward": -0.11793380603194237, + "reward_after_mean": -0.11793380603194237, + "reward_after_std": 0.45353105291724205, + "reward_before_mean": 0.2935123089700937, + "reward_before_std": 0.3566841436550021, + "reward_change_max": 0.0, + "reward_change_mean": -0.41144610941410065, + "reward_change_min": -0.6089312434196472, + "reward_change_std": 0.23675687983632088, + "reward_std": 0.4535310585051775, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": 0.022678975947201252, + "step": 327 + }, + { + "clip_fraction": 0.0, + "completion_length": 3518.8958435058594, + "epoch": 0.37485714285714283, + "grad_norm": 0.05192619562149048, + "kl": 2.485513687133789e-05, + "lambda_div_used": 0.5715365409851074, + "learning_rate": 3.902018669163384e-07, + "loss": 0.0155, + "reward": -0.4497411046177149, + "reward_after_mean": -0.4497411046177149, + "reward_after_std": 0.40177739411592484, + "reward_before_mean": -0.21567542850971222, + "reward_before_std": 0.32284149527549744, + "reward_change_max": 0.0, + "reward_change_mean": -0.23406569100916386, + "reward_change_min": -0.33814629539847374, + "reward_change_std": 0.12489900179207325, + "reward_std": 0.40177739597857, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2573420889675617, + "step": 328 + }, + { + "clip_fraction": 0.0, + "completion_length": 2243.0416946411133, + "epoch": 0.376, + "grad_norm": 0.10950610786676407, + "kl": 2.469681203365326e-05, + "lambda_div_used": 0.6176783889532089, + "learning_rate": 3.872689434630585e-07, + "loss": -0.0562, + "reward": -0.013319691643118858, + "reward_after_mean": -0.013319691643118858, + "reward_after_std": 0.5674807205796242, + "reward_before_mean": 0.3346351385116577, + "reward_before_std": 0.5435493532568216, + "reward_change_max": 0.0, + "reward_change_mean": -0.34795483760535717, + "reward_change_min": -0.5437430925667286, + "reward_change_std": 0.22110824659466743, + "reward_std": 0.5674807410687208, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": 0.06380179291591048, + "step": 329 + }, + { + "clip_fraction": 0.0, + "completion_length": 2141.479190826416, + "epoch": 0.37714285714285717, + "grad_norm": 0.10051169991493225, + "kl": 4.89354133605957e-05, + "lambda_div_used": 0.5594945177435875, + "learning_rate": 3.843439512918949e-07, + "loss": -0.0064, + "reward": -0.43674849811941385, + "reward_after_mean": -0.43674849811941385, + "reward_after_std": 0.32430145144462585, + "reward_before_mean": -0.17029727809131145, + "reward_before_std": 0.2681279256939888, + "reward_change_max": 0.0, + "reward_change_mean": -0.266451220959425, + "reward_change_min": -0.44791119545698166, + "reward_change_std": 0.1575082140043378, + "reward_std": 0.32430145516991615, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.2119639493757859, + "step": 330 + }, + { + "clip_fraction": 0.0, + "completion_length": 2564.604202270508, + "epoch": 0.3782857142857143, + "grad_norm": 0.12382425367832184, + "kl": 4.225596785545349e-05, + "lambda_div_used": 0.5574543103575706, + "learning_rate": 3.8142703296283953e-07, + "loss": 0.0413, + "reward": -0.41342686861753464, + "reward_after_mean": -0.41342686861753464, + "reward_after_std": 0.33145094849169254, + "reward_before_mean": -0.13134576752781868, + "reward_before_std": 0.25930391903966665, + "reward_change_max": 0.0, + "reward_change_mean": -0.2820810880511999, + "reward_change_min": -0.4496995583176613, + "reward_change_std": 0.16215670108795166, + "reward_std": 0.33145095966756344, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.17301243357360363, + "step": 331 + }, + { + "clip_fraction": 0.0, + "completion_length": 2420.6666870117188, + "epoch": 0.37942857142857145, + "grad_norm": 0.07938601076602936, + "kl": 2.1241605281829834e-05, + "lambda_div_used": 0.5416555106639862, + "learning_rate": 3.785183306423767e-07, + "loss": 0.0287, + "reward": -0.21761326864361763, + "reward_after_mean": -0.21761326864361763, + "reward_after_std": 0.34953613951802254, + "reward_before_mean": 0.2352729644626379, + "reward_before_std": 0.18298226408660412, + "reward_change_max": 0.0, + "reward_change_mean": -0.45288625732064247, + "reward_change_min": -0.6215734221041203, + "reward_change_std": 0.2345103258267045, + "reward_std": 0.34953614324331284, + "rewards/accuracy_reward": 0.25, + "rewards/cosine_scaled_reward": -0.014727018773555756, + "step": 332 + }, + { + "clip_fraction": 0.0, + "completion_length": 2350.6875381469727, + "epoch": 0.38057142857142856, + "grad_norm": 0.08657004684209824, + "kl": 2.1725893020629883e-05, + "lambda_div_used": 0.6729468256235123, + "learning_rate": 3.7561798609655373e-07, + "loss": 0.0369, + "reward": -0.01155824027955532, + "reward_after_mean": -0.01155824027955532, + "reward_after_std": 0.8104779217392206, + "reward_before_mean": 0.23989354074001312, + "reward_before_std": 0.8117707390338182, + "reward_change_max": 0.0, + "reward_change_mean": -0.2514517717063427, + "reward_change_min": -0.5040576457977295, + "reward_change_std": 0.19348772894591093, + "reward_std": 0.810477938503027, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.030939804390072823, + "step": 333 + }, + { + "clip_fraction": 0.0, + "completion_length": 3491.6041870117188, + "epoch": 0.38171428571428573, + "grad_norm": 0.05208956450223923, + "kl": 1.1973665095865726e-05, + "lambda_div_used": 0.5869207382202148, + "learning_rate": 3.72726140684072e-07, + "loss": -0.009, + "reward": -0.2944636158645153, + "reward_after_mean": -0.2944636158645153, + "reward_after_std": 0.43844909220933914, + "reward_before_mean": -0.013160821050405502, + "reward_before_std": 0.40495526185259223, + "reward_change_max": 0.0, + "reward_change_mean": -0.2813027612864971, + "reward_change_min": -0.4717176593840122, + "reward_change_std": 0.1835445323958993, + "reward_std": 0.4384490940719843, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.13816083781421185, + "step": 334 + }, + { + "clip_fraction": 0.0, + "completion_length": 2589.0833892822266, + "epoch": 0.38285714285714284, + "grad_norm": 0.0683097094297409, + "kl": 1.9595026969909668e-05, + "lambda_div_used": 0.6027273386716843, + "learning_rate": 3.6984293534939737e-07, + "loss": -0.0488, + "reward": 0.005996011197566986, + "reward_after_mean": 0.005996011197566986, + "reward_after_std": 0.5422599408775568, + "reward_before_mean": 0.41999348998069763, + "reward_before_std": 0.4701922629028559, + "reward_change_max": 0.0, + "reward_change_mean": -0.41399746760725975, + "reward_change_min": -0.623339332640171, + "reward_change_std": 0.24341293703764677, + "reward_std": 0.5422599650919437, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.08666014112532139, + "step": 335 + }, + { + "clip_fraction": 0.0, + "completion_length": 3162.1458740234375, + "epoch": 0.384, + "grad_norm": 0.067026786506176, + "kl": 2.0024715922772884e-05, + "lambda_div_used": 0.5950900241732597, + "learning_rate": 3.6696851061588994e-07, + "loss": 0.0138, + "reward": -0.16571878641843796, + "reward_after_mean": -0.16571878641843796, + "reward_after_std": 0.4802239239215851, + "reward_before_mean": 0.14471458829939365, + "reward_before_std": 0.4368621027097106, + "reward_change_max": 0.0, + "reward_change_mean": -0.31043340265750885, + "reward_change_min": -0.45523249730467796, + "reward_change_std": 0.18632046319544315, + "reward_std": 0.48022393323481083, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.06361875729635358, + "step": 336 + }, + { + "clip_fraction": 0.0, + "completion_length": 3228.916717529297, + "epoch": 0.3851428571428571, + "grad_norm": 0.05528295040130615, + "kl": 1.9825994968414307e-05, + "lambda_div_used": 0.6006625667214394, + "learning_rate": 3.641030065789562e-07, + "loss": 0.0552, + "reward": -0.2433483125641942, + "reward_after_mean": -0.2433483125641942, + "reward_after_std": 0.5013067033141851, + "reward_before_mean": 0.03825543075799942, + "reward_before_std": 0.4624515902251005, + "reward_change_max": 0.0, + "reward_change_mean": -0.28160375356674194, + "reward_change_min": -0.4847262054681778, + "reward_change_std": 0.17836903873831034, + "reward_std": 0.501306714490056, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.10757789574563503, + "step": 337 + }, + { + "clip_fraction": 0.0, + "completion_length": 2393.645866394043, + "epoch": 0.3862857142857143, + "grad_norm": 0.10292567312717438, + "kl": 3.5159289836883545e-05, + "lambda_div_used": 0.6184682846069336, + "learning_rate": 3.612465628992203e-07, + "loss": 0.1082, + "reward": -0.010080082342028618, + "reward_after_mean": -0.010080082342028618, + "reward_after_std": 0.6298389285802841, + "reward_before_mean": 0.3685674872249365, + "reward_before_std": 0.546540604904294, + "reward_change_max": 0.0, + "reward_change_mean": -0.3786476030945778, + "reward_change_min": -0.5825354494154453, + "reward_change_std": 0.2281006295233965, + "reward_std": 0.6298389509320259, + "rewards/accuracy_reward": 0.3125000037252903, + "rewards/cosine_scaled_reward": 0.05606749979779124, + "step": 338 + }, + { + "clip_fraction": 0.0, + "completion_length": 3026.9583892822266, + "epoch": 0.38742857142857146, + "grad_norm": 0.0688738226890564, + "kl": 2.4460256099700928e-05, + "lambda_div_used": 0.5621443763375282, + "learning_rate": 3.5839931879571725e-07, + "loss": -0.0178, + "reward": -0.32898029685020447, + "reward_after_mean": -0.32898029685020447, + "reward_after_std": 0.3227359801530838, + "reward_before_mean": -0.01589403674006462, + "reward_before_std": 0.2781273443251848, + "reward_change_max": 0.0, + "reward_change_mean": -0.31308628246188164, + "reward_change_min": -0.48185204714536667, + "reward_change_std": 0.1813461184501648, + "reward_std": 0.3227359913289547, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.12006070092320442, + "step": 339 + }, + { + "clip_fraction": 0.0, + "completion_length": 2447.937515258789, + "epoch": 0.38857142857142857, + "grad_norm": 0.10290948301553726, + "kl": 2.363821113249287e-05, + "lambda_div_used": 0.5592405423521996, + "learning_rate": 3.555614130391079e-07, + "loss": -0.0046, + "reward": -0.42997122183442116, + "reward_after_mean": -0.42997122183442116, + "reward_after_std": 0.33643546886742115, + "reward_before_mean": -0.1512418081983924, + "reward_before_std": 0.2663228642195463, + "reward_change_max": 0.0, + "reward_change_mean": -0.2787294201552868, + "reward_change_min": -0.4129678010940552, + "reward_change_std": 0.1523351836949587, + "reward_std": 0.3364354781806469, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.17207514215260744, + "step": 340 + }, + { + "clip_fraction": 0.0, + "completion_length": 2488.9583587646484, + "epoch": 0.38971428571428574, + "grad_norm": 0.061042170971632004, + "kl": 1.909211277961731e-05, + "lambda_div_used": 0.5834595933556557, + "learning_rate": 3.5273298394491515e-07, + "loss": -0.0023, + "reward": 0.009830114664509892, + "reward_after_mean": 0.009830114664509892, + "reward_after_std": 0.5199873205274343, + "reward_before_mean": 0.4715769328176975, + "reward_before_std": 0.38039534725248814, + "reward_change_max": 0.0, + "reward_change_mean": -0.46174679696559906, + "reward_change_min": -0.6357490979135036, + "reward_change_std": 0.2529716519638896, + "reward_std": 0.5199873335659504, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.13824361143633723, + "step": 341 + }, + { + "clip_fraction": 0.0, + "completion_length": 3020.520854949951, + "epoch": 0.39085714285714285, + "grad_norm": 0.07325860857963562, + "kl": 2.4762004613876343e-05, + "lambda_div_used": 0.5985761880874634, + "learning_rate": 3.4991416936678276e-07, + "loss": 0.0007, + "reward": -0.06264204811304808, + "reward_after_mean": -0.06264204811304808, + "reward_after_std": 0.5387034490704536, + "reward_before_mean": 0.33080895524472, + "reward_before_std": 0.45346040464937687, + "reward_change_max": 0.0, + "reward_change_mean": -0.3934510052204132, + "reward_change_min": -0.5952773988246918, + "reward_change_std": 0.23232300952076912, + "reward_std": 0.5387034583836794, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.03914228640496731, + "step": 342 + }, + { + "clip_fraction": 0.0, + "completion_length": 3405.1875610351562, + "epoch": 0.392, + "grad_norm": 0.04665439575910568, + "kl": 2.7433037757873535e-05, + "lambda_div_used": 0.6150833070278168, + "learning_rate": 3.471051066897562e-07, + "loss": 0.0509, + "reward": -0.1582264108583331, + "reward_after_mean": -0.1582264108583331, + "reward_after_std": 0.5790915079414845, + "reward_before_mean": 0.13680532574653625, + "reward_before_std": 0.5264813583344221, + "reward_change_max": 0.0, + "reward_change_mean": -0.29503174126148224, + "reward_change_min": -0.4708465002477169, + "reward_change_std": 0.1767220702022314, + "reward_std": 0.5790915191173553, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.050694676116108894, + "step": 343 + }, + { + "clip_fraction": 0.0, + "completion_length": 2360.875045776367, + "epoch": 0.3931428571428571, + "grad_norm": 0.07271434366703033, + "kl": 1.041218638420105e-05, + "lambda_div_used": 0.5770246461033821, + "learning_rate": 3.4430593282358777e-07, + "loss": -0.0008, + "reward": 0.04454847797751427, + "reward_after_mean": 0.04454847797751427, + "reward_after_std": 0.5018230397254229, + "reward_before_mean": 0.5492894258350134, + "reward_before_std": 0.35145203582942486, + "reward_change_max": 0.0, + "reward_change_mean": -0.5047409404069185, + "reward_change_min": -0.7455123476684093, + "reward_change_std": 0.2790954224765301, + "reward_std": 0.5018230620771646, + "rewards/accuracy_reward": 0.4375, + "rewards/cosine_scaled_reward": 0.1117894072085619, + "step": 344 + }, + { + "clip_fraction": 0.0, + "completion_length": 3030.0833740234375, + "epoch": 0.3942857142857143, + "grad_norm": 0.0703606829047203, + "kl": 3.1925737857818604e-05, + "lambda_div_used": 0.5804503262042999, + "learning_rate": 3.4151678419606233e-07, + "loss": 0.0646, + "reward": -0.22463034093379974, + "reward_after_mean": -0.22463034093379974, + "reward_after_std": 0.41369511373341084, + "reward_before_mean": 0.09700941108167171, + "reward_before_std": 0.37086532823741436, + "reward_change_max": 0.0, + "reward_change_mean": -0.32163975574076176, + "reward_change_min": -0.5076877251267433, + "reward_change_std": 0.1956562791019678, + "reward_std": 0.4136951379477978, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.06965726800262928, + "step": 345 + }, + { + "clip_fraction": 0.0, + "completion_length": 3186.6458740234375, + "epoch": 0.3954285714285714, + "grad_norm": 0.05292130261659622, + "kl": 1.4215707778930664e-05, + "lambda_div_used": 0.5958857163786888, + "learning_rate": 3.387377967463493e-07, + "loss": 0.0331, + "reward": -0.24454469978809357, + "reward_after_mean": -0.24454469978809357, + "reward_after_std": 0.48920249938964844, + "reward_before_mean": 0.04633236164227128, + "reward_before_std": 0.44041235372424126, + "reward_change_max": 0.0, + "reward_change_mean": -0.29087705351412296, + "reward_change_min": -0.4697858430445194, + "reward_change_std": 0.179568306542933, + "reward_std": 0.4892025087028742, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.12033431546296924, + "step": 346 + }, + { + "clip_fraction": 0.0, + "completion_length": 3055.4583587646484, + "epoch": 0.3965714285714286, + "grad_norm": 0.07354738563299179, + "kl": 6.0675665736198425e-06, + "lambda_div_used": 0.5582710355520248, + "learning_rate": 3.359691059183761e-07, + "loss": 0.0132, + "reward": -0.42621116526424885, + "reward_after_mean": -0.42621116526424885, + "reward_after_std": 0.3322943150997162, + "reward_before_mean": -0.15516536496579647, + "reward_before_std": 0.2633455842733383, + "reward_change_max": 0.0, + "reward_change_mean": -0.2710457965731621, + "reward_change_min": -0.4307188205420971, + "reward_change_std": 0.15532648842781782, + "reward_std": 0.33229432441294193, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.19683203659951687, + "step": 347 + }, + { + "clip_fraction": 0.0, + "completion_length": 2852.4583435058594, + "epoch": 0.3977142857142857, + "grad_norm": 0.09605983644723892, + "kl": 3.167241811752319e-05, + "lambda_div_used": 0.5788434967398643, + "learning_rate": 3.3321084665422803e-07, + "loss": -0.0017, + "reward": -0.2087371125817299, + "reward_after_mean": -0.2087371125817299, + "reward_after_std": 0.40125221759080887, + "reward_before_mean": 0.12491290923207998, + "reward_before_std": 0.36025606002658606, + "reward_change_max": 0.0, + "reward_change_mean": -0.3336500097066164, + "reward_change_min": -0.5146553069353104, + "reward_change_std": 0.20004253275692463, + "reward_std": 0.40125224366784096, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.04175376985222101, + "step": 348 + }, + { + "clip_fraction": 0.0, + "completion_length": 2964.875015258789, + "epoch": 0.39885714285714285, + "grad_norm": 0.06869405508041382, + "kl": 1.731887459754944e-05, + "lambda_div_used": 0.5606642514467239, + "learning_rate": 3.3046315338757026e-07, + "loss": 0.0031, + "reward": -0.2807905152440071, + "reward_after_mean": -0.2807905152440071, + "reward_after_std": 0.39816635474562645, + "reward_before_mean": 0.08202904835343361, + "reward_before_std": 0.2763519547879696, + "reward_change_max": 0.0, + "reward_change_mean": -0.36281956918537617, + "reward_change_min": -0.54488330706954, + "reward_change_std": 0.20150058157742023, + "reward_std": 0.39816636219620705, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.0846376121044159, + "step": 349 + }, + { + "clip_fraction": 0.0, + "completion_length": 2521.9791870117188, + "epoch": 0.4, + "grad_norm": 0.10303740203380585, + "kl": 2.6671215891838074e-05, + "lambda_div_used": 0.6121688261628151, + "learning_rate": 3.2772616003709616e-07, + "loss": -0.0711, + "reward": -0.04820730444043875, + "reward_after_mean": -0.04820730444043875, + "reward_after_std": 0.5604431573301554, + "reward_before_mean": 0.287683189380914, + "reward_before_std": 0.5214535812847316, + "reward_change_max": 0.0, + "reward_change_mean": -0.3358904607594013, + "reward_change_min": -0.5252225995063782, + "reward_change_std": 0.21241459622979164, + "reward_std": 0.5604431666433811, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": 0.016849845182150602, + "step": 350 + }, + { + "clip_fraction": 0.0, + "completion_length": 3268.250030517578, + "epoch": 0.40114285714285713, + "grad_norm": 0.04833042621612549, + "kl": 2.4568289518356323e-05, + "lambda_div_used": 0.5601532310247421, + "learning_rate": 3.250000000000001e-07, + "loss": 0.0519, + "reward": -0.250777292996645, + "reward_after_mean": -0.250777292996645, + "reward_after_std": 0.38171103224158287, + "reward_before_mean": 0.1175131555646658, + "reward_before_std": 0.2718004435300827, + "reward_change_max": 0.0, + "reward_change_mean": -0.36829047463834286, + "reward_change_min": -0.5422449931502342, + "reward_change_std": 0.20415859669446945, + "reward_std": 0.3817110415548086, + "rewards/accuracy_reward": 0.1875, + "rewards/cosine_scaled_reward": -0.06998682580888271, + "step": 351 + }, + { + "clip_fraction": 0.0, + "completion_length": 2589.333366394043, + "epoch": 0.4022857142857143, + "grad_norm": 0.06552080065011978, + "kl": 1.905485987663269e-05, + "lambda_div_used": 0.6104472130537033, + "learning_rate": 3.222848061454764e-07, + "loss": 0.0025, + "reward": -0.09347447147592902, + "reward_after_mean": -0.09347447147592902, + "reward_after_std": 0.6156985405832529, + "reward_before_mean": 0.26158976616716245, + "reward_before_std": 0.5061299707740545, + "reward_change_max": 0.0, + "reward_change_mean": -0.3550642393529415, + "reward_change_min": -0.5232127867639065, + "reward_change_std": 0.1957317991182208, + "reward_std": 0.6156985703855753, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": 0.01158975763246417, + "step": 352 + }, + { + "clip_fraction": 0.0, + "completion_length": 2533.7916946411133, + "epoch": 0.4034285714285714, + "grad_norm": 0.07949981093406677, + "kl": 4.393863491714001e-05, + "lambda_div_used": 0.6204324588179588, + "learning_rate": 3.195807108082429e-07, + "loss": 0.0456, + "reward": -0.12379613053053617, + "reward_after_mean": -0.12379613053053617, + "reward_after_std": 0.6454745382070541, + "reward_before_mean": 0.203694608528167, + "reward_before_std": 0.5591385969892144, + "reward_change_max": 0.0, + "reward_change_mean": -0.327490733936429, + "reward_change_min": -0.5519858598709106, + "reward_change_std": 0.1984806014224887, + "reward_std": 0.6454745400696993, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": -0.0463053984567523, + "step": 353 + }, + { + "clip_fraction": 0.0, + "completion_length": 1914.562515258789, + "epoch": 0.4045714285714286, + "grad_norm": 0.07853464037179947, + "kl": 2.3346394300460815e-05, + "lambda_div_used": 0.5937526449561119, + "learning_rate": 3.168878457820915e-07, + "loss": -0.0345, + "reward": 0.036306386813521385, + "reward_after_mean": 0.036306386813521385, + "reward_after_std": 0.5187021996825933, + "reward_before_mean": 0.4854283039458096, + "reward_before_std": 0.42800275422632694, + "reward_change_max": 0.0, + "reward_change_mean": -0.4491219110786915, + "reward_change_min": -0.637521255761385, + "reward_change_std": 0.252533134073019, + "reward_std": 0.5187022183090448, + "rewards/accuracy_reward": 0.33333334140479565, + "rewards/cosine_scaled_reward": 0.15209495089948177, + "step": 354 + }, + { + "clip_fraction": 0.0, + "completion_length": 2335.8958740234375, + "epoch": 0.4057142857142857, + "grad_norm": 0.09039249271154404, + "kl": 2.7535483241081238e-05, + "lambda_div_used": 0.6392721608281136, + "learning_rate": 3.142063423134644e-07, + "loss": 0.0158, + "reward": 0.017912205308675766, + "reward_after_mean": 0.017912205308675766, + "reward_after_std": 0.6705970745533705, + "reward_before_mean": 0.355070261284709, + "reward_before_std": 0.6396346259862185, + "reward_change_max": 0.0, + "reward_change_mean": -0.33715808019042015, + "reward_change_min": -0.5581717267632484, + "reward_change_std": 0.21758130192756653, + "reward_std": 0.6705970987677574, + "rewards/accuracy_reward": 0.354166679084301, + "rewards/cosine_scaled_reward": 0.0009035973343998194, + "step": 355 + }, + { + "clip_fraction": 0.0, + "completion_length": 2585.4791870117188, + "epoch": 0.40685714285714286, + "grad_norm": 0.06332956254482269, + "kl": 1.6089528799057007e-05, + "lambda_div_used": 0.6472217217087746, + "learning_rate": 3.115363310950578e-07, + "loss": -0.0329, + "reward": -0.00402236171066761, + "reward_after_mean": -0.00402236171066761, + "reward_after_std": 0.6937260664999485, + "reward_before_mean": 0.2978215580806136, + "reward_before_std": 0.6849680617451668, + "reward_change_max": 0.0, + "reward_change_mean": -0.3018439058214426, + "reward_change_min": -0.4991762898862362, + "reward_change_std": 0.2024542335420847, + "reward_std": 0.6937260907143354, + "rewards/accuracy_reward": 0.2916666753590107, + "rewards/cosine_scaled_reward": 0.00615486316382885, + "step": 356 + }, + { + "clip_fraction": 0.0, + "completion_length": 3264.8125610351562, + "epoch": 0.408, + "grad_norm": 0.057182345539331436, + "kl": 2.91336327791214e-05, + "lambda_div_used": 0.5862653851509094, + "learning_rate": 3.0887794225945143e-07, + "loss": 0.0104, + "reward": -0.31888771802186966, + "reward_after_mean": -0.31888771802186966, + "reward_after_std": 0.43322005309164524, + "reward_before_mean": -0.049007685855031013, + "reward_before_std": 0.39840223640203476, + "reward_change_max": 0.0, + "reward_change_mean": -0.2698800265789032, + "reward_change_min": -0.48054200410842896, + "reward_change_std": 0.1758509548380971, + "reward_std": 0.43322005309164524, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.1531743509694934, + "step": 357 + }, + { + "clip_fraction": 0.0, + "completion_length": 3050.2500534057617, + "epoch": 0.40914285714285714, + "grad_norm": 0.05330098420381546, + "kl": 2.4370267055928707e-05, + "lambda_div_used": 0.6431510671973228, + "learning_rate": 3.062313053727671e-07, + "loss": 0.003, + "reward": -0.07201657444238663, + "reward_after_mean": -0.07201657444238663, + "reward_after_std": 0.688179362565279, + "reward_before_mean": 0.2027557131368667, + "reward_before_std": 0.6673171781003475, + "reward_change_max": 0.0, + "reward_change_mean": -0.27477228827774525, + "reward_change_min": -0.4944618083536625, + "reward_change_std": 0.18934499006718397, + "reward_std": 0.6881793700158596, + "rewards/accuracy_reward": 0.2291666716337204, + "rewards/cosine_scaled_reward": -0.026410957798361778, + "step": 358 + }, + { + "clip_fraction": 0.0, + "completion_length": 2671.937530517578, + "epoch": 0.4102857142857143, + "grad_norm": 0.07592163234949112, + "kl": 3.884732723236084e-05, + "lambda_div_used": 0.5795001536607742, + "learning_rate": 3.0359654942835247e-07, + "loss": 0.0464, + "reward": -0.2641681991517544, + "reward_after_mean": -0.2641681991517544, + "reward_after_std": 0.4112328961491585, + "reward_before_mean": 0.03885669261217117, + "reward_before_std": 0.36149344593286514, + "reward_change_max": 0.0, + "reward_change_mean": -0.30302487686276436, + "reward_change_min": -0.4962599165737629, + "reward_change_std": 0.18252001702785492, + "reward_std": 0.4112329035997391, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.10697665251791477, + "step": 359 + }, + { + "clip_fraction": 0.0, + "completion_length": 2798.937530517578, + "epoch": 0.4114285714285714, + "grad_norm": 0.06665439158678055, + "kl": 3.1637027859687805e-05, + "lambda_div_used": 0.6062901616096497, + "learning_rate": 3.0097380284049523e-07, + "loss": -0.0029, + "reward": -0.0033570118248462677, + "reward_after_mean": -0.0033570118248462677, + "reward_after_std": 0.5593161657452583, + "reward_before_mean": 0.4031880460679531, + "reward_before_std": 0.4958424214273691, + "reward_change_max": 0.0, + "reward_change_mean": -0.40654509887099266, + "reward_change_min": -0.6473257802426815, + "reward_change_std": 0.25588439870625734, + "reward_std": 0.5593161787837744, + "rewards/accuracy_reward": 0.3333333358168602, + "rewards/cosine_scaled_reward": 0.0698547288775444, + "step": 360 + }, + { + "clip_fraction": 0.0, + "completion_length": 3150.625030517578, + "epoch": 0.4125714285714286, + "grad_norm": 0.06971946358680725, + "kl": 2.9014074243605137e-05, + "lambda_div_used": 0.6219506114721298, + "learning_rate": 2.9836319343816397e-07, + "loss": 0.0048, + "reward": -0.11981392558664083, + "reward_after_mean": -0.11981392558664083, + "reward_after_std": 0.5868565142154694, + "reward_before_mean": 0.17182117886841297, + "reward_before_std": 0.564973471686244, + "reward_change_max": 0.0, + "reward_change_mean": -0.29163510724902153, + "reward_change_min": -0.5129627995193005, + "reward_change_std": 0.19524423126131296, + "reward_std": 0.5868565402925014, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.057345494627952576, + "step": 361 + }, + { + "clip_fraction": 0.0, + "completion_length": 1471.916706085205, + "epoch": 0.4137142857142857, + "grad_norm": 0.12450961768627167, + "kl": 2.4568289518356323e-05, + "lambda_div_used": 0.571638435125351, + "learning_rate": 2.9576484845877793e-07, + "loss": -0.0303, + "reward": 0.013313630130141973, + "reward_after_mean": 0.013313630130141973, + "reward_after_std": 0.4897063076496124, + "reward_before_mean": 0.505017813295126, + "reward_before_std": 0.32288316125050187, + "reward_change_max": 0.0, + "reward_change_mean": -0.4917041715234518, + "reward_change_min": -0.6615025624632835, + "reward_change_std": 0.25815817900002, + "reward_std": 0.4897063188254833, + "rewards/accuracy_reward": 0.37500000558793545, + "rewards/cosine_scaled_reward": 0.1300177900120616, + "step": 362 + }, + { + "clip_fraction": 0.0, + "completion_length": 2174.270881652832, + "epoch": 0.41485714285714287, + "grad_norm": 0.08419760316610336, + "kl": 4.357472062110901e-05, + "lambda_div_used": 0.6037088930606842, + "learning_rate": 2.931788945420058e-07, + "loss": 0.0176, + "reward": -0.09517315030097961, + "reward_after_mean": -0.09517315030097961, + "reward_after_std": 0.57970448769629, + "reward_before_mean": 0.282845395617187, + "reward_before_std": 0.4830307289958, + "reward_change_max": 0.0, + "reward_change_mean": -0.37801856361329556, + "reward_change_min": -0.5875305943191051, + "reward_change_std": 0.22446841653436422, + "reward_std": 0.57970448769629, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": 0.03284538397565484, + "step": 363 + }, + { + "clip_fraction": 0.0, + "completion_length": 2614.9167098999023, + "epoch": 0.416, + "grad_norm": 0.07822614908218384, + "kl": 2.289540134370327e-05, + "lambda_div_used": 0.5602610111236572, + "learning_rate": 2.9060545772359305e-07, + "loss": 0.0079, + "reward": -0.4077296704053879, + "reward_after_mean": -0.4077296704053879, + "reward_after_std": 0.3345623780041933, + "reward_before_mean": -0.1330121699720621, + "reward_before_std": 0.27498441375792027, + "reward_change_max": 0.0, + "reward_change_mean": -0.2747174873948097, + "reward_change_min": -0.4260003827512264, + "reward_change_std": 0.16245513781905174, + "reward_std": 0.3345623817294836, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.195512181147933, + "step": 364 + }, + { + "clip_fraction": 0.0, + "completion_length": 3003.604202270508, + "epoch": 0.41714285714285715, + "grad_norm": 0.04828161001205444, + "kl": 6.87967985868454e-06, + "lambda_div_used": 0.6036554351449013, + "learning_rate": 2.8804466342921987e-07, + "loss": 0.0555, + "reward": -0.31679424038156867, + "reward_after_mean": -0.31679424038156867, + "reward_after_std": 0.5247047282755375, + "reward_before_mean": -0.07733823172748089, + "reward_before_std": 0.4768796032294631, + "reward_change_max": 0.0, + "reward_change_mean": -0.23945600911974907, + "reward_change_min": -0.42952996119856834, + "reward_change_std": 0.15284609980881214, + "reward_std": 0.5247047450393438, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.18150490219704807, + "step": 365 + }, + { + "clip_fraction": 0.0, + "completion_length": 1893.00004196167, + "epoch": 0.41828571428571426, + "grad_norm": 0.08456390351057053, + "kl": 1.6473233699798584e-05, + "lambda_div_used": 0.6091032102704048, + "learning_rate": 2.854966364683872e-07, + "loss": -0.0195, + "reward": 0.005720527842640877, + "reward_after_mean": 0.005720527842640877, + "reward_after_std": 0.6514943428337574, + "reward_before_mean": 0.43899719044566154, + "reward_before_std": 0.49832448456436396, + "reward_change_max": 0.0, + "reward_change_mean": -0.4332766607403755, + "reward_change_min": -0.6058122254908085, + "reward_change_std": 0.23482245951890945, + "reward_std": 0.6514943763613701, + "rewards/accuracy_reward": 0.39583333767950535, + "rewards/cosine_scaled_reward": 0.043163834139704704, + "step": 366 + }, + { + "clip_fraction": 0.0, + "completion_length": 2780.937515258789, + "epoch": 0.41942857142857143, + "grad_norm": 0.06994574517011642, + "kl": 2.946704626083374e-05, + "lambda_div_used": 0.6079575195908546, + "learning_rate": 2.829615010283344e-07, + "loss": 0.0087, + "reward": -0.16347008850425482, + "reward_after_mean": -0.16347008850425482, + "reward_after_std": 0.5466162711381912, + "reward_before_mean": 0.1403359491378069, + "reward_before_std": 0.49341694079339504, + "reward_change_max": 0.0, + "reward_change_mean": -0.30380604416131973, + "reward_change_min": -0.4883100688457489, + "reward_change_std": 0.18378268275409937, + "reward_std": 0.5466162823140621, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.0471640374744311, + "step": 367 + }, + { + "clip_fraction": 0.0, + "completion_length": 2967.041679382324, + "epoch": 0.4205714285714286, + "grad_norm": 0.07414139062166214, + "kl": 4.155188798904419e-05, + "lambda_div_used": 0.5803077146410942, + "learning_rate": 2.8043938066798645e-07, + "loss": 0.0504, + "reward": -0.27402403950691223, + "reward_after_mean": -0.27402403950691223, + "reward_after_std": 0.4059589561074972, + "reward_before_mean": 0.008191026747226715, + "reward_before_std": 0.371637674048543, + "reward_change_max": 0.0, + "reward_change_mean": -0.28221505135297775, + "reward_change_min": -0.44202784821391106, + "reward_change_std": 0.17603088915348053, + "reward_std": 0.40595896542072296, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.15847563743591309, + "step": 368 + }, + { + "clip_fraction": 0.0, + "completion_length": 2863.1667404174805, + "epoch": 0.4217142857142857, + "grad_norm": 0.07171155512332916, + "kl": 2.0891427993774414e-05, + "lambda_div_used": 0.6472674459218979, + "learning_rate": 2.7793039831193133e-07, + "loss": -0.0044, + "reward": -0.04304695222526789, + "reward_after_mean": -0.04304695222526789, + "reward_after_std": 0.687617726624012, + "reward_before_mean": 0.25146727729588747, + "reward_before_std": 0.6888436311855912, + "reward_change_max": 0.0, + "reward_change_mean": -0.29451421834528446, + "reward_change_min": -0.6169496960937977, + "reward_change_std": 0.22693553566932678, + "reward_std": 0.6876177359372377, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": 0.0014672745019197464, + "step": 369 + }, + { + "clip_fraction": 0.0, + "completion_length": 3261.2291870117188, + "epoch": 0.4228571428571429, + "grad_norm": 0.058094967156648636, + "kl": 2.1554529666900635e-05, + "lambda_div_used": 0.5657742545008659, + "learning_rate": 2.7543467624442956e-07, + "loss": 0.0031, + "reward": -0.2483972143381834, + "reward_after_mean": -0.2483972143381834, + "reward_after_std": 0.4499538466334343, + "reward_before_mean": 0.13386818021535873, + "reward_before_std": 0.29791492875665426, + "reward_change_max": 0.0, + "reward_change_mean": -0.38226539455354214, + "reward_change_min": -0.51585279032588, + "reward_change_std": 0.19617348536849022, + "reward_std": 0.4499538540840149, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.03279848676174879, + "step": 370 + }, + { + "clip_fraction": 0.0, + "completion_length": 1885.645866394043, + "epoch": 0.424, + "grad_norm": 0.12381042540073395, + "kl": 1.5240773791447282e-05, + "lambda_div_used": 0.5325883999466896, + "learning_rate": 2.729523361034538e-07, + "loss": -0.0304, + "reward": -0.14332129061222076, + "reward_after_mean": -0.14332129061222076, + "reward_after_std": 0.3713335506618023, + "reward_before_mean": 0.3775896169245243, + "reward_before_std": 0.14313361048698425, + "reward_change_max": 0.0, + "reward_change_mean": -0.5209108907729387, + "reward_change_min": -0.6799871250987053, + "reward_change_std": 0.26304140500724316, + "reward_std": 0.3713335543870926, + "rewards/accuracy_reward": 0.375, + "rewards/cosine_scaled_reward": 0.0025896020233631134, + "step": 371 + }, + { + "clip_fraction": 0.0, + "completion_length": 3061.6250228881836, + "epoch": 0.42514285714285716, + "grad_norm": 0.0565616711974144, + "kl": 1.3771001249551773e-05, + "lambda_div_used": 0.6153379678726196, + "learning_rate": 2.7048349887476037e-07, + "loss": 0.0019, + "reward": 0.08842738252133131, + "reward_after_mean": 0.08842738252133131, + "reward_after_std": 0.6899000462144613, + "reward_before_mean": 0.5513297475408763, + "reward_before_std": 0.5352799613028765, + "reward_change_max": 0.0, + "reward_change_mean": -0.46290238574147224, + "reward_change_min": -0.6905807815492153, + "reward_change_std": 0.26435263454914093, + "reward_std": 0.6899000480771065, + "rewards/accuracy_reward": 0.3958333358168602, + "rewards/cosine_scaled_reward": 0.15549640776589513, + "step": 372 + }, + { + "clip_fraction": 0.0, + "completion_length": 1999.791690826416, + "epoch": 0.42628571428571427, + "grad_norm": 0.11289634555578232, + "kl": 2.6244670152664185e-05, + "lambda_div_used": 0.5901408270001411, + "learning_rate": 2.6802828488599294e-07, + "loss": -0.0028, + "reward": -0.25084885116666555, + "reward_after_mean": -0.25084885116666555, + "reward_after_std": 0.47954990342259407, + "reward_before_mean": 0.03629123326390982, + "reward_before_std": 0.41134614683687687, + "reward_change_max": 0.0, + "reward_change_mean": -0.28714008443057537, + "reward_change_min": -0.4200817756354809, + "reward_change_std": 0.1626526527106762, + "reward_std": 0.47954992204904556, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.08870877418667078, + "step": 373 + }, + { + "clip_fraction": 0.0, + "completion_length": 2700.791717529297, + "epoch": 0.42742857142857144, + "grad_norm": 0.0688440129160881, + "kl": 2.674013376235962e-05, + "lambda_div_used": 0.6036395579576492, + "learning_rate": 2.655868138008171e-07, + "loss": -0.0525, + "reward": -0.17511339485645294, + "reward_after_mean": -0.17511339485645294, + "reward_after_std": 0.5073369853198528, + "reward_before_mean": 0.13348117470741272, + "reward_before_std": 0.48259105905890465, + "reward_change_max": 0.0, + "reward_change_mean": -0.30859458073973656, + "reward_change_min": -0.5309739634394646, + "reward_change_std": 0.20630291104316711, + "reward_std": 0.5073370076715946, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.07485215552151203, + "step": 374 + }, + { + "clip_fraction": 0.0, + "completion_length": 3033.791702270508, + "epoch": 0.42857142857142855, + "grad_norm": 0.05866050720214844, + "kl": 1.8252991139888763e-05, + "lambda_div_used": 0.6017400398850441, + "learning_rate": 2.631592046130896e-07, + "loss": 0.0477, + "reward": -0.18571221362799406, + "reward_after_mean": -0.18571221362799406, + "reward_after_std": 0.5023255608975887, + "reward_before_mean": 0.11062880232930183, + "reward_before_std": 0.46861019916832447, + "reward_change_max": 0.0, + "reward_change_mean": -0.2963410019874573, + "reward_change_min": -0.45132842287421227, + "reward_change_std": 0.1817708509042859, + "reward_std": 0.5023255832493305, + "rewards/accuracy_reward": 0.20833334140479565, + "rewards/cosine_scaled_reward": -0.097704553976655, + "step": 375 + }, + { + "clip_fraction": 0.0, + "completion_length": 2246.770851135254, + "epoch": 0.4297142857142857, + "grad_norm": 0.08337263762950897, + "kl": 8.605420589447021e-06, + "lambda_div_used": 0.5853807479143143, + "learning_rate": 2.6074557564105724e-07, + "loss": 0.0128, + "reward": -0.3334618601948023, + "reward_after_mean": -0.3334618601948023, + "reward_after_std": 0.4486316703259945, + "reward_before_mean": -0.05933803477091715, + "reward_before_std": 0.3892780668102205, + "reward_change_max": 0.0, + "reward_change_mean": -0.2741238009184599, + "reward_change_min": -0.46821001917123795, + "reward_change_std": 0.1663337228819728, + "reward_std": 0.44863167591392994, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.163504708558321, + "step": 376 + }, + { + "clip_fraction": 0.0, + "completion_length": 3488.3541870117188, + "epoch": 0.4308571428571429, + "grad_norm": 0.055966660380363464, + "kl": 2.5756657123565674e-05, + "lambda_div_used": 0.5742340236902237, + "learning_rate": 2.583460445215911e-07, + "loss": -0.0032, + "reward": -0.3738445993512869, + "reward_after_mean": -0.3738445993512869, + "reward_after_std": 0.39967909548431635, + "reward_before_mean": -0.10715527582215145, + "reward_before_std": 0.33878533728420734, + "reward_change_max": 0.0, + "reward_change_mean": -0.26668932288885117, + "reward_change_min": -0.4109174869954586, + "reward_change_std": 0.15686567965894938, + "reward_std": 0.39967911317944527, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.19048861227929592, + "step": 377 + }, + { + "clip_fraction": 0.0, + "completion_length": 2134.3333740234375, + "epoch": 0.432, + "grad_norm": 0.13181807100772858, + "kl": 2.8654932975769043e-05, + "lambda_div_used": 0.6004306748509407, + "learning_rate": 2.5596072820445254e-07, + "loss": -0.0331, + "reward": -0.06989812850952148, + "reward_after_mean": -0.06989812850952148, + "reward_after_std": 0.5459046512842178, + "reward_before_mean": 0.31881395215168595, + "reward_before_std": 0.46263560838997364, + "reward_change_max": 0.0, + "reward_change_mean": -0.3887120857834816, + "reward_change_min": -0.5917952992022038, + "reward_change_std": 0.23005317710340023, + "reward_std": 0.5459046605974436, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.027147281914949417, + "step": 378 + }, + { + "clip_fraction": 0.0, + "completion_length": 3121.0416870117188, + "epoch": 0.43314285714285716, + "grad_norm": 0.07427296042442322, + "kl": 1.5733763575553894e-05, + "lambda_div_used": 0.5833418369293213, + "learning_rate": 2.5358974294659373e-07, + "loss": -0.0048, + "reward": -0.29101302847266197, + "reward_after_mean": -0.29101302847266197, + "reward_after_std": 0.4290795102715492, + "reward_before_mean": -0.009382706135511398, + "reward_before_std": 0.38597164303064346, + "reward_change_max": 0.0, + "reward_change_mean": -0.2816303465515375, + "reward_change_min": -0.47839244455099106, + "reward_change_std": 0.17911072820425034, + "reward_std": 0.4290795363485813, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.13438269309699535, + "step": 379 + }, + { + "clip_fraction": 0.0, + "completion_length": 2823.416702270508, + "epoch": 0.4342857142857143, + "grad_norm": 0.06611248105764389, + "kl": 2.2601336240768433e-05, + "lambda_div_used": 0.6066281795501709, + "learning_rate": 2.512332043064913e-07, + "loss": 0.0533, + "reward": -0.17613293696194887, + "reward_after_mean": -0.17613293696194887, + "reward_after_std": 0.495351817458868, + "reward_before_mean": 0.1346770692616701, + "reward_before_std": 0.48867712169885635, + "reward_change_max": 0.0, + "reward_change_mean": -0.31081000342965126, + "reward_change_min": -0.5689719989895821, + "reward_change_std": 0.21053248085081577, + "reward_std": 0.4953518286347389, + "rewards/accuracy_reward": 0.1875000074505806, + "rewards/cosine_scaled_reward": -0.05282294252538122, + "step": 380 + }, + { + "clip_fraction": 0.0, + "completion_length": 2983.7916984558105, + "epoch": 0.43542857142857144, + "grad_norm": 0.08519254624843597, + "kl": 3.2504089176654816e-05, + "lambda_div_used": 0.6109358817338943, + "learning_rate": 2.488912271385139e-07, + "loss": 0.0165, + "reward": -0.2218069350346923, + "reward_after_mean": -0.2218069350346923, + "reward_after_std": 0.5642109382897615, + "reward_before_mean": 0.040106164291501045, + "reward_before_std": 0.5099438140168786, + "reward_change_max": 0.0, + "reward_change_mean": -0.261913089081645, + "reward_change_min": -0.45296039059758186, + "reward_change_std": 0.1642473293468356, + "reward_std": 0.5642109606415033, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.10572716826573014, + "step": 381 + }, + { + "clip_fraction": 0.0, + "completion_length": 2500.437545776367, + "epoch": 0.43657142857142855, + "grad_norm": 0.07325682789087296, + "kl": 3.65767627954483e-05, + "lambda_div_used": 0.5600408837199211, + "learning_rate": 2.465639255873246e-07, + "loss": -0.0096, + "reward": -0.4078812226653099, + "reward_after_mean": -0.4078812226653099, + "reward_after_std": 0.3340430334210396, + "reward_before_mean": -0.13890772312879562, + "reward_before_std": 0.27443209011107683, + "reward_change_max": 0.0, + "reward_change_mean": -0.26897350139915943, + "reward_change_min": -0.42554087564349174, + "reward_change_std": 0.15754834469407797, + "reward_std": 0.3340430427342653, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.20140772499144077, + "step": 382 + }, + { + "clip_fraction": 0.0, + "completion_length": 2824.291717529297, + "epoch": 0.4377142857142857, + "grad_norm": 0.08796168118715286, + "kl": 4.8138201236724854e-05, + "lambda_div_used": 0.5781937688589096, + "learning_rate": 2.4425141308231765e-07, + "loss": -0.0086, + "reward": -0.16291768848896027, + "reward_after_mean": -0.16291768848896027, + "reward_after_std": 0.387674568220973, + "reward_before_mean": 0.1924248207360506, + "reward_before_std": 0.35597400926053524, + "reward_change_max": 0.0, + "reward_change_mean": -0.3553424943238497, + "reward_change_min": -0.5255323797464371, + "reward_change_std": 0.21135967783629894, + "reward_std": 0.3876745719462633, + "rewards/accuracy_reward": 0.2083333432674408, + "rewards/cosine_scaled_reward": -0.01590852066874504, + "step": 383 + }, + { + "clip_fraction": 0.0, + "completion_length": 2347.6042404174805, + "epoch": 0.43885714285714283, + "grad_norm": 0.09212023764848709, + "kl": 2.508983016014099e-05, + "lambda_div_used": 0.6615950018167496, + "learning_rate": 2.4195380233209006e-07, + "loss": 0.0408, + "reward": 0.27819300815463066, + "reward_after_mean": 0.27819300815463066, + "reward_after_std": 0.7268820777535439, + "reward_before_mean": 0.6785084716975689, + "reward_before_std": 0.7540020374581218, + "reward_change_max": 0.0, + "reward_change_mean": -0.4003154691308737, + "reward_change_min": -0.6902611702680588, + "reward_change_std": 0.283959056250751, + "reward_std": 0.7268821019679308, + "rewards/accuracy_reward": 0.4583333432674408, + "rewards/cosine_scaled_reward": 0.22017512656748295, + "step": 384 + }, + { + "clip_fraction": 0.0, + "completion_length": 2711.958396911621, + "epoch": 0.44, + "grad_norm": 0.07038652151823044, + "kl": 1.942366361618042e-05, + "lambda_div_used": 0.6165113672614098, + "learning_rate": 2.3967120531894857e-07, + "loss": 0.012, + "reward": -0.25626325886696577, + "reward_after_mean": -0.25626325886696577, + "reward_after_std": 0.5892766248434782, + "reward_before_mean": -0.0074533987790346146, + "reward_before_std": 0.5360177559778094, + "reward_change_max": 0.0, + "reward_change_mean": -0.24880987033247948, + "reward_change_min": -0.44854912906885147, + "reward_change_std": 0.1595793990418315, + "reward_std": 0.5892766322940588, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.13245339877903461, + "step": 385 + }, + { + "clip_fraction": 0.0, + "completion_length": 2889.1042404174805, + "epoch": 0.44114285714285717, + "grad_norm": 0.060970280319452286, + "kl": 1.8231570720672607e-05, + "lambda_div_used": 0.5940364003181458, + "learning_rate": 2.374037332934512e-07, + "loss": -0.0287, + "reward": -0.24576714914292097, + "reward_after_mean": -0.24576714914292097, + "reward_after_std": 0.4883785657584667, + "reward_before_mean": 0.04634229093790054, + "reward_before_std": 0.4279829766601324, + "reward_change_max": 0.0, + "reward_change_mean": -0.29210945032536983, + "reward_change_min": -0.4432514049112797, + "reward_change_std": 0.17009661067277193, + "reward_std": 0.48837856762111187, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.07865770626813173, + "step": 386 + }, + { + "clip_fraction": 0.0, + "completion_length": 2997.3125, + "epoch": 0.4422857142857143, + "grad_norm": 0.07781542837619781, + "kl": 1.3821758329868317e-05, + "lambda_div_used": 0.5551474094390869, + "learning_rate": 2.3515149676898552e-07, + "loss": -0.0529, + "reward": -0.38486793637275696, + "reward_after_mean": -0.38486793637275696, + "reward_after_std": 0.31045267172157764, + "reward_before_mean": -0.10442159324884415, + "reward_before_std": 0.24840335873886943, + "reward_change_max": 0.0, + "reward_change_mean": -0.28044634498655796, + "reward_change_min": -0.4241391494870186, + "reward_change_std": 0.15805031638592482, + "reward_std": 0.31045267917215824, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.20858826488256454, + "step": 387 + }, + { + "clip_fraction": 0.0, + "completion_length": 2572.2708587646484, + "epoch": 0.44342857142857145, + "grad_norm": 0.06563723832368851, + "kl": 2.9848888516426086e-06, + "lambda_div_used": 0.5865650475025177, + "learning_rate": 2.3291460551638237e-07, + "loss": 0.0407, + "reward": -0.15434425324201584, + "reward_after_mean": -0.15434425324201584, + "reward_after_std": 0.47601727209985256, + "reward_before_mean": 0.20021704956889153, + "reward_before_std": 0.39916726760566235, + "reward_change_max": 0.0, + "reward_change_mean": -0.354561323300004, + "reward_change_min": -0.5467559210956097, + "reward_change_std": 0.21171134896576405, + "reward_std": 0.47601727209985256, + "rewards/accuracy_reward": 0.2708333358168602, + "rewards/cosine_scaled_reward": -0.07061627879738808, + "step": 388 + }, + { + "clip_fraction": 0.0, + "completion_length": 2650.937545776367, + "epoch": 0.44457142857142856, + "grad_norm": 0.07840488851070404, + "kl": 1.73286534845829e-05, + "lambda_div_used": 0.6045578718185425, + "learning_rate": 2.306931685585657e-07, + "loss": -0.0394, + "reward": -0.26122746989130974, + "reward_after_mean": -0.26122746989130974, + "reward_after_std": 0.5210577324032784, + "reward_before_mean": 0.002998221665620804, + "reward_before_std": 0.486898148432374, + "reward_change_max": 0.0, + "reward_change_mean": -0.26422569528222084, + "reward_change_min": -0.4951511509716511, + "reward_change_std": 0.18348107766360044, + "reward_std": 0.5210577566176653, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.1428351059439592, + "step": 389 + }, + { + "clip_fraction": 0.0, + "completion_length": 3007.666679382324, + "epoch": 0.44571428571428573, + "grad_norm": 0.07837636768817902, + "kl": 1.6082078218460083e-05, + "lambda_div_used": 0.61600511521101, + "learning_rate": 2.2848729416523859e-07, + "loss": 0.0713, + "reward": -0.18647840432822704, + "reward_after_mean": -0.18647840432822704, + "reward_after_std": 0.5758322961628437, + "reward_before_mean": 0.08954079262912273, + "reward_before_std": 0.5347468825057149, + "reward_change_max": 0.0, + "reward_change_mean": -0.27601918019354343, + "reward_change_min": -0.47189946845173836, + "reward_change_std": 0.17452176753431559, + "reward_std": 0.5758323054760695, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.07712590089067817, + "step": 390 + }, + { + "clip_fraction": 0.0, + "completion_length": 2716.708351135254, + "epoch": 0.44685714285714284, + "grad_norm": 0.09803462773561478, + "kl": 2.5559216737747192e-05, + "lambda_div_used": 0.6362191960215569, + "learning_rate": 2.2629708984760706e-07, + "loss": 0.0166, + "reward": -0.013867436617147177, + "reward_after_mean": -0.013867436617147177, + "reward_after_std": 0.6719868443906307, + "reward_before_mean": 0.30256712157279253, + "reward_before_std": 0.6324740117415786, + "reward_change_max": 0.0, + "reward_change_mean": -0.3164345696568489, + "reward_change_min": -0.5261501334607601, + "reward_change_std": 0.20623697619885206, + "reward_std": 0.6719868592917919, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.03173377411440015, + "step": 391 + }, + { + "clip_fraction": 0.0, + "completion_length": 2130.9791984558105, + "epoch": 0.448, + "grad_norm": 0.11540649086236954, + "kl": 1.8610619008541107e-05, + "lambda_div_used": 0.6027960926294327, + "learning_rate": 2.2412266235313973e-07, + "loss": -0.0919, + "reward": -0.16835473664104939, + "reward_after_mean": -0.16835473664104939, + "reward_after_std": 0.5122340489178896, + "reward_before_mean": 0.1393747702240944, + "reward_before_std": 0.4794600326567888, + "reward_change_max": 0.0, + "reward_change_mean": -0.3077295087277889, + "reward_change_min": -0.4898943528532982, + "reward_change_std": 0.1954572731629014, + "reward_std": 0.5122340768575668, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.027291906299069524, + "step": 392 + }, + { + "clip_fraction": 0.0, + "completion_length": 2649.562530517578, + "epoch": 0.4491428571428571, + "grad_norm": 0.07226411253213882, + "kl": 2.648681402206421e-05, + "lambda_div_used": 0.5953627824783325, + "learning_rate": 2.2196411766036487e-07, + "loss": -0.067, + "reward": -0.20140517689287663, + "reward_after_mean": -0.20140517689287663, + "reward_after_std": 0.4766168761998415, + "reward_before_mean": 0.09705937840044498, + "reward_before_std": 0.4375506564974785, + "reward_change_max": 0.0, + "reward_change_mean": -0.2984645199030638, + "reward_change_min": -0.47758448868989944, + "reward_change_std": 0.18242334388196468, + "reward_std": 0.47661688551306725, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.09044064581394196, + "step": 393 + }, + { + "clip_fraction": 0.0, + "completion_length": 3104.1458740234375, + "epoch": 0.4502857142857143, + "grad_norm": 0.07446404546499252, + "kl": 2.555176615715027e-05, + "lambda_div_used": 0.5502220839262009, + "learning_rate": 2.1982156097370557e-07, + "loss": -0.1089, + "reward": -0.46874738670885563, + "reward_after_mean": -0.46874738670885563, + "reward_after_std": 0.30342659167945385, + "reward_before_mean": -0.20392105542123318, + "reward_before_std": 0.22309745661914349, + "reward_change_max": 0.0, + "reward_change_mean": -0.2648263294249773, + "reward_change_min": -0.3818593733012676, + "reward_change_std": 0.14172559697180986, + "reward_std": 0.30342659167945385, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.22475438751280308, + "step": 394 + }, + { + "clip_fraction": 0.0, + "completion_length": 2284.6458435058594, + "epoch": 0.4514285714285714, + "grad_norm": 0.12902304530143738, + "kl": 4.9501657485961914e-05, + "lambda_div_used": 0.5602920204401016, + "learning_rate": 2.1769509671835223e-07, + "loss": -0.1675, + "reward": -0.3165151756256819, + "reward_after_mean": -0.3165151756256819, + "reward_after_std": 0.3793674483895302, + "reward_before_mean": 0.0216768067330122, + "reward_before_std": 0.27139727398753166, + "reward_change_max": 0.0, + "reward_change_mean": -0.3381919823586941, + "reward_change_min": -0.48955149203538895, + "reward_change_std": 0.17990652937442064, + "reward_std": 0.3793674521148205, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.12415653187781572, + "step": 395 + }, + { + "clip_fraction": 0.0, + "completion_length": 3048.1458587646484, + "epoch": 0.45257142857142857, + "grad_norm": 0.054839495569467545, + "kl": 7.04331323504448e-05, + "lambda_div_used": 0.6417915895581245, + "learning_rate": 2.1558482853517253e-07, + "loss": 0.0192, + "reward": -0.09411877207458019, + "reward_after_mean": -0.09411877207458019, + "reward_after_std": 0.6742421016097069, + "reward_before_mean": 0.1871709941624431, + "reward_before_std": 0.659186695702374, + "reward_change_max": 0.0, + "reward_change_mean": -0.2812897562980652, + "reward_change_min": -0.5664796940982342, + "reward_change_std": 0.20405743923038244, + "reward_std": 0.6742421071976423, + "rewards/accuracy_reward": 0.20833333767950535, + "rewards/cosine_scaled_reward": -0.021162351593375206, + "step": 396 + }, + { + "clip_fraction": 0.0, + "completion_length": 2982.750030517578, + "epoch": 0.45371428571428574, + "grad_norm": 0.05798732861876488, + "kl": 2.0850449800491333e-05, + "lambda_div_used": 0.5765915215015411, + "learning_rate": 2.134908592756607e-07, + "loss": 0.0391, + "reward": -0.36006474308669567, + "reward_after_mean": -0.36006474308669567, + "reward_after_std": 0.40458301082253456, + "reward_before_mean": -0.08499279711395502, + "reward_before_std": 0.3492864612489939, + "reward_change_max": 0.0, + "reward_change_mean": -0.2750719413161278, + "reward_change_min": -0.4323217496275902, + "reward_change_std": 0.16262990981340408, + "reward_std": 0.40458302199840546, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.16832613572478294, + "step": 397 + }, + { + "clip_fraction": 0.0, + "completion_length": 2822.5209045410156, + "epoch": 0.45485714285714285, + "grad_norm": 0.06360434740781784, + "kl": 2.2810418158769608e-05, + "lambda_div_used": 0.6480221897363663, + "learning_rate": 2.1141329099692406e-07, + "loss": 0.0465, + "reward": -0.09229415841400623, + "reward_after_mean": -0.09229415841400623, + "reward_after_std": 0.7079674638807774, + "reward_before_mean": 0.1752314588520676, + "reward_before_std": 0.6917341919615865, + "reward_change_max": 0.0, + "reward_change_mean": -0.26752561889588833, + "reward_change_min": -0.5181693434715271, + "reward_change_std": 0.19102454278618097, + "reward_std": 0.7079674787819386, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.03310186788439751, + "step": 398 + }, + { + "clip_fraction": 0.0, + "completion_length": 2191.583351135254, + "epoch": 0.456, + "grad_norm": 0.07585066556930542, + "kl": 3.449246287345886e-05, + "lambda_div_used": 0.6071069464087486, + "learning_rate": 2.0935222495670968e-07, + "loss": -0.0428, + "reward": -0.06968086212873459, + "reward_after_mean": -0.06968086212873459, + "reward_after_std": 0.5623702071607113, + "reward_before_mean": 0.29000685061328113, + "reward_before_std": 0.4939764440059662, + "reward_change_max": 0.0, + "reward_change_mean": -0.35968772508203983, + "reward_change_min": -0.5781176537275314, + "reward_change_std": 0.2176226656883955, + "reward_std": 0.5623702295124531, + "rewards/accuracy_reward": 0.27083333767950535, + "rewards/cosine_scaled_reward": 0.01917351223528385, + "step": 399 + }, + { + "clip_fraction": 0.0, + "completion_length": 1814.4375305175781, + "epoch": 0.45714285714285713, + "grad_norm": 0.09906096011400223, + "kl": 3.718771040439606e-05, + "lambda_div_used": 0.615766242146492, + "learning_rate": 2.0730776160846853e-07, + "loss": -0.0203, + "reward": 0.023047026246786118, + "reward_after_mean": 0.023047026246786118, + "reward_after_std": 0.6236102003604174, + "reward_before_mean": 0.43256790889427066, + "reward_before_std": 0.5322064086794853, + "reward_change_max": 0.0, + "reward_change_mean": -0.40952087566256523, + "reward_change_min": -0.6220400035381317, + "reward_change_std": 0.2354581467807293, + "reward_std": 0.62361023388803, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.12006791215389967, + "step": 400 + }, + { + "clip_fraction": 0.0, + "completion_length": 3091.7916870117188, + "epoch": 0.4582857142857143, + "grad_norm": 0.05475914105772972, + "kl": 3.5446137189865112e-06, + "lambda_div_used": 0.562153548002243, + "learning_rate": 2.0528000059645995e-07, + "loss": 0.0508, + "reward": -0.29909564182162285, + "reward_after_mean": -0.29909564182162285, + "reward_after_std": 0.3837758805602789, + "reward_before_mean": 0.04575100168585777, + "reward_before_std": 0.28164876997470856, + "reward_change_max": 0.0, + "reward_change_mean": -0.3448466807603836, + "reward_change_min": -0.5300594680011272, + "reward_change_std": 0.19381076097488403, + "reward_std": 0.38377588987350464, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.12091565364971757, + "step": 401 + }, + { + "clip_fraction": 0.0, + "completion_length": 2440.1458740234375, + "epoch": 0.4594285714285714, + "grad_norm": 0.11770489811897278, + "kl": 1.8846243619918823e-05, + "lambda_div_used": 0.5605455562472343, + "learning_rate": 2.032690407508949e-07, + "loss": -0.0261, + "reward": -0.2383374497294426, + "reward_after_mean": -0.2383374497294426, + "reward_after_std": 0.3794025480747223, + "reward_before_mean": 0.14294641837477684, + "reward_before_std": 0.27562023140490055, + "reward_change_max": 0.0, + "reward_change_mean": -0.38128384202718735, + "reward_change_min": -0.5627436973154545, + "reward_change_std": 0.2136234436184168, + "reward_std": 0.3794025518000126, + "rewards/accuracy_reward": 0.1875, + "rewards/cosine_scaled_reward": -0.044553620740771294, + "step": 402 + }, + { + "clip_fraction": 0.0, + "completion_length": 1985.6875343322754, + "epoch": 0.4605714285714286, + "grad_norm": 0.10363903641700745, + "kl": 2.3838132619857788e-05, + "lambda_div_used": 0.5793934315443039, + "learning_rate": 2.0127498008311922e-07, + "loss": 0.003, + "reward": -0.17537187691777945, + "reward_after_mean": -0.17537187691777945, + "reward_after_std": 0.46652381494641304, + "reward_before_mean": 0.20323466695845127, + "reward_before_std": 0.3647587588056922, + "reward_change_max": 0.0, + "reward_change_mean": -0.37860656157135963, + "reward_change_min": -0.5506669841706753, + "reward_change_std": 0.21334033645689487, + "reward_std": 0.4665238317102194, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.025931986048817635, + "step": 403 + }, + { + "clip_fraction": 0.0, + "completion_length": 2137.375, + "epoch": 0.4617142857142857, + "grad_norm": 0.10717064887285233, + "kl": 3.7202611565589905e-05, + "lambda_div_used": 0.5993036478757858, + "learning_rate": 1.9929791578083655e-07, + "loss": -0.0043, + "reward": -0.053446926176548004, + "reward_after_mean": -0.053446926176548004, + "reward_after_std": 0.4612566214054823, + "reward_before_mean": 0.3198554217815399, + "reward_before_std": 0.4542539082467556, + "reward_change_max": 0.0, + "reward_change_mean": -0.3733023554086685, + "reward_change_min": -0.5774630047380924, + "reward_change_std": 0.2339334823191166, + "reward_std": 0.4612566400319338, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.04902207478880882, + "step": 404 + }, + { + "clip_fraction": 0.0, + "completion_length": 2319.5000228881836, + "epoch": 0.46285714285714286, + "grad_norm": 0.13891565799713135, + "kl": 2.9403716325759888e-05, + "lambda_div_used": 0.6237343773245811, + "learning_rate": 1.9733794420337213e-07, + "loss": -0.0602, + "reward": 0.1285000964999199, + "reward_after_mean": 0.1285000964999199, + "reward_after_std": 0.7067493386566639, + "reward_before_mean": 0.5776062086224556, + "reward_before_std": 0.5746421907097101, + "reward_change_max": 0.0, + "reward_change_mean": -0.449106115847826, + "reward_change_min": -0.6593505516648293, + "reward_change_std": 0.263070298358798, + "reward_std": 0.7067493461072445, + "rewards/accuracy_reward": 0.4166666679084301, + "rewards/cosine_scaled_reward": 0.1609395444393158, + "step": 405 + }, + { + "clip_fraction": 0.0, + "completion_length": 2442.1458892822266, + "epoch": 0.464, + "grad_norm": 0.07689624279737473, + "kl": 8.527189493179321e-06, + "lambda_div_used": 0.6336105018854141, + "learning_rate": 1.9539516087697517e-07, + "loss": -0.0, + "reward": -0.11170937749557197, + "reward_after_mean": -0.11170937749557197, + "reward_after_std": 0.6453428398817778, + "reward_before_mean": 0.17541324836201966, + "reward_before_std": 0.6142212487757206, + "reward_change_max": 0.0, + "reward_change_mean": -0.2871226370334625, + "reward_change_min": -0.5107162520289421, + "reward_change_std": 0.19029081240296364, + "reward_std": 0.645342854782939, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.032920089550316334, + "step": 406 + }, + { + "clip_fraction": 0.0, + "completion_length": 2516.6041946411133, + "epoch": 0.46514285714285714, + "grad_norm": 0.09361904859542847, + "kl": 2.995133399963379e-05, + "lambda_div_used": 0.5926980003714561, + "learning_rate": 1.934696604901642e-07, + "loss": 0.011, + "reward": -0.16482560662552714, + "reward_after_mean": -0.16482560662552714, + "reward_after_std": 0.5236028637737036, + "reward_before_mean": 0.18611273169517517, + "reward_before_std": 0.4258856289088726, + "reward_change_max": 0.0, + "reward_change_mean": -0.35093834809958935, + "reward_change_min": -0.5517234578728676, + "reward_change_std": 0.20658957865089178, + "reward_std": 0.5236028656363487, + "rewards/accuracy_reward": 0.25000000186264515, + "rewards/cosine_scaled_reward": -0.06388726737350225, + "step": 407 + }, + { + "clip_fraction": 0.0, + "completion_length": 2709.4791984558105, + "epoch": 0.4662857142857143, + "grad_norm": 0.09306048601865768, + "kl": 3.203703090548515e-05, + "lambda_div_used": 0.5583930537104607, + "learning_rate": 1.915615368891117e-07, + "loss": -0.111, + "reward": -0.17684245621785522, + "reward_after_mean": -0.17684245621785522, + "reward_after_std": 0.42921141162514687, + "reward_before_mean": 0.2615569829940796, + "reward_before_std": 0.26226984336972237, + "reward_change_max": 0.0, + "reward_change_mean": -0.43839943781495094, + "reward_change_min": -0.6072936318814754, + "reward_change_std": 0.22694199439138174, + "reward_std": 0.42921141907572746, + "rewards/accuracy_reward": 0.27083333395421505, + "rewards/cosine_scaled_reward": -0.009276359807699919, + "step": 408 + }, + { + "clip_fraction": 0.0, + "completion_length": 3529.3541870117188, + "epoch": 0.4674285714285714, + "grad_norm": 0.048929836601018906, + "kl": 7.6089054346084595e-06, + "lambda_div_used": 0.6116252392530441, + "learning_rate": 1.8967088307307e-07, + "loss": 0.001, + "reward": -0.17062923312187195, + "reward_after_mean": -0.17062923312187195, + "reward_after_std": 0.5682655684649944, + "reward_before_mean": 0.12847105879336596, + "reward_before_std": 0.5097848381847143, + "reward_change_max": 0.0, + "reward_change_mean": -0.2991002984344959, + "reward_change_min": -0.43997345492243767, + "reward_change_std": 0.17078326642513275, + "reward_std": 0.5682655889540911, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.038195611676201224, + "step": 409 + }, + { + "clip_fraction": 0.0, + "completion_length": 2435.625045776367, + "epoch": 0.4685714285714286, + "grad_norm": 0.08304905891418457, + "kl": 1.5234574675559998e-05, + "lambda_div_used": 0.6268336623907089, + "learning_rate": 1.8779779118983867e-07, + "loss": 0.0499, + "reward": -0.03433734131976962, + "reward_after_mean": -0.03433734131976962, + "reward_after_std": 0.6416803412139416, + "reward_before_mean": 0.2960522407665849, + "reward_before_std": 0.5826444877311587, + "reward_change_max": 0.0, + "reward_change_mean": -0.3303895927965641, + "reward_change_min": -0.4872638136148453, + "reward_change_std": 0.19508948456496, + "reward_std": 0.6416803747415543, + "rewards/accuracy_reward": 0.25000000931322575, + "rewards/cosine_scaled_reward": 0.04605222793179564, + "step": 410 + }, + { + "clip_fraction": 0.0, + "completion_length": 3163.1041717529297, + "epoch": 0.4697142857142857, + "grad_norm": 0.06846357136964798, + "kl": 1.529604196548462e-05, + "lambda_div_used": 0.5874563306570053, + "learning_rate": 1.8594235253127372e-07, + "loss": 0.0214, + "reward": -0.22584301605820656, + "reward_after_mean": -0.22584301605820656, + "reward_after_std": 0.42311797849833965, + "reward_before_mean": 0.0774743240326643, + "reward_before_std": 0.4020446836948395, + "reward_change_max": 0.0, + "reward_change_mean": -0.3033173345029354, + "reward_change_min": -0.48705917969346046, + "reward_change_std": 0.1920422399416566, + "reward_std": 0.42311798594892025, + "rewards/accuracy_reward": 0.1666666716337204, + "rewards/cosine_scaled_reward": -0.08919236063957214, + "step": 411 + }, + { + "clip_fraction": 0.0, + "completion_length": 2888.125015258789, + "epoch": 0.47085714285714286, + "grad_norm": 0.06694348156452179, + "kl": 2.148747444152832e-05, + "lambda_div_used": 0.6005539819598198, + "learning_rate": 1.8410465752883758e-07, + "loss": 0.049, + "reward": -0.21726901549845934, + "reward_after_mean": -0.21726901549845934, + "reward_after_std": 0.48868509009480476, + "reward_before_mean": 0.08159955404698849, + "reward_before_std": 0.463420107960701, + "reward_change_max": 0.0, + "reward_change_mean": -0.2988685816526413, + "reward_change_min": -0.554147582501173, + "reward_change_std": 0.19932363275438547, + "reward_std": 0.48868510872125626, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.06423377990722656, + "step": 412 + }, + { + "clip_fraction": 0.0, + "completion_length": 2530.2708702087402, + "epoch": 0.472, + "grad_norm": 0.08418448269367218, + "kl": 2.208724617958069e-05, + "lambda_div_used": 0.6184473037719727, + "learning_rate": 1.822847957491922e-07, + "loss": 0.0462, + "reward": 0.05549921467900276, + "reward_after_mean": 0.05549921467900276, + "reward_after_std": 0.6010984163731337, + "reward_before_mean": 0.4671566132456064, + "reward_before_std": 0.546179112046957, + "reward_change_max": 0.0, + "reward_change_mean": -0.4116574004292488, + "reward_change_min": -0.6464647352695465, + "reward_change_std": 0.253952544182539, + "reward_std": 0.6010984499007463, + "rewards/accuracy_reward": 0.37500000931322575, + "rewards/cosine_scaled_reward": 0.09215660532936454, + "step": 413 + }, + { + "clip_fraction": 0.0, + "completion_length": 2948.562530517578, + "epoch": 0.47314285714285714, + "grad_norm": 0.06618716567754745, + "kl": 1.0458752512931824e-05, + "lambda_div_used": 0.6192215830087662, + "learning_rate": 1.804828558898332e-07, + "loss": -0.0483, + "reward": -0.21263186633586884, + "reward_after_mean": -0.21263186633586884, + "reward_after_std": 0.5998369809240103, + "reward_before_mean": 0.04798411298543215, + "reward_before_std": 0.555210480466485, + "reward_change_max": 0.0, + "reward_change_mean": -0.2606159746646881, + "reward_change_min": -0.4564817361533642, + "reward_change_std": 0.16899613942950964, + "reward_std": 0.5998369976878166, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.0978492172434926, + "step": 414 + }, + { + "clip_fraction": 0.0, + "completion_length": 3234.8125, + "epoch": 0.4742857142857143, + "grad_norm": 0.06085793301463127, + "kl": 2.8954818844795227e-05, + "lambda_div_used": 0.5570018589496613, + "learning_rate": 1.7869892577476722e-07, + "loss": 0.0073, + "reward": -0.46066740388050675, + "reward_after_mean": -0.46066740388050675, + "reward_after_std": 0.3419057931751013, + "reward_before_mean": -0.2106443401426077, + "reward_before_std": 0.2546057654544711, + "reward_change_max": 0.0, + "reward_change_mean": -0.2500230632722378, + "reward_change_min": -0.36453111097216606, + "reward_change_std": 0.1324605904519558, + "reward_std": 0.3419058118015528, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.23147768154740334, + "step": 415 + }, + { + "clip_fraction": 0.0, + "completion_length": 1748.9583892822266, + "epoch": 0.4754285714285714, + "grad_norm": 0.09120064228773117, + "kl": 8.981674909591675e-06, + "lambda_div_used": 0.6067373231053352, + "learning_rate": 1.7693309235023127e-07, + "loss": -0.0771, + "reward": -0.13937147613614798, + "reward_after_mean": -0.13937147613614798, + "reward_after_std": 0.6240962240844965, + "reward_before_mean": 0.20647221896797419, + "reward_before_std": 0.48884215706493706, + "reward_change_max": 0.0, + "reward_change_mean": -0.34584368020296097, + "reward_change_min": -0.5141387544572353, + "reward_change_std": 0.18731264490634203, + "reward_std": 0.6240962333977222, + "rewards/accuracy_reward": 0.2500000037252903, + "rewards/cosine_scaled_reward": -0.04352780181216076, + "step": 416 + }, + { + "clip_fraction": 0.0, + "completion_length": 3307.250015258789, + "epoch": 0.4765714285714286, + "grad_norm": 0.05766940861940384, + "kl": 3.1495699658989906e-05, + "lambda_div_used": 0.5544050261378288, + "learning_rate": 1.7518544168045524e-07, + "loss": -0.0019, + "reward": -0.4980150870978832, + "reward_after_mean": -0.4980150870978832, + "reward_after_std": 0.3123048096895218, + "reward_before_mean": -0.2511206082999706, + "reward_before_std": 0.24326619878411293, + "reward_change_max": 0.0, + "reward_change_mean": -0.24689447320997715, + "reward_change_min": -0.3710709176957607, + "reward_change_std": 0.13346257898956537, + "reward_std": 0.3123048171401024, + "rewards/accuracy_reward": 0.02083333395421505, + "rewards/cosine_scaled_reward": -0.2719539441168308, + "step": 417 + }, + { + "clip_fraction": 0.0, + "completion_length": 2171.625015258789, + "epoch": 0.4777142857142857, + "grad_norm": 0.0854194164276123, + "kl": 2.8558075428009033e-05, + "lambda_div_used": 0.6033797562122345, + "learning_rate": 1.7345605894346726e-07, + "loss": 0.0073, + "reward": -0.0949536501429975, + "reward_after_mean": -0.0949536501429975, + "reward_after_std": 0.540143633261323, + "reward_before_mean": 0.25413690507411957, + "reward_before_std": 0.4725718079134822, + "reward_change_max": 0.0, + "reward_change_mean": -0.34909053333103657, + "reward_change_min": -0.5411153584718704, + "reward_change_std": 0.2063078135251999, + "reward_std": 0.5401436407119036, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": -0.03752976981922984, + "step": 418 + }, + { + "clip_fraction": 0.0, + "completion_length": 2464.3750228881836, + "epoch": 0.47885714285714287, + "grad_norm": 0.08315848559141159, + "kl": 3.0182301998138428e-05, + "lambda_div_used": 0.5937136337161064, + "learning_rate": 1.7174502842694212e-07, + "loss": 0.03, + "reward": -0.27206650376319885, + "reward_after_mean": -0.27206650376319885, + "reward_after_std": 0.49108788557350636, + "reward_before_mean": 0.0038498505018651485, + "reward_before_std": 0.4298001816496253, + "reward_change_max": 0.0, + "reward_change_mean": -0.27591635659337044, + "reward_change_min": -0.4215012192726135, + "reward_change_std": 0.16396540496498346, + "reward_std": 0.49108790047466755, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.12115013878792524, + "step": 419 + }, + { + "clip_fraction": 0.0, + "completion_length": 1972.7083625793457, + "epoch": 0.48, + "grad_norm": 0.11145550012588501, + "kl": 3.493949770927429e-05, + "lambda_div_used": 0.5797194987535477, + "learning_rate": 1.7005243352409333e-07, + "loss": -0.0607, + "reward": -0.1652970388531685, + "reward_after_mean": -0.1652970388531685, + "reward_after_std": 0.4477591011673212, + "reward_before_mean": 0.18980162939988077, + "reward_before_std": 0.36517443507909775, + "reward_change_max": 0.0, + "reward_change_mean": -0.35509867407381535, + "reward_change_min": -0.5067496970295906, + "reward_change_std": 0.19984195847064257, + "reward_std": 0.4477591197937727, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.03936504106968641, + "step": 420 + }, + { + "clip_fraction": 0.0, + "completion_length": 3191.937515258789, + "epoch": 0.48114285714285715, + "grad_norm": 0.07170204073190689, + "kl": 3.11434268951416e-05, + "lambda_div_used": 0.5753046199679375, + "learning_rate": 1.6837835672960831e-07, + "loss": 0.0088, + "reward": -0.38796948781237006, + "reward_after_mean": -0.38796948781237006, + "reward_after_std": 0.4225973077118397, + "reward_before_mean": -0.12759371474385262, + "reward_before_std": 0.3405588921159506, + "reward_change_max": 0.0, + "reward_change_mean": -0.2603757604956627, + "reward_change_min": -0.37436668202281, + "reward_change_std": 0.13996374886482954, + "reward_std": 0.4225973132997751, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.16926039289683104, + "step": 421 + }, + { + "clip_fraction": 0.0, + "completion_length": 2776.437545776367, + "epoch": 0.48228571428571426, + "grad_norm": 0.06681375205516815, + "kl": 1.9788742065429688e-05, + "lambda_div_used": 0.5904401987791061, + "learning_rate": 1.6672287963562852e-07, + "loss": 0.0293, + "reward": -0.25396816432476044, + "reward_after_mean": -0.25396816432476044, + "reward_after_std": 0.4787884410470724, + "reward_before_mean": 0.03486193250864744, + "reward_before_std": 0.4093271289020777, + "reward_change_max": 0.0, + "reward_change_mean": -0.2888300847262144, + "reward_change_min": -0.43693453073501587, + "reward_change_std": 0.16317449882626534, + "reward_std": 0.4787884559482336, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.11097141215577722, + "step": 422 + }, + { + "clip_fraction": 0.0, + "completion_length": 2945.9583740234375, + "epoch": 0.48342857142857143, + "grad_norm": 0.06767871230840683, + "kl": 2.205371856689453e-05, + "lambda_div_used": 0.611551009118557, + "learning_rate": 1.6508608292777203e-07, + "loss": 0.0153, + "reward": -0.16482537053525448, + "reward_after_mean": -0.16482537053525448, + "reward_after_std": 0.5363675616681576, + "reward_before_mean": 0.12604539189487696, + "reward_before_std": 0.518328445032239, + "reward_change_max": 0.0, + "reward_change_mean": -0.2908707782626152, + "reward_change_min": -0.4775175042450428, + "reward_change_std": 0.18889948446303606, + "reward_std": 0.5363675802946091, + "rewards/accuracy_reward": 0.16666666977107525, + "rewards/cosine_scaled_reward": -0.040621266700327396, + "step": 423 + }, + { + "clip_fraction": 0.0, + "completion_length": 3215.0208740234375, + "epoch": 0.4845714285714286, + "grad_norm": 0.061716482043266296, + "kl": 1.1460855603218079e-05, + "lambda_div_used": 0.5937408357858658, + "learning_rate": 1.6346804638120098e-07, + "loss": 0.0327, + "reward": -0.3415182586759329, + "reward_after_mean": -0.3415182586759329, + "reward_after_std": 0.48476750776171684, + "reward_before_mean": -0.08947536488994956, + "reward_before_std": 0.4272408355027437, + "reward_change_max": 0.0, + "reward_change_mean": -0.2520429063588381, + "reward_change_min": -0.42084335163235664, + "reward_change_std": 0.15120856929570436, + "reward_std": 0.4847675133496523, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.17280870315153152, + "step": 424 + }, + { + "clip_fraction": 0.0, + "completion_length": 2154.9375610351562, + "epoch": 0.4857142857142857, + "grad_norm": 0.09371069073677063, + "kl": 3.830809146165848e-05, + "lambda_div_used": 0.6315138638019562, + "learning_rate": 1.6186884885673413e-07, + "loss": 0.0058, + "reward": 0.34828917868435383, + "reward_after_mean": 0.34828917868435383, + "reward_after_std": 0.7491090279072523, + "reward_before_mean": 0.91605463065207, + "reward_before_std": 0.6105400957167149, + "reward_change_max": 0.0, + "reward_change_mean": -0.5677654705941677, + "reward_change_min": -0.9185826852917671, + "reward_change_std": 0.34253316558897495, + "reward_std": 0.7491090428084135, + "rewards/accuracy_reward": 0.5833333358168602, + "rewards/cosine_scaled_reward": 0.3327212668955326, + "step": 425 + }, + { + "clip_fraction": 0.0, + "completion_length": 2205.3125190734863, + "epoch": 0.4868571428571429, + "grad_norm": 0.09298980236053467, + "kl": 2.3175030946731567e-05, + "lambda_div_used": 0.6215192526578903, + "learning_rate": 1.6028856829700258e-07, + "loss": -0.002, + "reward": -0.029936233535408974, + "reward_after_mean": -0.029936233535408974, + "reward_after_std": 0.5604026541113853, + "reward_before_mean": 0.3139693345874548, + "reward_before_std": 0.5635973755270243, + "reward_change_max": 0.0, + "reward_change_mean": -0.34390556812286377, + "reward_change_min": -0.5907084755599499, + "reward_change_std": 0.23498705588281155, + "reward_std": 0.5604026671499014, + "rewards/accuracy_reward": 0.3125000074505806, + "rewards/cosine_scaled_reward": 0.0014693308621644974, + "step": 426 + }, + { + "clip_fraction": 0.0, + "completion_length": 3295.7708740234375, + "epoch": 0.488, + "grad_norm": 0.0548052154481411, + "kl": 2.4527311325073242e-05, + "lambda_div_used": 0.5990054532885551, + "learning_rate": 1.5872728172265146e-07, + "loss": 0.0088, + "reward": -0.2071160115301609, + "reward_after_mean": -0.2071160115301609, + "reward_after_std": 0.49672279693186283, + "reward_before_mean": 0.08893106225878, + "reward_before_std": 0.454012256115675, + "reward_change_max": 0.0, + "reward_change_mean": -0.2960470784455538, + "reward_change_min": -0.4723533205688, + "reward_change_std": 0.18442231137305498, + "reward_std": 0.4967228155583143, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.09856893587857485, + "step": 427 + }, + { + "clip_fraction": 0.0, + "completion_length": 2302.520881652832, + "epoch": 0.48914285714285716, + "grad_norm": 0.11483462899923325, + "kl": 0.000337366946041584, + "lambda_div_used": 0.5547136962413788, + "learning_rate": 1.5718506522858572e-07, + "loss": 0.0311, + "reward": -0.42663199454545975, + "reward_after_mean": -0.42663199454545975, + "reward_after_std": 0.29241302236914635, + "reward_before_mean": -0.1547260768711567, + "reward_before_std": 0.24498768709599972, + "reward_change_max": 0.0, + "reward_change_mean": -0.27190591767430305, + "reward_change_min": -0.42438486963510513, + "reward_change_std": 0.1579811777919531, + "reward_std": 0.2924130354076624, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.2380594089627266, + "step": 428 + }, + { + "clip_fraction": 0.0, + "completion_length": 2431.520881652832, + "epoch": 0.49028571428571427, + "grad_norm": 0.0878632664680481, + "kl": 2.4685636162757874e-05, + "lambda_div_used": 0.6025609895586967, + "learning_rate": 1.5566199398026147e-07, + "loss": 0.0378, + "reward": -0.057155030546709895, + "reward_after_mean": -0.057155030546709895, + "reward_after_std": 0.5567535478621721, + "reward_before_mean": 0.3208682704716921, + "reward_before_std": 0.47341752983629704, + "reward_change_max": 0.0, + "reward_change_mean": -0.3780232574790716, + "reward_change_min": -0.5763550102710724, + "reward_change_std": 0.22364642471075058, + "reward_std": 0.5567535553127527, + "rewards/accuracy_reward": 0.29166666977107525, + "rewards/cosine_scaled_reward": 0.029201554832980037, + "step": 429 + }, + { + "clip_fraction": 0.0, + "completion_length": 2577.3125610351562, + "epoch": 0.49142857142857144, + "grad_norm": 0.08103640377521515, + "kl": 1.4763325452804565e-05, + "lambda_div_used": 0.6012379750609398, + "learning_rate": 1.5415814221002265e-07, + "loss": -0.0049, + "reward": 0.031602535396814346, + "reward_after_mean": 0.031602535396814346, + "reward_after_std": 0.5440461356192827, + "reward_before_mean": 0.4564003311097622, + "reward_before_std": 0.46839726250618696, + "reward_change_max": 0.0, + "reward_change_mean": -0.42479780688881874, + "reward_change_min": -0.6648109555244446, + "reward_change_std": 0.2579617351293564, + "reward_std": 0.5440461542457342, + "rewards/accuracy_reward": 0.3541666753590107, + "rewards/cosine_scaled_reward": 0.1022336557507515, + "step": 430 + }, + { + "clip_fraction": 0.0, + "completion_length": 2690.6458740234375, + "epoch": 0.49257142857142855, + "grad_norm": 0.06538030505180359, + "kl": 2.8170645236968994e-05, + "lambda_div_used": 0.5918548330664635, + "learning_rate": 1.5267358321348285e-07, + "loss": -0.0328, + "reward": -0.24143310775980353, + "reward_after_mean": -0.24143310775980353, + "reward_after_std": 0.481786971911788, + "reward_before_mean": 0.05766227189451456, + "reward_before_std": 0.41715206764638424, + "reward_change_max": 0.0, + "reward_change_mean": -0.29909538850188255, + "reward_change_min": -0.4450513869524002, + "reward_change_std": 0.1692206682637334, + "reward_std": 0.4817869979888201, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.08817105047637597, + "step": 431 + }, + { + "clip_fraction": 0.0, + "completion_length": 2843.750045776367, + "epoch": 0.4937142857142857, + "grad_norm": 0.05780716612935066, + "kl": 3.104284405708313e-05, + "lambda_div_used": 0.576793298125267, + "learning_rate": 1.5120838934595337e-07, + "loss": 0.0053, + "reward": -0.42155745439231396, + "reward_after_mean": -0.42155745439231396, + "reward_after_std": 0.4230448566377163, + "reward_before_mean": -0.19034346495755017, + "reward_before_std": 0.35024640895426273, + "reward_change_max": 0.0, + "reward_change_mean": -0.23121398873627186, + "reward_change_min": -0.3715337961912155, + "reward_change_std": 0.1338915005326271, + "reward_std": 0.4230448678135872, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.2528434665873647, + "step": 432 + }, + { + "clip_fraction": 0.0, + "completion_length": 2992.062530517578, + "epoch": 0.4948571428571429, + "grad_norm": 0.06562553346157074, + "kl": 3.0972063541412354e-05, + "lambda_div_used": 0.6079665347933769, + "learning_rate": 1.4976263201891613e-07, + "loss": 0.1201, + "reward": -0.26266857516020536, + "reward_after_mean": -0.26266857516020536, + "reward_after_std": 0.5306479204446077, + "reward_before_mean": -0.0102414321154356, + "reward_before_std": 0.5030816271901131, + "reward_change_max": 0.0, + "reward_change_mean": -0.25242713652551174, + "reward_change_min": -0.4955142140388489, + "reward_change_std": 0.1781205264851451, + "reward_std": 0.5306479260325432, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.15607476886361837, + "step": 433 + }, + { + "clip_fraction": 0.0, + "completion_length": 2915.625030517578, + "epoch": 0.496, + "grad_norm": 0.06934478878974915, + "kl": 3.8422178477048874e-05, + "lambda_div_used": 0.5604980885982513, + "learning_rate": 1.483363816965435e-07, + "loss": -0.004, + "reward": -0.4280230412259698, + "reward_after_mean": -0.4280230412259698, + "reward_after_std": 0.32627478428184986, + "reward_before_mean": -0.15262745507061481, + "reward_before_std": 0.2731653768569231, + "reward_change_max": 0.0, + "reward_change_mean": -0.27539557591080666, + "reward_change_min": -0.44822341948747635, + "reward_change_std": 0.16339552495628595, + "reward_std": 0.32627478800714016, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.1942941201850772, + "step": 434 + }, + { + "clip_fraction": 0.0, + "completion_length": 2320.750026702881, + "epoch": 0.49714285714285716, + "grad_norm": 0.13675029575824738, + "kl": 4.8140063881874084e-05, + "lambda_div_used": 0.5811168253421783, + "learning_rate": 1.469297078922642e-07, + "loss": -0.0374, + "reward": -0.17623315937817097, + "reward_after_mean": -0.17623315937817097, + "reward_after_std": 0.483237961307168, + "reward_before_mean": 0.19585114251822233, + "reward_before_std": 0.3759047882631421, + "reward_change_max": 0.0, + "reward_change_mean": -0.37208431400358677, + "reward_change_min": -0.5468379594385624, + "reward_change_std": 0.20962903555482626, + "reward_std": 0.4832379762083292, + "rewards/accuracy_reward": 0.20833333395421505, + "rewards/cosine_scaled_reward": -0.01248218398541212, + "step": 435 + }, + { + "clip_fraction": 0.0, + "completion_length": 2316.3125381469727, + "epoch": 0.4982857142857143, + "grad_norm": 0.09783437103033066, + "kl": 2.839416265487671e-05, + "lambda_div_used": 0.6289880573749542, + "learning_rate": 1.4554267916537495e-07, + "loss": 0.0826, + "reward": 0.12041758373379707, + "reward_after_mean": 0.12041758373379707, + "reward_after_std": 0.6426001656800508, + "reward_before_mean": 0.5348608233034611, + "reward_before_std": 0.6021685730665922, + "reward_change_max": 0.0, + "reward_change_mean": -0.41444322280585766, + "reward_change_min": -0.6993351243436337, + "reward_change_std": 0.271469890139997, + "reward_std": 0.6426001694053411, + "rewards/accuracy_reward": 0.3958333395421505, + "rewards/cosine_scaled_reward": 0.13902746886014938, + "step": 436 + }, + { + "clip_fraction": 0.0, + "completion_length": 2954.0416717529297, + "epoch": 0.49942857142857144, + "grad_norm": 0.05496148392558098, + "kl": 0.00023943744599819183, + "lambda_div_used": 0.5626013651490211, + "learning_rate": 1.4417536311769885e-07, + "loss": 0.05, + "reward": -0.42826351523399353, + "reward_after_mean": -0.42826351523399353, + "reward_after_std": 0.33650987036526203, + "reward_before_mean": -0.16868688352406025, + "reward_before_std": 0.2832178361713886, + "reward_change_max": 0.0, + "reward_change_mean": -0.25957662612199783, + "reward_change_min": -0.42033713683485985, + "reward_change_std": 0.1560918828472495, + "reward_std": 0.3365098759531975, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.2311868779361248, + "step": 437 + }, + { + "clip_fraction": 0.0, + "completion_length": 3007.500015258789, + "epoch": 0.5005714285714286, + "grad_norm": 0.05682806298136711, + "kl": 4.358415026217699e-06, + "lambda_div_used": 0.6039799600839615, + "learning_rate": 1.4282782639029128e-07, + "loss": 0.0289, + "reward": -0.29300207551568747, + "reward_after_mean": -0.29300207551568747, + "reward_after_std": 0.5204536523669958, + "reward_before_mean": -0.03094739466905594, + "reward_before_std": 0.4829921592026949, + "reward_change_max": 0.0, + "reward_change_mean": -0.2620546855032444, + "reward_change_min": -0.5145607963204384, + "reward_change_std": 0.18371508549898863, + "reward_std": 0.5204536579549313, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.15594739385414869, + "step": 438 + }, + { + "clip_fraction": 0.0, + "completion_length": 2249.5000076293945, + "epoch": 0.5017142857142857, + "grad_norm": 0.0828532725572586, + "kl": 3.217160701751709e-05, + "lambda_div_used": 0.5708463862538338, + "learning_rate": 1.4150013466019114e-07, + "loss": 0.021, + "reward": -0.40206424333155155, + "reward_after_mean": -0.40206424333155155, + "reward_after_std": 0.40689974650740623, + "reward_before_mean": -0.14063972979784012, + "reward_before_std": 0.32079045102000237, + "reward_change_max": 0.0, + "reward_change_mean": -0.261424507945776, + "reward_change_min": -0.38026969507336617, + "reward_change_std": 0.14067976083606482, + "reward_std": 0.406899768859148, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.1823064020718448, + "step": 439 + }, + { + "clip_fraction": 0.0, + "completion_length": 2964.000011444092, + "epoch": 0.5028571428571429, + "grad_norm": 0.07350306212902069, + "kl": 2.7701258659362793e-05, + "lambda_div_used": 0.5365751013159752, + "learning_rate": 1.4019235263722034e-07, + "loss": 0.0201, + "reward": -0.5367953963577747, + "reward_after_mean": -0.5367953963577747, + "reward_after_std": 0.231884878128767, + "reward_before_mean": -0.28081973269581795, + "reward_before_std": 0.16059848852455616, + "reward_change_max": 0.0, + "reward_change_mean": -0.25597566179931164, + "reward_change_min": -0.38940757140517235, + "reward_change_std": 0.13862022012472153, + "reward_std": 0.2318848893046379, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.28081973642110825, + "step": 440 + }, + { + "clip_fraction": 0.0, + "completion_length": 3007.770896911621, + "epoch": 0.504, + "grad_norm": 0.07498018443584442, + "kl": 2.944841980934143e-05, + "lambda_div_used": 0.6011143997311592, + "learning_rate": 1.3890454406082956e-07, + "loss": -0.0116, + "reward": -0.13889071717858315, + "reward_after_mean": -0.13889071717858315, + "reward_after_std": 0.4864419028162956, + "reward_before_mean": 0.17084867507219315, + "reward_before_std": 0.4689741334877908, + "reward_change_max": 0.0, + "reward_change_mean": -0.3097394183278084, + "reward_change_min": -0.514279019087553, + "reward_change_std": 0.20097953081130981, + "reward_std": 0.48644191212952137, + "rewards/accuracy_reward": 0.2291666753590107, + "rewards/cosine_scaled_reward": -0.05831799004226923, + "step": 441 + }, + { + "clip_fraction": 0.0, + "completion_length": 2837.4583435058594, + "epoch": 0.5051428571428571, + "grad_norm": 0.0673719272017479, + "kl": 3.160163760185242e-05, + "lambda_div_used": 0.5689455196261406, + "learning_rate": 1.3763677169699217e-07, + "loss": 0.0084, + "reward": -0.31079866108484566, + "reward_after_mean": -0.31079866108484566, + "reward_after_std": 0.3982803635299206, + "reward_before_mean": -0.010625829687342048, + "reward_before_std": 0.31661996035836637, + "reward_change_max": 0.0, + "reward_change_mean": -0.30017283000051975, + "reward_change_min": -0.4261031448841095, + "reward_change_std": 0.16800945159047842, + "reward_std": 0.3982803765684366, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.1147925069089979, + "step": 442 + }, + { + "clip_fraction": 0.0, + "completion_length": 3194.270854949951, + "epoch": 0.5062857142857143, + "grad_norm": 0.05984271690249443, + "kl": 1.745857298374176e-05, + "lambda_div_used": 0.5607094466686249, + "learning_rate": 1.3638909733514452e-07, + "loss": 0.0053, + "reward": -0.38279488682746887, + "reward_after_mean": -0.38279488682746887, + "reward_after_std": 0.33906700275838375, + "reward_before_mean": -0.09817820321768522, + "reward_before_std": 0.2764374865218997, + "reward_change_max": 0.0, + "reward_change_mean": -0.2846166864037514, + "reward_change_min": -0.4438001811504364, + "reward_change_std": 0.16594033408910036, + "reward_std": 0.3390670083463192, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.16067820694297552, + "step": 443 + }, + { + "clip_fraction": 0.0, + "completion_length": 2902.2291717529297, + "epoch": 0.5074285714285715, + "grad_norm": 0.0638304203748703, + "kl": 1.5033408999443054e-05, + "lambda_div_used": 0.5774018093943596, + "learning_rate": 1.351615817851748e-07, + "loss": -0.0355, + "reward": -0.3352237604558468, + "reward_after_mean": -0.3352237604558468, + "reward_after_std": 0.4039857666939497, + "reward_before_mean": -0.05285666696727276, + "reward_before_std": 0.35096561443060637, + "reward_change_max": 0.0, + "reward_change_mean": -0.2823671065270901, + "reward_change_min": -0.42594215646386147, + "reward_change_std": 0.16656992863863707, + "reward_std": 0.40398577041924, + "rewards/accuracy_reward": 0.10416666977107525, + "rewards/cosine_scaled_reward": -0.15702332742512226, + "step": 444 + }, + { + "clip_fraction": 0.0, + "completion_length": 3006.916717529297, + "epoch": 0.5085714285714286, + "grad_norm": 0.06397882103919983, + "kl": 2.842303365468979e-05, + "lambda_div_used": 0.6194909662008286, + "learning_rate": 1.3395428487445914e-07, + "loss": 0.0715, + "reward": -0.05263347551226616, + "reward_after_mean": -0.05263347551226616, + "reward_after_std": 0.5679135732352734, + "reward_before_mean": 0.2764017879962921, + "reward_before_std": 0.5510673765093088, + "reward_change_max": 0.0, + "reward_change_mean": -0.3290352523326874, + "reward_change_min": -0.5197134874761105, + "reward_change_std": 0.21186745446175337, + "reward_std": 0.5679135844111443, + "rewards/accuracy_reward": 0.27083334140479565, + "rewards/cosine_scaled_reward": 0.0055684298276901245, + "step": 445 + }, + { + "clip_fraction": 0.0, + "completion_length": 3028.8333892822266, + "epoch": 0.5097142857142857, + "grad_norm": 0.06140498071908951, + "kl": 3.2648444175720215e-05, + "lambda_div_used": 0.606703408062458, + "learning_rate": 1.3276726544494571e-07, + "loss": -0.0241, + "reward": -0.2073356769979, + "reward_after_mean": -0.2073356769979, + "reward_after_std": 0.5240648984909058, + "reward_before_mean": 0.0723939798772335, + "reward_before_std": 0.4952498711645603, + "reward_change_max": 0.0, + "reward_change_mean": -0.27972964756190777, + "reward_change_min": -0.5379137881100178, + "reward_change_std": 0.1909602265805006, + "reward_std": 0.5240649078041315, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.07343936339020729, + "step": 446 + }, + { + "clip_fraction": 0.0, + "completion_length": 2571.166721343994, + "epoch": 0.5108571428571429, + "grad_norm": 0.08778514713048935, + "kl": 9.072478860616684e-06, + "lambda_div_used": 0.5970501974225044, + "learning_rate": 1.316005813502869e-07, + "loss": 0.0018, + "reward": -0.19514422863721848, + "reward_after_mean": -0.19514422863721848, + "reward_after_std": 0.4898754768073559, + "reward_before_mean": 0.10967571474611759, + "reward_before_std": 0.4461768325418234, + "reward_change_max": 0.0, + "reward_change_mean": -0.3048199340701103, + "reward_change_min": -0.48957687243819237, + "reward_change_std": 0.18721044715493917, + "reward_std": 0.4898754861205816, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.07782429456710815, + "step": 447 + }, + { + "clip_fraction": 0.0, + "completion_length": 2039.6042022705078, + "epoch": 0.512, + "grad_norm": 0.12122303992509842, + "kl": 3.530830144882202e-05, + "lambda_div_used": 0.5591797679662704, + "learning_rate": 1.3045428945301953e-07, + "loss": 0.0057, + "reward": -0.3564775697886944, + "reward_after_mean": -0.3564775697886944, + "reward_after_std": 0.33722602762281895, + "reward_before_mean": -0.06660962291061878, + "reward_before_std": 0.27086107339709997, + "reward_change_max": 0.0, + "reward_change_mean": -0.28986795619130135, + "reward_change_min": -0.4345347508788109, + "reward_change_std": 0.1656003799289465, + "reward_std": 0.33722603134810925, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.14994295500218868, + "step": 448 + }, + { + "clip_fraction": 0.0, + "completion_length": 2680.0208435058594, + "epoch": 0.5131428571428571, + "grad_norm": 0.07745695859193802, + "kl": 3.194063901901245e-05, + "lambda_div_used": 0.5827131420373917, + "learning_rate": 1.2932844562179352e-07, + "loss": -0.0247, + "reward": -0.278532937169075, + "reward_after_mean": -0.278532937169075, + "reward_after_std": 0.44566163793206215, + "reward_before_mean": 0.01611769199371338, + "reward_before_std": 0.3775772461667657, + "reward_change_max": 0.0, + "reward_change_mean": -0.29465061984956264, + "reward_change_min": -0.4545559212565422, + "reward_change_std": 0.17489958554506302, + "reward_std": 0.44566163793206215, + "rewards/accuracy_reward": 0.18750000558793545, + "rewards/cosine_scaled_reward": -0.17138232477009296, + "step": 449 + }, + { + "clip_fraction": 0.0, + "completion_length": 2405.0833702087402, + "epoch": 0.5142857142857142, + "grad_norm": 0.07359456270933151, + "kl": 4.271417856216431e-05, + "lambda_div_used": 0.5313882529735565, + "learning_rate": 1.2822310472864885e-07, + "loss": 0.0047, + "reward": -0.40455422177910805, + "reward_after_mean": -0.40455422177910805, + "reward_after_std": 0.27055008336901665, + "reward_before_mean": -0.05470774322748184, + "reward_before_std": 0.13779542688280344, + "reward_change_max": 0.0, + "reward_change_mean": -0.34984651021659374, + "reward_change_min": -0.482446551322937, + "reward_change_std": 0.18013106007128954, + "reward_std": 0.2705500964075327, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.17970772832632065, + "step": 450 + }, + { + "clip_fraction": 0.0, + "completion_length": 2726.7916870117188, + "epoch": 0.5154285714285715, + "grad_norm": 0.10456772893667221, + "kl": 5.123019218444824e-05, + "lambda_div_used": 0.5659012496471405, + "learning_rate": 1.2713832064634125e-07, + "loss": 0.0437, + "reward": -0.233762688934803, + "reward_after_mean": -0.233762688934803, + "reward_after_std": 0.3937326893210411, + "reward_before_mean": 0.14449233608320355, + "reward_before_std": 0.30359079129993916, + "reward_change_max": 0.0, + "reward_change_mean": -0.3782550562173128, + "reward_change_min": -0.5748258531093597, + "reward_change_std": 0.21745321340858936, + "reward_std": 0.3937326930463314, + "rewards/accuracy_reward": 0.1875, + "rewards/cosine_scaled_reward": -0.0430076620541513, + "step": 451 + }, + { + "clip_fraction": 0.0, + "completion_length": 3173.812530517578, + "epoch": 0.5165714285714286, + "grad_norm": 0.05382152646780014, + "kl": 1.621060073375702e-05, + "lambda_div_used": 0.628274716436863, + "learning_rate": 1.260741462457165e-07, + "loss": 0.0152, + "reward": 0.08829959481954575, + "reward_after_mean": 0.08829959481954575, + "reward_after_std": 0.6491915434598923, + "reward_before_mean": 0.49067158019170165, + "reward_before_std": 0.5948440972715616, + "reward_change_max": 0.0, + "reward_change_mean": -0.4023719858378172, + "reward_change_min": -0.6429549157619476, + "reward_change_std": 0.2526876190677285, + "reward_std": 0.6491915658116341, + "rewards/accuracy_reward": 0.37500000558793545, + "rewards/cosine_scaled_reward": 0.11567158252000809, + "step": 452 + }, + { + "clip_fraction": 0.0, + "completion_length": 2601.1666870117188, + "epoch": 0.5177142857142857, + "grad_norm": 0.10710335522890091, + "kl": 4.57763671875e-05, + "lambda_div_used": 0.5769856572151184, + "learning_rate": 1.2503063339313356e-07, + "loss": -0.0233, + "reward": -0.4120060931891203, + "reward_after_mean": -0.4120060931891203, + "reward_after_std": 0.4247955661267042, + "reward_before_mean": -0.17395742796361446, + "reward_before_std": 0.3489131908863783, + "reward_change_max": 0.0, + "reward_change_mean": -0.23804865032434464, + "reward_change_min": -0.3744778670370579, + "reward_change_std": 0.12989939749240875, + "reward_std": 0.4247955847531557, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.21562409959733486, + "step": 453 + }, + { + "clip_fraction": 0.0, + "completion_length": 2794.8958740234375, + "epoch": 0.5188571428571429, + "grad_norm": 0.059399981051683426, + "kl": 1.829490065574646e-05, + "lambda_div_used": 0.604412779211998, + "learning_rate": 1.2400783294793668e-07, + "loss": 0.0084, + "reward": -0.23108407109975815, + "reward_after_mean": -0.23108407109975815, + "reward_after_std": 0.5207228269428015, + "reward_before_mean": 0.05918463226407766, + "reward_before_std": 0.4853199487552047, + "reward_change_max": 0.0, + "reward_change_mean": -0.29026869870722294, + "reward_change_min": -0.5276229903101921, + "reward_change_std": 0.1960765514522791, + "reward_std": 0.5207228306680918, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.08664871653309092, + "step": 454 + }, + { + "clip_fraction": 0.0, + "completion_length": 3120.0416717529297, + "epoch": 0.52, + "grad_norm": 0.07823975384235382, + "kl": 1.1070631444454193e-05, + "lambda_div_used": 0.5381916239857674, + "learning_rate": 1.2300579475997657e-07, + "loss": 0.0035, + "reward": -0.5376988351345062, + "reward_after_mean": -0.5376988351345062, + "reward_after_std": 0.2399882897734642, + "reward_before_mean": -0.28268107399344444, + "reward_before_std": 0.16770172398537397, + "reward_change_max": 0.0, + "reward_change_mean": -0.25501775927841663, + "reward_change_min": -0.38403724879026413, + "reward_change_std": 0.1382020702585578, + "reward_std": 0.23998829536139965, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.28268107026815414, + "step": 455 + }, + { + "clip_fraction": 0.0, + "completion_length": 3183.125, + "epoch": 0.5211428571428571, + "grad_norm": 0.07441714406013489, + "kl": 1.7248094081878662e-05, + "lambda_div_used": 0.535825714468956, + "learning_rate": 1.220245676671809e-07, + "loss": -0.0061, + "reward": -0.5242688432335854, + "reward_after_mean": -0.5242688432335854, + "reward_after_std": 0.2355129700154066, + "reward_before_mean": -0.2603513076901436, + "reward_before_std": 0.15733365854248405, + "reward_change_max": 0.0, + "reward_change_mean": -0.2639175299555063, + "reward_change_min": -0.3936190530657768, + "reward_change_std": 0.14126356784254313, + "reward_std": 0.23551297932863235, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2603513114154339, + "step": 456 + }, + { + "clip_fraction": 0.0, + "completion_length": 3090.9166717529297, + "epoch": 0.5222857142857142, + "grad_norm": 0.06740739941596985, + "kl": 1.6771256923675537e-05, + "lambda_div_used": 0.5661213397979736, + "learning_rate": 1.2106419949317388e-07, + "loss": 0.0183, + "reward": -0.2666686773300171, + "reward_after_mean": -0.2666686773300171, + "reward_after_std": 0.3630063198506832, + "reward_before_mean": 0.06939902156591415, + "reward_before_std": 0.3003841144964099, + "reward_change_max": 0.0, + "reward_change_mean": -0.3360677044838667, + "reward_change_min": -0.5109980814158916, + "reward_change_std": 0.19570716377347708, + "reward_std": 0.36300632916390896, + "rewards/accuracy_reward": 0.1458333395421505, + "rewards/cosine_scaled_reward": -0.0764343123883009, + "step": 457 + }, + { + "clip_fraction": 0.0, + "completion_length": 2337.0416717529297, + "epoch": 0.5234285714285715, + "grad_norm": 0.10038017481565475, + "kl": 1.5580561012029648e-05, + "lambda_div_used": 0.587464913725853, + "learning_rate": 1.2012473704494537e-07, + "loss": 0.0337, + "reward": -0.2921891398727894, + "reward_after_mean": -0.2921891398727894, + "reward_after_std": 0.46863692067563534, + "reward_before_mean": -0.017078701872378588, + "reward_before_std": 0.3978575337678194, + "reward_change_max": 0.0, + "reward_change_mean": -0.27511043660342693, + "reward_change_min": -0.4165792725980282, + "reward_change_std": 0.15801922790706158, + "reward_std": 0.4686369299888611, + "rewards/accuracy_reward": 0.1250000037252903, + "rewards/cosine_scaled_reward": -0.14207870024256408, + "step": 458 + }, + { + "clip_fraction": 0.0, + "completion_length": 1897.9791870117188, + "epoch": 0.5245714285714286, + "grad_norm": 0.10566994547843933, + "kl": 3.1791627407073975e-05, + "lambda_div_used": 0.6483638733625412, + "learning_rate": 1.1920622611056974e-07, + "loss": -0.0327, + "reward": 0.04763873480260372, + "reward_after_mean": 0.04763873480260372, + "reward_after_std": 0.6893943976610899, + "reward_before_mean": 0.37104589492082596, + "reward_before_std": 0.6905025038868189, + "reward_change_max": 0.0, + "reward_change_mean": -0.3234071359038353, + "reward_change_min": -0.573150984942913, + "reward_change_std": 0.22597133833914995, + "reward_std": 0.6893944274634123, + "rewards/accuracy_reward": 0.29166667349636555, + "rewards/cosine_scaled_reward": 0.07937921397387981, + "step": 459 + }, + { + "clip_fraction": 0.0, + "completion_length": 3152.9583587646484, + "epoch": 0.5257142857142857, + "grad_norm": 0.0553775392472744, + "kl": 1.6976147890090942e-05, + "lambda_div_used": 0.6212490126490593, + "learning_rate": 1.1830871145697412e-07, + "loss": -0.0403, + "reward": -0.2310712798498571, + "reward_after_mean": -0.2310712798498571, + "reward_after_std": 0.5967046339064837, + "reward_before_mean": 0.018506756518036127, + "reward_before_std": 0.5584379080682993, + "reward_change_max": 0.0, + "reward_change_mean": -0.24957803264260292, + "reward_change_min": -0.45073715783655643, + "reward_change_std": 0.16405375488102436, + "reward_std": 0.5967046469449997, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.1273265864001587, + "step": 460 + }, + { + "clip_fraction": 0.0, + "completion_length": 2965.625015258789, + "epoch": 0.5268571428571428, + "grad_norm": 0.0683179497718811, + "kl": 3.898143768310547e-05, + "lambda_div_used": 0.5838751494884491, + "learning_rate": 1.1743223682775649e-07, + "loss": 0.0201, + "reward": -0.36440329253673553, + "reward_after_mean": -0.36440329253673553, + "reward_after_std": 0.44049660488963127, + "reward_before_mean": -0.10854836623184383, + "reward_before_std": 0.3863721750676632, + "reward_change_max": 0.0, + "reward_change_mean": -0.25585492700338364, + "reward_change_min": -0.41783962957561016, + "reward_change_std": 0.15460436698049307, + "reward_std": 0.44049660861492157, + "rewards/accuracy_reward": 0.08333333395421505, + "rewards/cosine_scaled_reward": -0.19188169576227665, + "step": 461 + }, + { + "clip_fraction": 0.0, + "completion_length": 2874.3125076293945, + "epoch": 0.528, + "grad_norm": 0.07832667976617813, + "kl": 2.7470290660858154e-05, + "lambda_div_used": 0.5419389680027962, + "learning_rate": 1.1657684494105386e-07, + "loss": -0.0381, + "reward": -0.5123195722699165, + "reward_after_mean": -0.5123195722699165, + "reward_after_std": 0.2568210382014513, + "reward_before_mean": -0.2502432279288769, + "reward_before_std": 0.1842196974903345, + "reward_change_max": 0.0, + "reward_change_mean": -0.2620763499289751, + "reward_change_min": -0.4005816727876663, + "reward_change_std": 0.14368234388530254, + "reward_std": 0.2568210456520319, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.2502432242035866, + "step": 462 + }, + { + "clip_fraction": 0.0, + "completion_length": 2986.8333740234375, + "epoch": 0.5291428571428571, + "grad_norm": 0.053950026631355286, + "kl": 8.771196007728577e-06, + "lambda_div_used": 0.622698076069355, + "learning_rate": 1.1574257748745986e-07, + "loss": 0.0106, + "reward": -0.2041953857988119, + "reward_after_mean": -0.2041953857988119, + "reward_after_std": 0.6069091446697712, + "reward_before_mean": 0.05050618201494217, + "reward_before_std": 0.5684067364782095, + "reward_change_max": 0.0, + "reward_change_mean": -0.25470156222581863, + "reward_change_min": -0.4568561315536499, + "reward_change_std": 0.16765617858618498, + "reward_std": 0.6069091446697712, + "rewards/accuracy_reward": 0.1458333358168602, + "rewards/cosine_scaled_reward": -0.09532715613022447, + "step": 463 + }, + { + "clip_fraction": 0.0, + "completion_length": 1749.0416717529297, + "epoch": 0.5302857142857142, + "grad_norm": 0.11516103893518448, + "kl": 3.466010093688965e-05, + "lambda_div_used": 0.5341480076313019, + "learning_rate": 1.1492947512799328e-07, + "loss": -0.0088, + "reward": -0.040389321744441986, + "reward_after_mean": -0.040389321744441986, + "reward_after_std": 0.37696985714137554, + "reward_before_mean": 0.5386080192402005, + "reward_before_std": 0.14989514648914337, + "reward_change_max": 0.0, + "reward_change_mean": -0.5789973307400942, + "reward_change_min": -0.7674467526376247, + "reward_change_std": 0.2917799688875675, + "reward_std": 0.37696986459195614, + "rewards/accuracy_reward": 0.375, + "rewards/cosine_scaled_reward": 0.16360801365226507, + "step": 464 + }, + { + "clip_fraction": 0.0, + "completion_length": 3036.854179382324, + "epoch": 0.5314285714285715, + "grad_norm": 0.08283663541078568, + "kl": 2.4955719709396362e-05, + "lambda_div_used": 0.572467640042305, + "learning_rate": 1.1413757749211602e-07, + "loss": 0.0373, + "reward": -0.40219551732297987, + "reward_after_mean": -0.40219551732297987, + "reward_after_std": 0.40797613374888897, + "reward_before_mean": -0.1455877646803856, + "reward_before_std": 0.32708599977195263, + "reward_change_max": 0.0, + "reward_change_mean": -0.256607748568058, + "reward_change_min": -0.37336331233382225, + "reward_change_std": 0.1377662243321538, + "reward_std": 0.40797614119946957, + "rewards/accuracy_reward": 0.0416666679084301, + "rewards/cosine_scaled_reward": -0.18725443072617054, + "step": 465 + }, + { + "clip_fraction": 0.0, + "completion_length": 3122.9583435058594, + "epoch": 0.5325714285714286, + "grad_norm": 0.07085831463336945, + "kl": 2.0582228899002075e-05, + "lambda_div_used": 0.5319224968552589, + "learning_rate": 1.1336692317580158e-07, + "loss": -0.0563, + "reward": -0.36368466913700104, + "reward_after_mean": -0.36368466913700104, + "reward_after_std": 0.27606455981731415, + "reward_before_mean": 0.005439521744847298, + "reward_before_std": 0.14014938473701477, + "reward_change_max": 0.0, + "reward_change_mean": -0.36912417598068714, + "reward_change_min": -0.5061142668128014, + "reward_change_std": 0.1899967910721898, + "reward_std": 0.2760645691305399, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.11956048291176558, + "step": 466 + }, + { + "clip_fraction": 0.0, + "completion_length": 3113.2708854675293, + "epoch": 0.5337142857142857, + "grad_norm": 0.07966674119234085, + "kl": 3.802310675382614e-05, + "lambda_div_used": 0.5612561926245689, + "learning_rate": 1.1261754973965422e-07, + "loss": -0.0067, + "reward": -0.3137255348265171, + "reward_after_mean": -0.3137255348265171, + "reward_after_std": 0.39207486622035503, + "reward_before_mean": 0.03320633992552757, + "reward_before_std": 0.2764817178249359, + "reward_change_max": 0.0, + "reward_change_mean": -0.3469318952411413, + "reward_change_min": -0.4828169047832489, + "reward_change_std": 0.18068420328199863, + "reward_std": 0.3920748811215162, + "rewards/accuracy_reward": 0.14583333395421505, + "rewards/cosine_scaled_reward": -0.11262698657810688, + "step": 467 + }, + { + "clip_fraction": 0.0, + "completion_length": 2898.770854949951, + "epoch": 0.5348571428571428, + "grad_norm": 0.07577298581600189, + "kl": 4.957616329193115e-05, + "lambda_div_used": 0.5363663136959076, + "learning_rate": 1.1188949370707787e-07, + "loss": 0.0338, + "reward": -0.5352834053337574, + "reward_after_mean": -0.5352834053337574, + "reward_after_std": 0.22454985231161118, + "reward_before_mean": -0.2775777019560337, + "reward_before_std": 0.15978037798777223, + "reward_change_max": 0.0, + "reward_change_mean": -0.2577057033777237, + "reward_change_min": -0.3789172098040581, + "reward_change_std": 0.14186344109475613, + "reward_std": 0.22454985417425632, + "rewards/accuracy_reward": 0.0, + "rewards/cosine_scaled_reward": -0.27757770381867886, + "step": 468 + }, + { + "clip_fraction": 0.0, + "completion_length": 2854.8750228881836, + "epoch": 0.536, + "grad_norm": 0.09182782471179962, + "kl": 2.0341947674751282e-05, + "lambda_div_used": 0.5550094842910767, + "learning_rate": 1.1118279056249653e-07, + "loss": 0.0301, + "reward": -0.35969678312540054, + "reward_after_mean": -0.35969678312540054, + "reward_after_std": 0.30523936823010445, + "reward_before_mean": -0.04356633126735687, + "reward_before_std": 0.2476841462776065, + "reward_change_max": 0.0, + "reward_change_mean": -0.31613045185804367, + "reward_change_min": -0.45744797959923744, + "reward_change_std": 0.17962745483964682, + "reward_std": 0.3052393738180399, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.12689966335892677, + "step": 469 + }, + { + "clip_fraction": 0.0, + "completion_length": 3333.562530517578, + "epoch": 0.5371428571428571, + "grad_norm": 0.05610484257340431, + "kl": 2.9811635613441467e-05, + "lambda_div_used": 0.5911690816283226, + "learning_rate": 1.1049747474962444e-07, + "loss": 0.0307, + "reward": -0.37843912467360497, + "reward_after_mean": -0.37843912467360497, + "reward_after_std": 0.49228920973837376, + "reward_before_mean": -0.14763083557772916, + "reward_before_std": 0.4128831513226032, + "reward_change_max": 0.0, + "reward_change_mean": -0.230808287858963, + "reward_change_min": -0.3457215949892998, + "reward_change_std": 0.12627399526536465, + "reward_std": 0.49228921718895435, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.21013083728030324, + "step": 470 + }, + { + "clip_fraction": 0.0, + "completion_length": 3032.312515258789, + "epoch": 0.5382857142857143, + "grad_norm": 0.053547170013189316, + "kl": 2.0138919353485107e-05, + "lambda_div_used": 0.5520770102739334, + "learning_rate": 1.0983357966978745e-07, + "loss": -0.0036, + "reward": -0.32707202807068825, + "reward_after_mean": -0.32707202807068825, + "reward_after_std": 0.30774626694619656, + "reward_before_mean": 0.006061417981982231, + "reward_before_std": 0.23228682670742273, + "reward_change_max": 0.0, + "reward_change_mean": -0.3331334535032511, + "reward_change_min": -0.47613633051514626, + "reward_change_std": 0.181988756172359, + "reward_std": 0.3077462762594223, + "rewards/accuracy_reward": 0.1041666716337204, + "rewards/cosine_scaled_reward": -0.09810524806380272, + "step": 471 + }, + { + "clip_fraction": 0.0, + "completion_length": 3115.8333435058594, + "epoch": 0.5394285714285715, + "grad_norm": 0.05183921754360199, + "kl": 1.757591962814331e-05, + "lambda_div_used": 0.6081600487232208, + "learning_rate": 1.0919113768029517e-07, + "loss": 0.0248, + "reward": -0.22974545462056994, + "reward_after_mean": -0.22974545462056994, + "reward_after_std": 0.5618578754365444, + "reward_before_mean": 0.03380160592496395, + "reward_before_std": 0.4960998175665736, + "reward_change_max": 0.0, + "reward_change_mean": -0.2635470647364855, + "reward_change_min": -0.4026007801294327, + "reward_change_std": 0.1525269951671362, + "reward_std": 0.5618579015135765, + "rewards/accuracy_reward": 0.14583333767950535, + "rewards/cosine_scaled_reward": -0.1120317269815132, + "step": 472 + }, + { + "clip_fraction": 0.0, + "completion_length": 3389.7291870117188, + "epoch": 0.5405714285714286, + "grad_norm": 0.05399390682578087, + "kl": 2.3730099201202393e-05, + "lambda_div_used": 0.6023471653461456, + "learning_rate": 1.0857018009286381e-07, + "loss": -0.0081, + "reward": -0.16527403378859162, + "reward_after_mean": -0.16527403378859162, + "reward_after_std": 0.4859522972255945, + "reward_before_mean": 0.15660450607538223, + "reward_before_std": 0.469901567324996, + "reward_change_max": 0.0, + "reward_change_mean": -0.3218785412609577, + "reward_change_min": -0.5562809407711029, + "reward_change_std": 0.21282578073441982, + "reward_std": 0.48595230281352997, + "rewards/accuracy_reward": 0.2083333395421505, + "rewards/cosine_scaled_reward": -0.051728841848671436, + "step": 473 + }, + { + "clip_fraction": 0.0, + "completion_length": 2609.3541946411133, + "epoch": 0.5417142857142857, + "grad_norm": 0.07730484753847122, + "kl": 1.3811513781547546e-05, + "lambda_div_used": 0.5951687097549438, + "learning_rate": 1.0797073717209013e-07, + "loss": 0.085, + "reward": 0.0976928174495697, + "reward_after_mean": 0.0976928174495697, + "reward_after_std": 0.5153173375874758, + "reward_before_mean": 0.5706379320472479, + "reward_before_std": 0.4373670984059572, + "reward_change_max": 0.0, + "reward_change_mean": -0.47294510155916214, + "reward_change_min": -0.6950589865446091, + "reward_change_std": 0.27640796452760696, + "reward_std": 0.515317365527153, + "rewards/accuracy_reward": 0.3958333432674408, + "rewards/cosine_scaled_reward": 0.1748045664280653, + "step": 474 + }, + { + "clip_fraction": 0.0, + "completion_length": 2361.020851135254, + "epoch": 0.5428571428571428, + "grad_norm": 0.08214546740055084, + "kl": 9.525567293167114e-06, + "lambda_div_used": 0.6207183972001076, + "learning_rate": 1.0739283813397639e-07, + "loss": 0.0204, + "reward": 0.0007177861407399178, + "reward_after_mean": 0.0007177861407399178, + "reward_after_std": 0.5817896965891123, + "reward_before_mean": 0.34289272502064705, + "reward_before_std": 0.5596933793276548, + "reward_change_max": 0.0, + "reward_change_mean": -0.3421749472618103, + "reward_change_min": -0.5431834943592548, + "reward_change_std": 0.21874003671109676, + "reward_std": 0.581789730116725, + "rewards/accuracy_reward": 0.29166667722165585, + "rewards/cosine_scaled_reward": 0.05122605012729764, + "step": 475 + }, + { + "clip_fraction": 0.0, + "completion_length": 2583.8333740234375, + "epoch": 0.544, + "grad_norm": 0.08052952587604523, + "kl": 3.0007213354110718e-05, + "lambda_div_used": 0.6784488782286644, + "learning_rate": 1.068365111445064e-07, + "loss": 0.0154, + "reward": -0.031517775263637304, + "reward_after_mean": -0.031517775263637304, + "reward_after_std": 0.8641770519316196, + "reward_before_mean": 0.2038349723443389, + "reward_before_std": 0.8286011293530464, + "reward_change_max": 0.0, + "reward_change_mean": -0.23535274900496006, + "reward_change_min": -0.41804002970457077, + "reward_change_std": 0.16118939872831106, + "reward_std": 0.8641770891845226, + "rewards/accuracy_reward": 0.22916666977107525, + "rewards/cosine_scaled_reward": -0.025331700686365366, + "step": 476 + }, + { + "clip_fraction": 0.0, + "completion_length": 2308.291732788086, + "epoch": 0.5451428571428572, + "grad_norm": 0.06839341670274734, + "kl": 8.73953104019165e-06, + "lambda_div_used": 0.6096622571349144, + "learning_rate": 1.063017833182728e-07, + "loss": 0.1111, + "reward": 0.19456686079502106, + "reward_after_mean": 0.19456686079502106, + "reward_after_std": 0.5847878716886044, + "reward_before_mean": 0.678333050571382, + "reward_before_std": 0.5006472393870354, + "reward_change_max": 0.0, + "reward_change_mean": -0.48376619443297386, + "reward_change_min": -0.6835142783820629, + "reward_change_std": 0.2774948738515377, + "reward_std": 0.5847878959029913, + "rewards/accuracy_reward": 0.45833334885537624, + "rewards/cosine_scaled_reward": 0.21999971382319927, + "step": 477 + }, + { + "clip_fraction": 0.0, + "completion_length": 3169.3125610351562, + "epoch": 0.5462857142857143, + "grad_norm": 0.052864328026771545, + "kl": 6.990041583776474e-06, + "lambda_div_used": 0.5758631080389023, + "learning_rate": 1.0578868071715544e-07, + "loss": 0.0349, + "reward": -0.23655124567449093, + "reward_after_mean": -0.23655124567449093, + "reward_after_std": 0.4532997701317072, + "reward_before_mean": 0.11144567281007767, + "reward_before_std": 0.3459607223048806, + "reward_change_max": 0.0, + "reward_change_mean": -0.3479969333857298, + "reward_change_min": -0.5187661461532116, + "reward_change_std": 0.19546091184020042, + "reward_std": 0.4532997887581587, + "rewards/accuracy_reward": 0.20833333395421505, + "rewards/cosine_scaled_reward": -0.09688764810562134, + "step": 478 + }, + { + "clip_fraction": 0.0, + "completion_length": 2825.916702270508, + "epoch": 0.5474285714285714, + "grad_norm": 0.08355361968278885, + "kl": 3.739655949175358e-05, + "lambda_div_used": 0.5902442187070847, + "learning_rate": 1.0529722834905125e-07, + "loss": 0.033, + "reward": -0.37585191056132317, + "reward_after_mean": -0.37585191056132317, + "reward_after_std": 0.45466959848999977, + "reward_before_mean": -0.12638765759766102, + "reward_before_std": 0.4168459586799145, + "reward_change_max": 0.0, + "reward_change_mean": -0.2494642548263073, + "reward_change_min": -0.4229847304522991, + "reward_change_std": 0.15689124166965485, + "reward_std": 0.4546696189790964, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.18888765573501587, + "step": 479 + }, + { + "clip_fraction": 0.0, + "completion_length": 2415.895866394043, + "epoch": 0.5485714285714286, + "grad_norm": 0.08123134076595306, + "kl": 1.4044344425201416e-05, + "lambda_div_used": 0.5890126600861549, + "learning_rate": 1.0482745016665526e-07, + "loss": 0.0739, + "reward": -0.36289478465914726, + "reward_after_mean": -0.36289478465914726, + "reward_after_std": 0.4815117195248604, + "reward_before_mean": -0.11086839716881514, + "reward_before_std": 0.4026770405471325, + "reward_change_max": 0.0, + "reward_change_mean": -0.25202639400959015, + "reward_change_min": -0.37619682028889656, + "reward_change_std": 0.13623447716236115, + "reward_std": 0.4815117232501507, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.17336839414201677, + "step": 480 + }, + { + "clip_fraction": 0.0, + "completion_length": 3057.625030517578, + "epoch": 0.5497142857142857, + "grad_norm": 0.05632660165429115, + "kl": 2.8233975172042847e-05, + "lambda_div_used": 0.5841230228543282, + "learning_rate": 1.0437936906629334e-07, + "loss": 0.0198, + "reward": -0.386552257463336, + "reward_after_mean": -0.386552257463336, + "reward_after_std": 0.4468122199177742, + "reward_before_mean": -0.14256569184362888, + "reward_before_std": 0.38717135414481163, + "reward_change_max": 0.0, + "reward_change_mean": -0.24398657120764256, + "reward_change_min": -0.42186837270855904, + "reward_change_std": 0.14736179821193218, + "reward_std": 0.4468122236430645, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.20506569370627403, + "step": 481 + }, + { + "clip_fraction": 0.0, + "completion_length": 2824.5833587646484, + "epoch": 0.5508571428571428, + "grad_norm": 0.0865793228149414, + "kl": 2.4417764507234097e-05, + "lambda_div_used": 0.622517004609108, + "learning_rate": 1.0395300688680625e-07, + "loss": -0.0313, + "reward": -0.05876433290541172, + "reward_after_mean": -0.05876433290541172, + "reward_after_std": 0.5989817604422569, + "reward_before_mean": 0.259432727470994, + "reward_before_std": 0.5737235806882381, + "reward_change_max": 0.0, + "reward_change_mean": -0.31819707341492176, + "reward_change_min": -0.5269695781171322, + "reward_change_std": 0.21138717606663704, + "reward_std": 0.5989817790687084, + "rewards/accuracy_reward": 0.22916667349636555, + "rewards/cosine_scaled_reward": 0.030266055837273598, + "step": 482 + }, + { + "clip_fraction": 0.0, + "completion_length": 2869.250015258789, + "epoch": 0.552, + "grad_norm": 0.08511485904455185, + "kl": 3.474205732345581e-05, + "lambda_div_used": 0.6013401597738266, + "learning_rate": 1.0354838440848501e-07, + "loss": -0.0837, + "reward": -0.2791392467916012, + "reward_after_mean": -0.2791392467916012, + "reward_after_std": 0.5272991992533207, + "reward_before_mean": -0.02371177263557911, + "reward_before_std": 0.4689871799200773, + "reward_change_max": 0.0, + "reward_change_mean": -0.2554274704307318, + "reward_change_min": -0.40494704991579056, + "reward_change_std": 0.15485912468284369, + "reward_std": 0.5272992141544819, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.1278784405440092, + "step": 483 + }, + { + "clip_fraction": 0.0, + "completion_length": 2591.7500228881836, + "epoch": 0.5531428571428572, + "grad_norm": 0.08745326101779938, + "kl": 3.6368146538734436e-07, + "lambda_div_used": 0.6466837078332901, + "learning_rate": 1.0316552135205837e-07, + "loss": 0.0082, + "reward": -0.04187892563641071, + "reward_after_mean": -0.04187892563641071, + "reward_after_std": 0.6843612100929022, + "reward_before_mean": 0.242530676885508, + "reward_before_std": 0.6844684220850468, + "reward_change_max": 0.0, + "reward_change_mean": -0.284409599378705, + "reward_change_min": -0.49204646795988083, + "reward_change_std": 0.1993796620517969, + "reward_std": 0.6843612212687731, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.0283026653341949, + "step": 484 + }, + { + "clip_fraction": 0.0, + "completion_length": 2046.2500381469727, + "epoch": 0.5542857142857143, + "grad_norm": 0.09076947718858719, + "kl": 2.752244472503662e-05, + "lambda_div_used": 0.5719796344637871, + "learning_rate": 1.0280443637773163e-07, + "loss": 0.0035, + "reward": -0.40644849208183587, + "reward_after_mean": -0.40644849208183587, + "reward_after_std": 0.398155614733696, + "reward_before_mean": -0.14403727410535794, + "reward_before_std": 0.3282261691056192, + "reward_change_max": 0.0, + "reward_change_mean": -0.2624112106859684, + "reward_change_min": -0.43003450334072113, + "reward_change_std": 0.15388297475874424, + "reward_std": 0.39815562404692173, + "rewards/accuracy_reward": 0.06250000186264515, + "rewards/cosine_scaled_reward": -0.20653727487660944, + "step": 485 + }, + { + "clip_fraction": 0.0, + "completion_length": 1967.7916946411133, + "epoch": 0.5554285714285714, + "grad_norm": 0.09958811849355698, + "kl": 3.91155481338501e-05, + "lambda_div_used": 0.5703976079821587, + "learning_rate": 1.0246514708427701e-07, + "loss": 0.0966, + "reward": -0.26609380822628736, + "reward_after_mean": -0.26609380822628736, + "reward_after_std": 0.4386676363646984, + "reward_before_mean": 0.07946969009935856, + "reward_before_std": 0.31633214373141527, + "reward_change_max": 0.0, + "reward_change_mean": -0.34556350111961365, + "reward_change_min": -0.48576217144727707, + "reward_change_std": 0.1828850321471691, + "reward_std": 0.4386676475405693, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.08719698037020862, + "step": 486 + }, + { + "clip_fraction": 0.0, + "completion_length": 1967.0625228881836, + "epoch": 0.5565714285714286, + "grad_norm": 0.10718576610088348, + "kl": 3.7085264921188354e-06, + "lambda_div_used": 0.6087248548865318, + "learning_rate": 1.0214767000817596e-07, + "loss": 0.0461, + "reward": 0.08780635055154562, + "reward_after_mean": 0.08780635055154562, + "reward_after_std": 0.6550032701343298, + "reward_before_mean": 0.5625860100844875, + "reward_before_std": 0.5011934200301766, + "reward_change_max": 0.0, + "reward_change_mean": -0.47477963753044605, + "reward_change_min": -0.7034505233168602, + "reward_change_std": 0.26853484753519297, + "reward_std": 0.6550032943487167, + "rewards/accuracy_reward": 0.3958333358168602, + "rewards/cosine_scaled_reward": 0.16675265738740563, + "step": 487 + }, + { + "clip_fraction": 0.0, + "completion_length": 2356.8125228881836, + "epoch": 0.5577142857142857, + "grad_norm": 0.08583448082208633, + "kl": 2.1670013666152954e-05, + "lambda_div_used": 0.5598724335432053, + "learning_rate": 1.0185202062281336e-07, + "loss": -0.0446, + "reward": -0.39712974801659584, + "reward_after_mean": -0.39712974801659584, + "reward_after_std": 0.3282298669219017, + "reward_before_mean": -0.11880321707576513, + "reward_before_std": 0.2731999019160867, + "reward_change_max": 0.0, + "reward_change_mean": -0.27832652255892754, + "reward_change_min": -0.42709672823548317, + "reward_change_std": 0.16301770228892565, + "reward_std": 0.3282298743724823, + "rewards/accuracy_reward": 0.0833333358168602, + "rewards/cosine_scaled_reward": -0.20213654916733503, + "step": 488 + }, + { + "clip_fraction": 0.0, + "completion_length": 3039.2916870117188, + "epoch": 0.5588571428571428, + "grad_norm": 0.08493036031723022, + "kl": 4.955753684043884e-05, + "lambda_div_used": 0.5549413114786148, + "learning_rate": 1.0157821333772304e-07, + "loss": -0.0127, + "reward": -0.4395306259393692, + "reward_after_mean": -0.4395306259393692, + "reward_after_std": 0.30519232526421547, + "reward_before_mean": -0.16155868768692017, + "reward_before_std": 0.24803727120161057, + "reward_change_max": 0.0, + "reward_change_mean": -0.27797193080186844, + "reward_change_min": -0.44194458797574043, + "reward_change_std": 0.1623495165258646, + "reward_std": 0.3051923308521509, + "rewards/accuracy_reward": 0.0625, + "rewards/cosine_scaled_reward": -0.22405867651104927, + "step": 489 + }, + { + "clip_fraction": 0.0, + "completion_length": 2838.0833892822266, + "epoch": 0.56, + "grad_norm": 0.06069381162524223, + "kl": 1.2032687664031982e-05, + "lambda_div_used": 0.583185076713562, + "learning_rate": 1.013262614978859e-07, + "loss": 0.0139, + "reward": -0.1338807214051485, + "reward_after_mean": -0.1338807214051485, + "reward_after_std": 0.4649778436869383, + "reward_before_mean": 0.25663699954748154, + "reward_before_std": 0.3789667785167694, + "reward_change_max": 0.0, + "reward_change_mean": -0.39051773957908154, + "reward_change_min": -0.6341840960085392, + "reward_change_std": 0.2300328817218542, + "reward_std": 0.46497784554958344, + "rewards/accuracy_reward": 0.2708333395421505, + "rewards/cosine_scaled_reward": -0.014196328818798065, + "step": 490 + }, + { + "clip_fraction": 0.0, + "completion_length": 2593.7291870117188, + "epoch": 0.5611428571428572, + "grad_norm": 0.07340901345014572, + "kl": 3.531062975525856e-05, + "lambda_div_used": 0.5965912714600563, + "learning_rate": 1.0109617738307911e-07, + "loss": 0.0268, + "reward": 0.01746355928480625, + "reward_after_mean": 0.01746355928480625, + "reward_after_std": 0.5940692499279976, + "reward_before_mean": 0.47233812790364027, + "reward_before_std": 0.4421461224555969, + "reward_change_max": 0.0, + "reward_change_mean": -0.4548745583742857, + "reward_change_min": -0.6400704979896545, + "reward_change_std": 0.24487750884145498, + "reward_std": 0.5940692741423845, + "rewards/accuracy_reward": 0.3541666679084301, + "rewards/cosine_scaled_reward": 0.11817142926156521, + "step": 491 + }, + { + "clip_fraction": 0.0, + "completion_length": 2583.5416679382324, + "epoch": 0.5622857142857143, + "grad_norm": 0.08282249420881271, + "kl": 2.4866312742233276e-05, + "lambda_div_used": 0.5367531701922417, + "learning_rate": 1.0088797220727779e-07, + "loss": 0.0376, + "reward": -0.3908469006419182, + "reward_after_mean": -0.3908469006419182, + "reward_after_std": 0.27869052439928055, + "reward_before_mean": -0.03699151985347271, + "reward_before_std": 0.16138391755521297, + "reward_change_max": 0.0, + "reward_change_mean": -0.3538553789258003, + "reward_change_min": -0.5079905577003956, + "reward_change_std": 0.18725439626723528, + "reward_std": 0.27869053184986115, + "rewards/accuracy_reward": 0.125, + "rewards/cosine_scaled_reward": -0.1619915273040533, + "step": 492 + }, + { + "clip_fraction": 0.0, + "completion_length": 2176.8333740234375, + "epoch": 0.5634285714285714, + "grad_norm": 0.09936831891536713, + "kl": 6.021931767463684e-06, + "lambda_div_used": 0.596942737698555, + "learning_rate": 1.0070165611810855e-07, + "loss": 0.0246, + "reward": -0.037120603024959564, + "reward_after_mean": -0.037120603024959564, + "reward_after_std": 0.46812310442328453, + "reward_before_mean": 0.34137603268027306, + "reward_before_std": 0.44823691714555025, + "reward_change_max": 0.0, + "reward_change_mean": -0.3784966245293617, + "reward_change_min": -0.5661123469471931, + "reward_change_std": 0.234741548076272, + "reward_std": 0.4681231305003166, + "rewards/accuracy_reward": 0.2708333432674408, + "rewards/cosine_scaled_reward": 0.07054269965738058, + "step": 493 + }, + { + "clip_fraction": 0.0, + "completion_length": 2352.583427429199, + "epoch": 0.5645714285714286, + "grad_norm": 0.07574566453695297, + "kl": 1.6648322343826294e-05, + "lambda_div_used": 0.6796721071004868, + "learning_rate": 1.005372381963547e-07, + "loss": 0.092, + "reward": 0.10963849350810051, + "reward_after_mean": 0.10963849350810051, + "reward_after_std": 0.8239130303263664, + "reward_before_mean": 0.4074633736163378, + "reward_before_std": 0.848267612978816, + "reward_change_max": 0.0, + "reward_change_mean": -0.2978248745203018, + "reward_change_min": -0.6511921100318432, + "reward_change_std": 0.2445714958012104, + "reward_std": 0.8239130582660437, + "rewards/accuracy_reward": 0.3125000037252903, + "rewards/cosine_scaled_reward": 0.09496336756274104, + "step": 494 + }, + { + "clip_fraction": 0.0, + "completion_length": 3203.8959045410156, + "epoch": 0.5657142857142857, + "grad_norm": 0.057527996599674225, + "kl": 2.3846514523029327e-05, + "lambda_div_used": 0.5994853675365448, + "learning_rate": 1.0039472645551372e-07, + "loss": 0.0637, + "reward": -0.21300244145095348, + "reward_after_mean": -0.21300244145095348, + "reward_after_std": 0.48377062380313873, + "reward_before_mean": 0.08397414721548557, + "reward_before_std": 0.4531490486115217, + "reward_change_max": 0.0, + "reward_change_mean": -0.29697658494114876, + "reward_change_min": -0.49157723411917686, + "reward_change_std": 0.1875515878200531, + "reward_std": 0.4837706368416548, + "rewards/accuracy_reward": 0.16666667349636555, + "rewards/cosine_scaled_reward": -0.08269252139143646, + "step": 495 + }, + { + "clip_fraction": 0.0, + "completion_length": 1895.9375457763672, + "epoch": 0.5668571428571428, + "grad_norm": 0.10791665315628052, + "kl": 1.890142448246479e-05, + "lambda_div_used": 0.6198414713144302, + "learning_rate": 1.002741278414069e-07, + "loss": 0.0183, + "reward": 0.09977892541792244, + "reward_after_mean": 0.09977892541792244, + "reward_after_std": 0.6343465056270361, + "reward_before_mean": 0.5255357641726732, + "reward_before_std": 0.555769513361156, + "reward_change_max": 0.0, + "reward_change_mean": -0.4257568307220936, + "reward_change_min": -0.6588771641254425, + "reward_change_std": 0.25742017664015293, + "reward_std": 0.634346516802907, + "rewards/accuracy_reward": 0.35416667349636555, + "rewards/cosine_scaled_reward": 0.17136909160763025, + "step": 496 + }, + { + "clip_fraction": 0.0, + "completion_length": 2577.645851135254, + "epoch": 0.568, + "grad_norm": 0.06887742131948471, + "kl": 2.493336796760559e-05, + "lambda_div_used": 0.6232517957687378, + "learning_rate": 1.0017544823184055e-07, + "loss": 0.0143, + "reward": 0.039356768131256104, + "reward_after_mean": 0.039356768131256104, + "reward_after_std": 0.5658366903662682, + "reward_before_mean": 0.401943476870656, + "reward_before_std": 0.5692849718034267, + "reward_change_max": 0.0, + "reward_change_mean": -0.362586697563529, + "reward_change_min": -0.6212925836443901, + "reward_change_std": 0.24119753576815128, + "reward_std": 0.5658366959542036, + "rewards/accuracy_reward": 0.3333333469927311, + "rewards/cosine_scaled_reward": 0.06861014291644096, + "step": 497 + }, + { + "clip_fraction": 0.0, + "completion_length": 2857.2916946411133, + "epoch": 0.5691428571428572, + "grad_norm": 0.06479962170124054, + "kl": 9.963754564523697e-06, + "lambda_div_used": 0.6236685812473297, + "learning_rate": 1.0009869243631952e-07, + "loss": 0.0008, + "reward": -0.18094617873430252, + "reward_after_mean": -0.18094617873430252, + "reward_after_std": 0.6155845355242491, + "reward_before_mean": 0.07468715589493513, + "reward_before_std": 0.5768492119386792, + "reward_change_max": 0.0, + "reward_change_mean": -0.2556333262473345, + "reward_change_min": -0.4431779384613037, + "reward_change_std": 0.16846577636897564, + "reward_std": 0.6155845616012812, + "rewards/accuracy_reward": 0.1666666679084301, + "rewards/cosine_scaled_reward": -0.0919795110821724, + "step": 498 + }, + { + "clip_fraction": 0.0, + "completion_length": 2707.791702270508, + "epoch": 0.5702857142857143, + "grad_norm": 0.06317302584648132, + "kl": 1.5532365068793297e-05, + "lambda_div_used": 0.6250721588730812, + "learning_rate": 1.000438641958131e-07, + "loss": -0.0298, + "reward": -0.12677906453609467, + "reward_after_mean": -0.12677906453609467, + "reward_after_std": 0.6044176463037729, + "reward_before_mean": 0.1698420336470008, + "reward_before_std": 0.5829241154715419, + "reward_change_max": 0.0, + "reward_change_mean": -0.2966211009770632, + "reward_change_min": -0.5905468240380287, + "reward_change_std": 0.2108100038021803, + "reward_std": 0.6044176481664181, + "rewards/accuracy_reward": 0.2083333358168602, + "rewards/cosine_scaled_reward": -0.03849130589514971, + "step": 499 + }, + { + "clip_fraction": 0.0, + "completion_length": 2980.9167098999023, + "epoch": 0.5714285714285714, + "grad_norm": 0.0621151477098465, + "kl": 1.033581793308258e-05, + "lambda_div_used": 0.6023986041545868, + "learning_rate": 1.0001096618257236e-07, + "loss": 0.0314, + "reward": -0.26747864981007297, + "reward_after_mean": -0.26747864981007297, + "reward_after_std": 0.5251844674348831, + "reward_before_mean": -0.0025623496621847153, + "reward_before_std": 0.4696931503713131, + "reward_change_max": 0.0, + "reward_change_mean": -0.26491627655923367, + "reward_change_min": -0.43205036222934723, + "reward_change_std": 0.15884541906416416, + "reward_std": 0.5251844730228186, + "rewards/accuracy_reward": 0.1041666679084301, + "rewards/cosine_scaled_reward": -0.10672901570796967, + "step": 500 + }, + { + "epoch": 0.5714285714285714, + "step": 500, "total_flos": 0.0, - "train_loss": 0.007373248229268939, - "train_runtime": 70012.1488, - "train_samples_per_second": 0.137, - "train_steps_per_second": 0.003 + "train_loss": 0.005314271912367986, + "train_runtime": 106074.3369, + "train_samples_per_second": 0.226, + "train_steps_per_second": 0.005 } ], "logging_steps": 1, - "max_steps": 200, + "max_steps": 500, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 10,