diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,7095 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.999066874027994,
+  "eval_steps": 500,
+  "global_step": 3212,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.006220839813374806,
+      "grad_norm": 4.5,
+      "learning_rate": 2.9953300124533003e-05,
+      "loss": 0.8649,
+      "mean_token_accuracy": 0.885539248585701,
+      "step": 5,
+      "train/kl_loss_qwen": 0.06747693261131645,
+      "train/kl_loss_r1": 0.04241067896800814,
+      "train/total_kl": 0.10988761270418763
+    },
+    {
+      "epoch": 0.012441679626749611,
+      "grad_norm": 14.6875,
+      "learning_rate": 2.9906600249066002e-05,
+      "loss": 1.2059,
+      "mean_token_accuracy": 0.8885251730680466,
+      "step": 10,
+      "train/kl_loss_qwen": 0.0929522925056517,
+      "train/kl_loss_r1": 0.1073197754449211,
+      "train/total_kl": 0.20027207368984817
+    },
+    {
+      "epoch": 0.01866251944012442,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.9859900373599004e-05,
+      "loss": 0.5277,
+      "mean_token_accuracy": 0.9018729269504547,
+      "step": 15,
+      "train/kl_loss_qwen": 0.03184658419340849,
+      "train/kl_loss_r1": 0.01505236077355221,
+      "train/total_kl": 0.04689894551411271
+    },
+    {
+      "epoch": 0.024883359253499222,
+      "grad_norm": 2.734375,
+      "learning_rate": 2.9813200498132003e-05,
+      "loss": 0.5372,
+      "mean_token_accuracy": 0.8993205606937409,
+      "step": 20,
+      "train/kl_loss_qwen": 0.03552339724265039,
+      "train/kl_loss_r1": 0.012143875833135098,
+      "train/total_kl": 0.04766727341338992
+    },
+    {
+      "epoch": 0.03110419906687403,
+      "grad_norm": 2.703125,
+      "learning_rate": 2.9766500622665006e-05,
+      "loss": 0.5124,
+      "mean_token_accuracy": 0.9028977572917938,
+      "step": 25,
+      "train/kl_loss_qwen": 0.031236130651086567,
+      "train/kl_loss_r1": 0.012251556990668178,
+      "train/total_kl": 0.04348768750205636
+    },
+    {
+      "epoch": 0.03732503888024884,
+      "grad_norm": 2.5625,
+      "learning_rate": 2.9719800747198008e-05,
+      "loss": 0.4767,
+      "mean_token_accuracy": 0.9011666357517243,
+      "step": 30,
+      "train/kl_loss_qwen": 0.026808999991044403,
+      "train/kl_loss_r1": 0.008258039911743254,
+      "train/total_kl": 0.03506704047322273
+    },
+    {
+      "epoch": 0.04354587869362364,
+      "grad_norm": 3.765625,
+      "learning_rate": 2.9673100871731007e-05,
+      "loss": 0.511,
+      "mean_token_accuracy": 0.9000001281499863,
+      "step": 35,
+      "train/kl_loss_qwen": 0.027317401487380266,
+      "train/kl_loss_r1": 0.013182285078801214,
+      "train/total_kl": 0.04049968663603067
+    },
+    {
+      "epoch": 0.049766718506998445,
+      "grad_norm": 2.609375,
+      "learning_rate": 2.962640099626401e-05,
+      "loss": 0.5154,
+      "mean_token_accuracy": 0.8928132474422454,
+      "step": 40,
+      "train/kl_loss_qwen": 0.027987364726141096,
+      "train/kl_loss_r1": 0.012156172015238553,
+      "train/total_kl": 0.04014353705570102
+    },
+    {
+      "epoch": 0.05598755832037325,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.9579701120797012e-05,
+      "loss": 0.4937,
+      "mean_token_accuracy": 0.8918720930814743,
+      "step": 45,
+      "train/kl_loss_qwen": 0.024307892844080924,
+      "train/kl_loss_r1": 0.009873847151175142,
+      "train/total_kl": 0.034181739576160906
+    },
+    {
+      "epoch": 0.06220839813374806,
+      "grad_norm": 11.0625,
+      "learning_rate": 2.9533001245330014e-05,
+      "loss": 0.5548,
+      "mean_token_accuracy": 0.9037962466478348,
+      "step": 50,
+      "train/kl_loss_qwen": 0.0403732392936945,
+      "train/kl_loss_r1": 0.01709484038874507,
+      "train/total_kl": 0.057468080334365367
+    },
+    {
+      "epoch": 0.06842923794712286,
+      "grad_norm": 2.71875,
+      "learning_rate": 2.9486301369863017e-05,
+      "loss": 0.473,
+      "mean_token_accuracy": 0.902271169424057,
+      "step": 55,
+      "train/kl_loss_qwen": 0.027851210488006473,
+      "train/kl_loss_r1": 0.009670931240543723,
+      "train/total_kl": 0.03752214219421148
+    },
+    {
+      "epoch": 0.07465007776049767,
+      "grad_norm": 2.46875,
+      "learning_rate": 2.9439601494396016e-05,
+      "loss": 0.4726,
+      "mean_token_accuracy": 0.8976862788200378,
+      "step": 60,
+      "train/kl_loss_qwen": 0.02319532949477434,
+      "train/kl_loss_r1": 0.011528853024356068,
+      "train/total_kl": 0.034724182728677985
+    },
+    {
+      "epoch": 0.08087091757387248,
+      "grad_norm": 2.734375,
+      "learning_rate": 2.9392901618929018e-05,
+      "loss": 0.4812,
+      "mean_token_accuracy": 0.8980915457010269,
+      "step": 65,
+      "train/kl_loss_qwen": 0.0253461551386863,
+      "train/kl_loss_r1": 0.011177089205011726,
+      "train/total_kl": 0.036523244343698025
+    },
+    {
+      "epoch": 0.08709175738724728,
+      "grad_norm": 2.25,
+      "learning_rate": 2.934620174346202e-05,
+      "loss": 0.4546,
+      "mean_token_accuracy": 0.9052111625671386,
+      "step": 70,
+      "train/kl_loss_qwen": 0.024780120234936476,
+      "train/kl_loss_r1": 0.010003501223400236,
+      "train/total_kl": 0.03478362150490284
+    },
+    {
+      "epoch": 0.09331259720062209,
+      "grad_norm": 2.5,
+      "learning_rate": 2.929950186799502e-05,
+      "loss": 0.4993,
+      "mean_token_accuracy": 0.8921610146760941,
+      "step": 75,
+      "train/kl_loss_qwen": 0.023273218562826513,
+      "train/kl_loss_r1": 0.011020376090891659,
+      "train/total_kl": 0.03429359458386898
+    },
+    {
+      "epoch": 0.09953343701399689,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.9252801992528022e-05,
+      "loss": 0.4828,
+      "mean_token_accuracy": 0.8996184289455413,
+      "step": 80,
+      "train/kl_loss_qwen": 0.024831860419362785,
+      "train/kl_loss_r1": 0.01152246268466115,
+      "train/total_kl": 0.03635432319715619
+    },
+    {
+      "epoch": 0.1057542768273717,
+      "grad_norm": 2.859375,
+      "learning_rate": 2.920610211706102e-05,
+      "loss": 0.4623,
+      "mean_token_accuracy": 0.898178032040596,
+      "step": 85,
+      "train/kl_loss_qwen": 0.022067550197243692,
+      "train/kl_loss_r1": 0.009043183457106353,
+      "train/total_kl": 0.031110733561217786
+    },
+    {
+      "epoch": 0.1119751166407465,
+      "grad_norm": 2.578125,
+      "learning_rate": 2.9159402241594023e-05,
+      "loss": 0.6165,
+      "mean_token_accuracy": 0.8967886596918107,
+      "step": 90,
+      "train/kl_loss_qwen": 0.04917705850675702,
+      "train/kl_loss_r1": 0.01726052793674171,
+      "train/total_kl": 0.06643758723512291
+    },
+    {
+      "epoch": 0.1181959564541213,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.9112702366127026e-05,
+      "loss": 0.4688,
+      "mean_token_accuracy": 0.8956692755222321,
+      "step": 95,
+      "train/kl_loss_qwen": 0.022392515651881696,
+      "train/kl_loss_r1": 0.010053132427856326,
+      "train/total_kl": 0.03244564794003964
+    },
+    {
+      "epoch": 0.12441679626749612,
+      "grad_norm": 2.421875,
+      "learning_rate": 2.9066002490660025e-05,
+      "loss": 0.4666,
+      "mean_token_accuracy": 0.9006453633308411,
+      "step": 100,
+      "train/kl_loss_qwen": 0.02538487664423883,
+      "train/kl_loss_r1": 0.00974747968139127,
+      "train/total_kl": 0.03513235654681921
+    },
+    {
+      "epoch": 0.13063763608087092,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.9019302615193027e-05,
+      "loss": 0.4403,
+      "mean_token_accuracy": 0.8980156928300858,
+      "step": 105,
+      "train/kl_loss_qwen": 0.019946316070854665,
+      "train/kl_loss_r1": 0.00894002104178071,
+      "train/total_kl": 0.0288863368332386
+    },
+    {
+      "epoch": 0.1368584758942457,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.8972602739726026e-05,
+      "loss": 0.4374,
+      "mean_token_accuracy": 0.9068128049373627,
+      "step": 110,
+      "train/kl_loss_qwen": 0.02424931791611016,
+      "train/kl_loss_r1": 0.009157051984220742,
+      "train/total_kl": 0.03340637004002929
+    },
+    {
+      "epoch": 0.14307931570762053,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.8925902864259028e-05,
+      "loss": 0.4392,
+      "mean_token_accuracy": 0.9012053400278092,
+      "step": 115,
+      "train/kl_loss_qwen": 0.020489979069679976,
+      "train/kl_loss_r1": 0.008880690694786609,
+      "train/total_kl": 0.02937066974118352
+    },
+    {
+      "epoch": 0.14930015552099535,
+      "grad_norm": 2.546875,
+      "learning_rate": 2.887920298879203e-05,
+      "loss": 0.4412,
+      "mean_token_accuracy": 0.899726277589798,
+      "step": 120,
+      "train/kl_loss_qwen": 0.0204013851005584,
+      "train/kl_loss_r1": 0.009058444690890611,
+      "train/total_kl": 0.029459829907864332
+    },
+    {
+      "epoch": 0.15552099533437014,
+      "grad_norm": 2.03125,
+      "learning_rate": 2.883250311332503e-05,
+      "loss": 0.4338,
+      "mean_token_accuracy": 0.9087229013442993,
+      "step": 125,
+      "train/kl_loss_qwen": 0.023963469313457608,
+      "train/kl_loss_r1": 0.010771268280223012,
+      "train/total_kl": 0.03473473777994514
+    },
+    {
+      "epoch": 0.16174183514774496,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.8785803237858032e-05,
+      "loss": 0.4331,
+      "mean_token_accuracy": 0.9061719745397567,
+      "step": 130,
+      "train/kl_loss_qwen": 0.022065827203914524,
+      "train/kl_loss_r1": 0.011356789106503128,
+      "train/total_kl": 0.03342261621728539
+    },
+    {
+      "epoch": 0.16796267496111975,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.8739103362391034e-05,
+      "loss": 0.4374,
+      "mean_token_accuracy": 0.9011161267757416,
+      "step": 135,
+      "train/kl_loss_qwen": 0.021853275410830975,
+      "train/kl_loss_r1": 0.010496085102204233,
+      "train/total_kl": 0.032349360454827544
+    },
+    {
+      "epoch": 0.17418351477449456,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.8692403486924037e-05,
+      "loss": 0.4457,
+      "mean_token_accuracy": 0.9047422379255294,
+      "step": 140,
+      "train/kl_loss_qwen": 0.02275628219358623,
+      "train/kl_loss_r1": 0.010783428023569287,
+      "train/total_kl": 0.03353970991447568
+    },
+    {
+      "epoch": 0.18040435458786935,
+      "grad_norm": 3.21875,
+      "learning_rate": 2.864570361145704e-05,
+      "loss": 0.4538,
+      "mean_token_accuracy": 0.9026034384965896,
+      "step": 145,
+      "train/kl_loss_qwen": 0.02352142836898565,
+      "train/kl_loss_r1": 0.011291519668884576,
+      "train/total_kl": 0.03481294792145491
+    },
+    {
+      "epoch": 0.18662519440124417,
+      "grad_norm": 3.078125,
+      "learning_rate": 2.8599003735990038e-05,
+      "loss": 0.458,
+      "mean_token_accuracy": 0.9031858772039414,
+      "step": 150,
+      "train/kl_loss_qwen": 0.022739559737965466,
+      "train/kl_loss_r1": 0.012792359199374914,
+      "train/total_kl": 0.03553191879764199
+    },
+    {
+      "epoch": 0.19284603421461896,
+      "grad_norm": 2.375,
+      "learning_rate": 2.855230386052304e-05,
+      "loss": 0.5434,
+      "mean_token_accuracy": 0.907307168841362,
+      "step": 155,
+      "train/kl_loss_qwen": 0.04449260346591473,
+      "train/kl_loss_r1": 0.014073733380064369,
+      "train/total_kl": 0.05856633689254522
+    },
+    {
+      "epoch": 0.19906687402799378,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.8505603985056043e-05,
+      "loss": 0.4605,
+      "mean_token_accuracy": 0.8996091216802597,
+      "step": 160,
+      "train/kl_loss_qwen": 0.025977824395522476,
+      "train/kl_loss_r1": 0.01165865061338991,
+      "train/total_kl": 0.03763647498562932
+    },
+    {
+      "epoch": 0.2052877138413686,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.8458904109589042e-05,
+      "loss": 0.528,
+      "mean_token_accuracy": 0.900575140118599,
+      "step": 165,
+      "train/kl_loss_qwen": 0.031398235633969304,
+      "train/kl_loss_r1": 0.023330681957304478,
+      "train/total_kl": 0.05472891740500927
+    },
+    {
+      "epoch": 0.2115085536547434,
+      "grad_norm": 2.65625,
+      "learning_rate": 2.8412204234122044e-05,
+      "loss": 0.4433,
+      "mean_token_accuracy": 0.9035054802894592,
+      "step": 170,
+      "train/kl_loss_qwen": 0.0246684439945966,
+      "train/kl_loss_r1": 0.011178959859535098,
+      "train/total_kl": 0.03584740431979298
+    },
+    {
+      "epoch": 0.2177293934681182,
+      "grad_norm": 2.78125,
+      "learning_rate": 2.8365504358655043e-05,
+      "loss": 0.4265,
+      "mean_token_accuracy": 0.9048573732376098,
+      "step": 175,
+      "train/kl_loss_qwen": 0.02242542668245733,
+      "train/kl_loss_r1": 0.010072631668299437,
+      "train/total_kl": 0.032498058304190634
+    },
+    {
+      "epoch": 0.223950233281493,
+      "grad_norm": 2.53125,
+      "learning_rate": 2.8318804483188046e-05,
+      "loss": 0.469,
+      "mean_token_accuracy": 0.9018107861280441,
+      "step": 180,
+      "train/kl_loss_qwen": 0.02478899694979191,
+      "train/kl_loss_r1": 0.012270277575589716,
+      "train/total_kl": 0.037059274548664686
+    },
+    {
+      "epoch": 0.2301710730948678,
+      "grad_norm": 2.25,
+      "learning_rate": 2.8272104607721048e-05,
+      "loss": 0.5042,
+      "mean_token_accuracy": 0.9139624059200286,
+      "step": 185,
+      "train/kl_loss_qwen": 0.03831529831513762,
+      "train/kl_loss_r1": 0.01714326792862266,
+      "train/total_kl": 0.05545856654644012
+    },
+    {
+      "epoch": 0.2363919129082426,
+      "grad_norm": 2.140625,
+      "learning_rate": 2.8225404732254047e-05,
+      "loss": 0.5493,
+      "mean_token_accuracy": 0.9074622631072998,
+      "step": 190,
+      "train/kl_loss_qwen": 0.04459263035096228,
+      "train/kl_loss_r1": 0.01827790851239115,
+      "train/total_kl": 0.06287053730338812
+    },
+    {
+      "epoch": 0.24261275272161742,
+      "grad_norm": 2.40625,
+      "learning_rate": 2.817870485678705e-05,
+      "loss": 0.4768,
+      "mean_token_accuracy": 0.9010340601205826,
+      "step": 195,
+      "train/kl_loss_qwen": 0.027825326565653084,
+      "train/kl_loss_r1": 0.01221497980877757,
+      "train/total_kl": 0.04004030600190163
+    },
+    {
+      "epoch": 0.24883359253499224,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.813200498132005e-05,
+      "loss": 0.464,
+      "mean_token_accuracy": 0.8969788819551467,
+      "step": 200,
+      "train/kl_loss_qwen": 0.02236967678181827,
+      "train/kl_loss_r1": 0.011131012113764882,
+      "train/total_kl": 0.03350068908184767
+    },
+    {
+      "epoch": 0.25505443234836706,
+      "grad_norm": 2.484375,
+      "learning_rate": 2.808530510585305e-05,
+      "loss": 0.4246,
+      "mean_token_accuracy": 0.9050369083881378,
+      "step": 205,
+      "train/kl_loss_qwen": 0.0205402294639498,
+      "train/kl_loss_r1": 0.009700670512393117,
+      "train/total_kl": 0.030240900162607432
+    },
+    {
+      "epoch": 0.26127527216174184,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.8038605230386053e-05,
+      "loss": 0.4422,
+      "mean_token_accuracy": 0.90710708796978,
+      "step": 210,
+      "train/kl_loss_qwen": 0.02467436045408249,
+      "train/kl_loss_r1": 0.011160785856191069,
+      "train/total_kl": 0.03583514606580138
+    },
+    {
+      "epoch": 0.26749611197511663,
+      "grad_norm": 3.40625,
+      "learning_rate": 2.7991905354919052e-05,
+      "loss": 0.4912,
+      "mean_token_accuracy": 0.9063985139131546,
+      "step": 215,
+      "train/kl_loss_qwen": 0.03254157407209277,
+      "train/kl_loss_r1": 0.014230213570408523,
+      "train/total_kl": 0.04677178747951984
+    },
+    {
+      "epoch": 0.2737169517884914,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.7945205479452054e-05,
+      "loss": 0.4077,
+      "mean_token_accuracy": 0.9085961043834686,
+      "step": 220,
+      "train/kl_loss_qwen": 0.019017737498506902,
+      "train/kl_loss_r1": 0.008564504305832088,
+      "train/total_kl": 0.027582241501659156
+    },
+    {
+      "epoch": 0.27993779160186627,
+      "grad_norm": 2.609375,
+      "learning_rate": 2.7898505603985057e-05,
+      "loss": 0.4285,
+      "mean_token_accuracy": 0.9006385743618012,
+      "step": 225,
+      "train/kl_loss_qwen": 0.020106442319229246,
+      "train/kl_loss_r1": 0.008756639622151851,
+      "train/total_kl": 0.028863081801682712
+    },
+    {
+      "epoch": 0.28615863141524106,
+      "grad_norm": 2.5625,
+      "learning_rate": 2.785180572851806e-05,
+      "loss": 0.4375,
+      "mean_token_accuracy": 0.9008216232061386,
+      "step": 230,
+      "train/kl_loss_qwen": 0.021523965429514645,
+      "train/kl_loss_r1": 0.008759856573306024,
+      "train/total_kl": 0.03028382211923599
+    },
+    {
+      "epoch": 0.29237947122861585,
+      "grad_norm": 2.578125,
+      "learning_rate": 2.780510585305106e-05,
+      "loss": 0.4448,
+      "mean_token_accuracy": 0.9039193242788315,
+      "step": 235,
+      "train/kl_loss_qwen": 0.02277344111353159,
+      "train/kl_loss_r1": 0.01099874887149781,
+      "train/total_kl": 0.03377219010144472
+    },
+    {
+      "epoch": 0.2986003110419907,
+      "grad_norm": 2.609375,
+      "learning_rate": 2.775840597758406e-05,
+      "loss": 0.4399,
+      "mean_token_accuracy": 0.8983087122440339,
+      "step": 240,
+      "train/kl_loss_qwen": 0.020881993137300013,
+      "train/kl_loss_r1": 0.009922460105735808,
+      "train/total_kl": 0.03080445323139429
+    },
+    {
+      "epoch": 0.3048211508553655,
+      "grad_norm": 2.796875,
+      "learning_rate": 2.7711706102117063e-05,
+      "loss": 0.417,
+      "mean_token_accuracy": 0.9045127362012864,
+      "step": 245,
+      "train/kl_loss_qwen": 0.020143334427848457,
+      "train/kl_loss_r1": 0.008726878045126795,
+      "train/total_kl": 0.02887021256610751
+    },
+    {
+      "epoch": 0.3110419906687403,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.7665006226650062e-05,
+      "loss": 0.4208,
+      "mean_token_accuracy": 0.9110885441303254,
+      "step": 250,
+      "train/kl_loss_qwen": 0.021725011849775912,
+      "train/kl_loss_r1": 0.012001089472323655,
+      "train/total_kl": 0.03372610127553344
+    },
+    {
+      "epoch": 0.31726283048211507,
+      "grad_norm": 2.46875,
+      "learning_rate": 2.7618306351183064e-05,
+      "loss": 0.4402,
+      "mean_token_accuracy": 0.9017707496881485,
+      "step": 255,
+      "train/kl_loss_qwen": 0.022058865614235403,
+      "train/kl_loss_r1": 0.010667199129238724,
+      "train/total_kl": 0.032726064510643485
+    },
+    {
+      "epoch": 0.3234836702954899,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.7571606475716067e-05,
+      "loss": 0.4254,
+      "mean_token_accuracy": 0.9135203003883362,
+      "step": 260,
+      "train/kl_loss_qwen": 0.025819759676232935,
+      "train/kl_loss_r1": 0.012692330474965274,
+      "train/total_kl": 0.0385120902210474
+    },
+    {
+      "epoch": 0.3297045101088647,
+      "grad_norm": 2.78125,
+      "learning_rate": 2.7524906600249066e-05,
+      "loss": 0.4498,
+      "mean_token_accuracy": 0.9065485000610352,
+      "step": 265,
+      "train/kl_loss_qwen": 0.02562568709254265,
+      "train/kl_loss_r1": 0.012835084449034185,
+      "train/total_kl": 0.03846077108755708
+    },
+    {
+      "epoch": 0.3359253499222395,
+      "grad_norm": 3.4375,
+      "learning_rate": 2.7478206724782068e-05,
+      "loss": 0.4544,
+      "mean_token_accuracy": 0.8968957453966141,
+      "step": 270,
+      "train/kl_loss_qwen": 0.020935205090790988,
+      "train/kl_loss_r1": 0.011926196550484746,
+      "train/total_kl": 0.03286140169948339
+    },
+    {
+      "epoch": 0.3421461897356143,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.743150684931507e-05,
+      "loss": 0.4695,
+      "mean_token_accuracy": 0.895763835310936,
+      "step": 275,
+      "train/kl_loss_qwen": 0.024620278738439083,
+      "train/kl_loss_r1": 0.00964386686682701,
+      "train/total_kl": 0.0342641456052661
+    },
+    {
+      "epoch": 0.3483670295489891,
+      "grad_norm": 3.546875,
+      "learning_rate": 2.738480697384807e-05,
+      "loss": 0.4383,
+      "mean_token_accuracy": 0.9082347899675369,
+      "step": 280,
+      "train/kl_loss_qwen": 0.022315250197425484,
+      "train/kl_loss_r1": 0.014057450694963336,
+      "train/total_kl": 0.036372700985521075
+    },
+    {
+      "epoch": 0.3545878693623639,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.7338107098381072e-05,
+      "loss": 0.4469,
+      "mean_token_accuracy": 0.9045476704835892,
+      "step": 285,
+      "train/kl_loss_qwen": 0.023055148823186756,
+      "train/kl_loss_r1": 0.009570301149506123,
+      "train/total_kl": 0.03262544954195619
+    },
+    {
+      "epoch": 0.3608087091757387,
+      "grad_norm": 2.421875,
+      "learning_rate": 2.729140722291407e-05,
+      "loss": 0.442,
+      "mean_token_accuracy": 0.9033053010702133,
+      "step": 290,
+      "train/kl_loss_qwen": 0.022126297745853663,
+      "train/kl_loss_r1": 0.012546762591227889,
+      "train/total_kl": 0.034673060104250905
+    },
+    {
+      "epoch": 0.36702954898911355,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.7244707347447073e-05,
+      "loss": 0.4162,
+      "mean_token_accuracy": 0.9015816122293472,
+      "step": 295,
+      "train/kl_loss_qwen": 0.02032669628970325,
+      "train/kl_loss_r1": 0.007502669095993042,
+      "train/total_kl": 0.027829365665093066
+    },
+    {
+      "epoch": 0.37325038880248834,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.7198007471980076e-05,
+      "loss": 0.4248,
+      "mean_token_accuracy": 0.9025011241436005,
+      "step": 300,
+      "train/kl_loss_qwen": 0.0200017383787781,
+      "train/kl_loss_r1": 0.009314235928468407,
+      "train/total_kl": 0.029315974470227955
+    },
+    {
+      "epoch": 0.37947122861586313,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.7151307596513075e-05,
+      "loss": 0.4476,
+      "mean_token_accuracy": 0.8975890159606934,
+      "step": 305,
+      "train/kl_loss_qwen": 0.021518654003739358,
+      "train/kl_loss_r1": 0.010331802139990031,
+      "train/total_kl": 0.03185045635327697
+    },
+    {
+      "epoch": 0.3856920684292379,
+      "grad_norm": 2.984375,
+      "learning_rate": 2.7104607721046077e-05,
+      "loss": 0.473,
+      "mean_token_accuracy": 0.9017946988344192,
+      "step": 310,
+      "train/kl_loss_qwen": 0.028143000835552812,
+      "train/kl_loss_r1": 0.01120884264819324,
+      "train/total_kl": 0.039351843390613796
+    },
+    {
+      "epoch": 0.39191290824261277,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.705790784557908e-05,
+      "loss": 0.3974,
+      "mean_token_accuracy": 0.9137770593166351,
+      "step": 315,
+      "train/kl_loss_qwen": 0.02027838435024023,
+      "train/kl_loss_r1": 0.009311219793744385,
+      "train/total_kl": 0.029589604306966066
+    },
+    {
+      "epoch": 0.39813374805598756,
+      "grad_norm": 2.546875,
+      "learning_rate": 2.701120797011208e-05,
+      "loss": 0.438,
+      "mean_token_accuracy": 0.9064665764570237,
+      "step": 320,
+      "train/kl_loss_qwen": 0.022875147871673107,
+      "train/kl_loss_r1": 0.010096224106382579,
+      "train/total_kl": 0.0329713718034327
+    },
+    {
+      "epoch": 0.40435458786936235,
+      "grad_norm": 2.578125,
+      "learning_rate": 2.6964508094645084e-05,
+      "loss": 0.4148,
+      "mean_token_accuracy": 0.9041642516851425,
+      "step": 325,
+      "train/kl_loss_qwen": 0.02039993805810809,
+      "train/kl_loss_r1": 0.009322560648433864,
+      "train/total_kl": 0.02972249872982502
+    },
+    {
+      "epoch": 0.4105754276827372,
+      "grad_norm": 3.15625,
+      "learning_rate": 2.6917808219178083e-05,
+      "loss": 0.4599,
+      "mean_token_accuracy": 0.8985224097967148,
+      "step": 330,
+      "train/kl_loss_qwen": 0.023622136563062668,
+      "train/kl_loss_r1": 0.010554712312296034,
+      "train/total_kl": 0.03417684901505709
+    },
+    {
+      "epoch": 0.416796267496112,
+      "grad_norm": 2.53125,
+      "learning_rate": 2.6871108343711085e-05,
+      "loss": 0.4479,
+      "mean_token_accuracy": 0.9001316875219345,
+      "step": 335,
+      "train/kl_loss_qwen": 0.023164877016097308,
+      "train/kl_loss_r1": 0.01062884961720556,
+      "train/total_kl": 0.03379372642375529
+    },
+    {
+      "epoch": 0.4230171073094868,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.6824408468244084e-05,
+      "loss": 0.4349,
+      "mean_token_accuracy": 0.8997949302196503,
+      "step": 340,
+      "train/kl_loss_qwen": 0.020284941466525198,
+      "train/kl_loss_r1": 0.008697902620770036,
+      "train/total_kl": 0.028982843924313783
+    },
+    {
+      "epoch": 0.42923794712286156,
+      "grad_norm": 2.484375,
+      "learning_rate": 2.6777708592777087e-05,
+      "loss": 0.4183,
+      "mean_token_accuracy": 0.9043066084384919,
+      "step": 345,
+      "train/kl_loss_qwen": 0.020022868039086462,
+      "train/kl_loss_r1": 0.009400142775848507,
+      "train/total_kl": 0.02942301072180271
+    },
+    {
+      "epoch": 0.4354587869362364,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.673100871731009e-05,
+      "loss": 0.4479,
+      "mean_token_accuracy": 0.9017944246530533,
+      "step": 350,
+      "train/kl_loss_qwen": 0.02255958765745163,
+      "train/kl_loss_r1": 0.011340912338346243,
+      "train/total_kl": 0.033900499902665616
+    },
+    {
+      "epoch": 0.4416796267496112,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.6684308841843088e-05,
+      "loss": 0.5377,
+      "mean_token_accuracy": 0.9042023122310638,
+      "step": 355,
+      "train/kl_loss_qwen": 0.03636851399205625,
+      "train/kl_loss_r1": 0.018854469433426856,
+      "train/total_kl": 0.055222983751446006
+    },
+    {
+      "epoch": 0.447900466562986,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.663760896637609e-05,
+      "loss": 0.4614,
+      "mean_token_accuracy": 0.9028053224086762,
+      "step": 360,
+      "train/kl_loss_qwen": 0.027840341813862322,
+      "train/kl_loss_r1": 0.012210264848545194,
+      "train/total_kl": 0.040050606802105905
+    },
+    {
+      "epoch": 0.45412130637636083,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.6590909090909093e-05,
+      "loss": 0.4881,
+      "mean_token_accuracy": 0.9026069700717926,
+      "step": 365,
+      "train/kl_loss_qwen": 0.0276740618981421,
+      "train/kl_loss_r1": 0.014049774222075938,
+      "train/total_kl": 0.04172383584082127
+    },
+    {
+      "epoch": 0.4603421461897356,
+      "grad_norm": 2.59375,
+      "learning_rate": 2.6544209215442092e-05,
+      "loss": 0.5749,
+      "mean_token_accuracy": 0.9026127278804779,
+      "step": 370,
+      "train/kl_loss_qwen": 0.04495285819284618,
+      "train/kl_loss_r1": 0.020119147619698196,
+      "train/total_kl": 0.06507200542837381
+    },
+    {
+      "epoch": 0.4665629860031104,
+      "grad_norm": 2.484375,
+      "learning_rate": 2.6497509339975094e-05,
+      "loss": 0.4118,
+      "mean_token_accuracy": 0.9034813940525055,
+      "step": 375,
+      "train/kl_loss_qwen": 0.017691825143992902,
+      "train/kl_loss_r1": 0.008957023778930307,
+      "train/total_kl": 0.02664884882979095
+    },
+    {
+      "epoch": 0.4727838258164852,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.6450809464508093e-05,
+      "loss": 0.4096,
+      "mean_token_accuracy": 0.910720819234848,
+      "step": 380,
+      "train/kl_loss_qwen": 0.02144034132361412,
+      "train/kl_loss_r1": 0.010176640888676047,
+      "train/total_kl": 0.03161698216572404
+    },
+    {
+      "epoch": 0.47900466562986005,
+      "grad_norm": 2.171875,
+      "learning_rate": 2.6404109589041096e-05,
+      "loss": 0.4332,
+      "mean_token_accuracy": 0.9097962707281113,
+      "step": 385,
+      "train/kl_loss_qwen": 0.026963545754551888,
+      "train/kl_loss_r1": 0.012227852316573262,
+      "train/total_kl": 0.03919139839708805
+    },
+    {
+      "epoch": 0.48522550544323484,
+      "grad_norm": 2.5625,
+      "learning_rate": 2.6357409713574098e-05,
+      "loss": 0.4293,
+      "mean_token_accuracy": 0.910787695646286,
+      "step": 390,
+      "train/kl_loss_qwen": 0.02452428713440895,
+      "train/kl_loss_r1": 0.011624911101534963,
+      "train/total_kl": 0.03614919856190681
+    },
+    {
+      "epoch": 0.49144634525660963,
+      "grad_norm": 2.625,
+      "learning_rate": 2.6310709838107097e-05,
+      "loss": 0.4018,
+      "mean_token_accuracy": 0.9127192169427871,
+      "step": 395,
+      "train/kl_loss_qwen": 0.019982953229919075,
+      "train/kl_loss_r1": 0.009762761346064509,
+      "train/total_kl": 0.029745714645832776
+    },
+    {
+      "epoch": 0.4976671850699845,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.62640099626401e-05,
+      "loss": 0.4309,
+      "mean_token_accuracy": 0.9098518759012222,
+      "step": 400,
+      "train/kl_loss_qwen": 0.024961045105010272,
+      "train/kl_loss_r1": 0.011534130433574319,
+      "train/total_kl": 0.03649517530575395
+    },
+    {
+      "epoch": 0.5038880248833593,
+      "grad_norm": 2.125,
+      "learning_rate": 2.6217310087173102e-05,
+      "loss": 0.3924,
+      "mean_token_accuracy": 0.9169116854667664,
+      "step": 405,
+      "train/kl_loss_qwen": 0.021090314723551273,
+      "train/kl_loss_r1": 0.010099838604219258,
+      "train/total_kl": 0.031190153490751982
+    },
+    {
+      "epoch": 0.5101088646967341,
+      "grad_norm": 2.765625,
+      "learning_rate": 2.6170610211706104e-05,
+      "loss": 0.4122,
+      "mean_token_accuracy": 0.9040325850248336,
+      "step": 410,
+      "train/kl_loss_qwen": 0.01960067362524569,
+      "train/kl_loss_r1": 0.008205437916330993,
+      "train/total_kl": 0.027806111704558135
+    },
+    {
+      "epoch": 0.5163297045101088,
+      "grad_norm": 2.28125,
+      "learning_rate": 2.6123910336239106e-05,
+      "loss": 0.3899,
+      "mean_token_accuracy": 0.9136340916156769,
+      "step": 415,
+      "train/kl_loss_qwen": 0.019829964730888605,
+      "train/kl_loss_r1": 0.009041843877639621,
+      "train/total_kl": 0.028871809039264916
+    },
+    {
+      "epoch": 0.5225505443234837,
+      "grad_norm": 2.6875,
+      "learning_rate": 2.6077210460772105e-05,
+      "loss": 0.4553,
+      "mean_token_accuracy": 0.9060747653245926,
+      "step": 420,
+      "train/kl_loss_qwen": 0.02692516231909394,
+      "train/kl_loss_r1": 0.013944354082923383,
+      "train/total_kl": 0.0408695163205266
+    },
+    {
+      "epoch": 0.5287713841368584,
+      "grad_norm": 2.0625,
+      "learning_rate": 2.6030510585305108e-05,
+      "loss": 0.414,
+      "mean_token_accuracy": 0.9070446848869324,
+      "step": 425,
+      "train/kl_loss_qwen": 0.020327215222641824,
+      "train/kl_loss_r1": 0.009817754128016531,
+      "train/total_kl": 0.030144969280809165
+    },
+    {
+      "epoch": 0.5349922239502333,
+      "grad_norm": 2.203125,
+      "learning_rate": 2.5983810709838107e-05,
+      "loss": 0.4792,
+      "mean_token_accuracy": 0.9056743770837784,
+      "step": 430,
+      "train/kl_loss_qwen": 0.029629909666255116,
+      "train/kl_loss_r1": 0.01435271161608398,
+      "train/total_kl": 0.043982621002942326
+    },
+    {
+      "epoch": 0.5412130637636081,
+      "grad_norm": 2.828125,
+      "learning_rate": 2.593711083437111e-05,
+      "loss": 0.4092,
+      "mean_token_accuracy": 0.9107071310281754,
+      "step": 435,
+      "train/kl_loss_qwen": 0.02214355799369514,
+      "train/kl_loss_r1": 0.010212240810506045,
+      "train/total_kl": 0.03235579859465361
+    },
+    {
+      "epoch": 0.5474339035769828,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.589041095890411e-05,
+      "loss": 0.4217,
+      "mean_token_accuracy": 0.9096204102039337,
+      "step": 440,
+      "train/kl_loss_qwen": 0.02451937450096011,
+      "train/kl_loss_r1": 0.010314402729272842,
+      "train/total_kl": 0.03483377709053457
+    },
+    {
+      "epoch": 0.5536547433903577,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.584371108343711e-05,
+      "loss": 0.511,
+      "mean_token_accuracy": 0.9087847948074341,
+      "step": 445,
+      "train/kl_loss_qwen": 0.03897065857890993,
+      "train/kl_loss_r1": 0.013747443165630101,
+      "train/total_kl": 0.052718101628124715
+    },
+    {
+      "epoch": 0.5598755832037325,
+      "grad_norm": 2.546875,
+      "learning_rate": 2.5797011207970113e-05,
+      "loss": 0.4145,
+      "mean_token_accuracy": 0.9128732204437255,
+      "step": 450,
+      "train/kl_loss_qwen": 0.02431764816865325,
+      "train/kl_loss_r1": 0.010849157627671958,
+      "train/total_kl": 0.03516680607572198
+    },
+    {
+      "epoch": 0.5660964230171073,
+      "grad_norm": 2.75,
+      "learning_rate": 2.5750311332503115e-05,
+      "loss": 0.4282,
+      "mean_token_accuracy": 0.9059774994850158,
+      "step": 455,
+      "train/kl_loss_qwen": 0.022139727883040906,
+      "train/kl_loss_r1": 0.011035876907408237,
+      "train/total_kl": 0.03317560441792011
+    },
+    {
+      "epoch": 0.5723172628304821,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.5703611457036114e-05,
+      "loss": 0.4504,
+      "mean_token_accuracy": 0.8980892598628998,
+      "step": 460,
+      "train/kl_loss_qwen": 0.020996935572475194,
+      "train/kl_loss_r1": 0.011420201789587737,
+      "train/total_kl": 0.032417137362062934
+    },
+    {
+      "epoch": 0.578538102643857,
+      "grad_norm": 2.421875,
+      "learning_rate": 2.5656911581569117e-05,
+      "loss": 0.4246,
+      "mean_token_accuracy": 0.9115836918354034,
+      "step": 465,
+      "train/kl_loss_qwen": 0.025193213764578105,
+      "train/kl_loss_r1": 0.012585591687820852,
+      "train/total_kl": 0.0377788052894175
+    },
+    {
+      "epoch": 0.5847589424572317,
+      "grad_norm": 2.53125,
+      "learning_rate": 2.5610211706102116e-05,
+      "loss": 0.4263,
+      "mean_token_accuracy": 0.9069680899381638,
+      "step": 470,
+      "train/kl_loss_qwen": 0.02035662648268044,
+      "train/kl_loss_r1": 0.010218093637377024,
+      "train/total_kl": 0.03057471993379295
+    },
+    {
+      "epoch": 0.5909797822706065,
+      "grad_norm": 2.40625,
+      "learning_rate": 2.5563511830635118e-05,
+      "loss": 0.414,
+      "mean_token_accuracy": 0.9063103586435318,
+      "step": 475,
+      "train/kl_loss_qwen": 0.020925334934145213,
+      "train/kl_loss_r1": 0.009019026858732104,
+      "train/total_kl": 0.029944361280649902
+    },
+    {
+      "epoch": 0.5972006220839814,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.551681195516812e-05,
+      "loss": 0.4135,
+      "mean_token_accuracy": 0.9108444273471832,
+      "step": 480,
+      "train/kl_loss_qwen": 0.0211629044264555,
+      "train/kl_loss_r1": 0.009259478491730987,
+      "train/total_kl": 0.03042238261550665
+    },
+    {
+      "epoch": 0.6034214618973561,
+      "grad_norm": 2.8125,
+      "learning_rate": 2.547011207970112e-05,
+      "loss": 0.4338,
+      "mean_token_accuracy": 0.899581903219223,
+      "step": 485,
+      "train/kl_loss_qwen": 0.02072710790671408,
+      "train/kl_loss_r1": 0.010435436002444476,
+      "train/total_kl": 0.03116254387423396
+    },
+    {
+      "epoch": 0.609642301710731,
+      "grad_norm": 2.484375,
+      "learning_rate": 2.5423412204234122e-05,
+      "loss": 0.5159,
+      "mean_token_accuracy": 0.9059219062328339,
+      "step": 490,
+      "train/kl_loss_qwen": 0.03367695207707584,
+      "train/kl_loss_r1": 0.018230854975990952,
+      "train/total_kl": 0.05190780712291598
+    },
+    {
+      "epoch": 0.6158631415241057,
+      "grad_norm": 2.734375,
+      "learning_rate": 2.5376712328767124e-05,
+      "loss": 0.5261,
+      "mean_token_accuracy": 0.9024108290672302,
+      "step": 495,
+      "train/kl_loss_qwen": 0.03762428821064532,
+      "train/kl_loss_r1": 0.014978185016661882,
+      "train/total_kl": 0.05260247401893139
+    },
+    {
+      "epoch": 0.6220839813374806,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.5330012453300127e-05,
+      "loss": 0.4235,
+      "mean_token_accuracy": 0.9005639761686325,
+      "step": 500,
+      "train/kl_loss_qwen": 0.01958498265594244,
+      "train/kl_loss_r1": 0.009812347800470888,
+      "train/total_kl": 0.029397330433130264
+    },
+    {
+      "epoch": 0.6283048211508554,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.528331257783313e-05,
+      "loss": 0.4486,
+      "mean_token_accuracy": 0.9037535905838012,
+      "step": 505,
+      "train/kl_loss_qwen": 0.024349090829491615,
+      "train/kl_loss_r1": 0.01013639271259308,
+      "train/total_kl": 0.034485483448952435
+    },
+    {
+      "epoch": 0.6345256609642301,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.5236612702366128e-05,
+      "loss": 0.4169,
+      "mean_token_accuracy": 0.909102076292038,
+      "step": 510,
+      "train/kl_loss_qwen": 0.021461383812129496,
+      "train/kl_loss_r1": 0.00960217888932675,
+      "train/total_kl": 0.03106356244534254
+    },
+    {
+      "epoch": 0.640746500777605,
+      "grad_norm": 2.078125,
+      "learning_rate": 2.518991282689913e-05,
+      "loss": 0.5199,
+      "mean_token_accuracy": 0.9052909880876541,
+      "step": 515,
+      "train/kl_loss_qwen": 0.04063015528954565,
+      "train/kl_loss_r1": 0.012601481808815151,
+      "train/total_kl": 0.05323163778521121
+    },
+    {
+      "epoch": 0.6469673405909798,
+      "grad_norm": 2.546875,
+      "learning_rate": 2.514321295143213e-05,
+      "loss": 0.4219,
+      "mean_token_accuracy": 0.9009439706802368,
+      "step": 520,
+      "train/kl_loss_qwen": 0.022013092087581752,
+      "train/kl_loss_r1": 0.009523639199323952,
+      "train/total_kl": 0.03153673121705651
+    },
+    {
+      "epoch": 0.6531881804043546,
+      "grad_norm": 2.453125,
+      "learning_rate": 2.509651307596513e-05,
+      "loss": 0.3976,
+      "mean_token_accuracy": 0.9118671506643296,
+      "step": 525,
+      "train/kl_loss_qwen": 0.020429779263213276,
+      "train/kl_loss_r1": 0.009877034788951277,
+      "train/total_kl": 0.030306814052164555
+    },
+    {
+      "epoch": 0.6594090202177294,
+      "grad_norm": 2.71875,
+      "learning_rate": 2.5049813200498134e-05,
+      "loss": 0.4137,
+      "mean_token_accuracy": 0.9093097120523452,
+      "step": 530,
+      "train/kl_loss_qwen": 0.02071451968513429,
+      "train/kl_loss_r1": 0.010642997454851866,
+      "train/total_kl": 0.03135751727968454
+    },
+    {
+      "epoch": 0.6656298600311042,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.5003113325031133e-05,
+      "loss": 0.4233,
+      "mean_token_accuracy": 0.9019553482532501,
+      "step": 535,
+      "train/kl_loss_qwen": 0.018656383082270623,
+      "train/kl_loss_r1": 0.00829046352300793,
+      "train/total_kl": 0.026946846721693875
+    },
+    {
+      "epoch": 0.671850699844479,
+      "grad_norm": 2.671875,
+      "learning_rate": 2.4956413449564135e-05,
+      "loss": 0.4466,
+      "mean_token_accuracy": 0.9081733852624894,
+      "step": 540,
+      "train/kl_loss_qwen": 0.02441343287937343,
+      "train/kl_loss_r1": 0.014263224578462541,
+      "train/total_kl": 0.03867665748111904
+    },
+    {
+      "epoch": 0.6780715396578538,
+      "grad_norm": 2.75,
+      "learning_rate": 2.4909713574097134e-05,
+      "loss": 0.4312,
+      "mean_token_accuracy": 0.9087280422449112,
+      "step": 545,
+      "train/kl_loss_qwen": 0.024243433307856322,
+      "train/kl_loss_r1": 0.01274189695250243,
+      "train/total_kl": 0.036985330190509556
+    },
+    {
+      "epoch": 0.6842923794712286,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.4863013698630137e-05,
+      "loss": 0.4367,
+      "mean_token_accuracy": 0.9086010038852692,
+      "step": 550,
+      "train/kl_loss_qwen": 0.02496406654827297,
+      "train/kl_loss_r1": 0.012116139847785234,
+      "train/total_kl": 0.03708020634949207
+    },
+    {
+      "epoch": 0.6905132192846034,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.481631382316314e-05,
+      "loss": 0.4226,
+      "mean_token_accuracy": 0.90582574903965,
+      "step": 555,
+      "train/kl_loss_qwen": 0.022924260469153523,
+      "train/kl_loss_r1": 0.010547170951031148,
+      "train/total_kl": 0.033471431396901605
+    },
+    {
+      "epoch": 0.6967340590979783,
+      "grad_norm": 2.0625,
+      "learning_rate": 2.4769613947696138e-05,
+      "loss": 0.436,
+      "mean_token_accuracy": 0.9055513799190521,
+      "step": 560,
+      "train/kl_loss_qwen": 0.022723140241578222,
+      "train/kl_loss_r1": 0.010732360580004752,
+      "train/total_kl": 0.03345550089143216
+    },
+    {
+      "epoch": 0.702954898911353,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.472291407222914e-05,
+      "loss": 0.4205,
+      "mean_token_accuracy": 0.9103393256664276,
+      "step": 565,
+      "train/kl_loss_qwen": 0.022250397596508265,
+      "train/kl_loss_r1": 0.010711905942298473,
+      "train/total_kl": 0.03296230314299464
+    },
+    {
+      "epoch": 0.7091757387247278,
+      "grad_norm": 2.40625,
+      "learning_rate": 2.4676214196762143e-05,
+      "loss": 0.4279,
+      "mean_token_accuracy": 0.9037681072950363,
+      "step": 570,
+      "train/kl_loss_qwen": 0.02226592218503356,
+      "train/kl_loss_r1": 0.011035812250338495,
+      "train/total_kl": 0.03330173455178738
+    },
+    {
+      "epoch": 0.7153965785381027,
+      "grad_norm": 2.3125,
+      "learning_rate": 2.4629514321295142e-05,
+      "loss": 0.4242,
+      "mean_token_accuracy": 0.9072872966527938,
+      "step": 575,
+      "train/kl_loss_qwen": 0.023182417685166,
+      "train/kl_loss_r1": 0.01031209627399221,
+      "train/total_kl": 0.03349451422691345
+    },
+    {
+      "epoch": 0.7216174183514774,
+      "grad_norm": 2.65625,
+      "learning_rate": 2.4582814445828148e-05,
+      "loss": 0.4304,
+      "mean_token_accuracy": 0.9093700498342514,
+      "step": 580,
+      "train/kl_loss_qwen": 0.022953651519492267,
+      "train/kl_loss_r1": 0.012493736506439746,
+      "train/total_kl": 0.03544738814234734
+    },
+    {
+      "epoch": 0.7278382581648523,
+      "grad_norm": 2.34375,
+      "learning_rate": 2.4536114570361147e-05,
+      "loss": 0.4096,
+      "mean_token_accuracy": 0.9129402995109558,
+      "step": 585,
+      "train/kl_loss_qwen": 0.022961481800302865,
+      "train/kl_loss_r1": 0.01040629018098116,
+      "train/total_kl": 0.03336777174845338
+    },
+    {
+      "epoch": 0.7340590979782271,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.448941469489415e-05,
+      "loss": 0.3991,
+      "mean_token_accuracy": 0.914150008559227,
+      "step": 590,
+      "train/kl_loss_qwen": 0.022263350780121982,
+      "train/kl_loss_r1": 0.011094499076716603,
+      "train/total_kl": 0.03335785013623536
+    },
+    {
+      "epoch": 0.7402799377916018,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.444271481942715e-05,
+      "loss": 0.4076,
+      "mean_token_accuracy": 0.9096796810626984,
+      "step": 595,
+      "train/kl_loss_qwen": 0.020499514462426306,
+      "train/kl_loss_r1": 0.009279927227180452,
+      "train/total_kl": 0.02977944165468216
+    },
+    {
+      "epoch": 0.7465007776049767,
+      "grad_norm": 2.65625,
+      "learning_rate": 2.439601494396015e-05,
+      "loss": 0.4216,
+      "mean_token_accuracy": 0.9014455765485764,
+      "step": 600,
+      "train/kl_loss_qwen": 0.018622068548575045,
+      "train/kl_loss_r1": 0.009365804423578084,
+      "train/total_kl": 0.027987872809171678
+    },
+    {
+      "epoch": 0.7527216174183515,
+      "grad_norm": 2.296875,
+      "learning_rate": 2.4349315068493153e-05,
+      "loss": 0.3906,
+      "mean_token_accuracy": 0.9145099073648453,
+      "step": 605,
+      "train/kl_loss_qwen": 0.019467818038538098,
+      "train/kl_loss_r1": 0.008078851248137653,
+      "train/total_kl": 0.02754666917026043
+    },
+    {
+      "epoch": 0.7589424572317263,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.4302615193026152e-05,
+      "loss": 0.3987,
+      "mean_token_accuracy": 0.9153716772794723,
+      "step": 610,
+      "train/kl_loss_qwen": 0.022999429237097503,
+      "train/kl_loss_r1": 0.00981712534558028,
+      "train/total_kl": 0.03281655451282859
+    },
+    {
+      "epoch": 0.7651632970451011,
+      "grad_norm": 2.703125,
+      "learning_rate": 2.4255915317559154e-05,
+      "loss": 0.3885,
+      "mean_token_accuracy": 0.913951313495636,
+      "step": 615,
+      "train/kl_loss_qwen": 0.01929868240840733,
+      "train/kl_loss_r1": 0.009192716074176132,
+      "train/total_kl": 0.0284913981333375
+    },
+    {
+      "epoch": 0.7713841368584758,
+      "grad_norm": 2.515625,
+      "learning_rate": 2.4209215442092156e-05,
+      "loss": 0.438,
+      "mean_token_accuracy": 0.9032023847103119,
+      "step": 620,
+      "train/kl_loss_qwen": 0.02276699091307819,
+      "train/kl_loss_r1": 0.010404858528636396,
+      "train/total_kl": 0.03317184979096055
+    },
+    {
+      "epoch": 0.7776049766718507,
+      "grad_norm": 2.75,
+      "learning_rate": 2.4162515566625155e-05,
+      "loss": 0.4249,
+      "mean_token_accuracy": 0.901955908536911,
+      "step": 625,
+      "train/kl_loss_qwen": 0.022086183354258538,
+      "train/kl_loss_r1": 0.010839143022894859,
+      "train/total_kl": 0.03292532628402114
+    },
+    {
+      "epoch": 0.7838258164852255,
+      "grad_norm": 2.421875,
+      "learning_rate": 2.4115815691158158e-05,
+      "loss": 0.4441,
+      "mean_token_accuracy": 0.9012407660484314,
+      "step": 630,
+      "train/kl_loss_qwen": 0.021360087487846614,
+      "train/kl_loss_r1": 0.010610865615308284,
+      "train/total_kl": 0.0319709531031549
+    },
+    {
+      "epoch": 0.7900466562986003,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.4069115815691157e-05,
+      "loss": 0.4119,
+      "mean_token_accuracy": 0.9116487741470337,
+      "step": 635,
+      "train/kl_loss_qwen": 0.02234299052506685,
+      "train/kl_loss_r1": 0.010703041870146989,
+      "train/total_kl": 0.033046032395213844
+    },
+    {
+      "epoch": 0.7962674961119751,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.402241594022416e-05,
+      "loss": 0.3923,
+      "mean_token_accuracy": 0.9124638080596924,
+      "step": 640,
+      "train/kl_loss_qwen": 0.02015606495551765,
+      "train/kl_loss_r1": 0.008730803325306624,
+      "train/total_kl": 0.02888686824589968
+    },
+    {
+      "epoch": 0.80248833592535,
+      "grad_norm": 2.5625,
+      "learning_rate": 2.397571606475716e-05,
+      "loss": 0.4379,
+      "mean_token_accuracy": 0.9059850156307221,
+      "step": 645,
+      "train/kl_loss_qwen": 0.024274708051234484,
+      "train/kl_loss_r1": 0.013806939870119096,
+      "train/total_kl": 0.0380816476419568
+    },
+    {
+      "epoch": 0.8087091757387247,
+      "grad_norm": 2.484375,
+      "learning_rate": 2.392901618929016e-05,
+      "loss": 0.4658,
+      "mean_token_accuracy": 0.903114652633667,
+      "step": 650,
+      "train/kl_loss_qwen": 0.02699953648261726,
+      "train/kl_loss_r1": 0.01478456505574286,
+      "train/total_kl": 0.04178410097956657
+    },
+    {
+      "epoch": 0.8149300155520995,
+      "grad_norm": 2.25,
+      "learning_rate": 2.3882316313823163e-05,
+      "loss": 0.4109,
+      "mean_token_accuracy": 0.9127248078584671,
+      "step": 655,
+      "train/kl_loss_qwen": 0.022482301201671363,
+      "train/kl_loss_r1": 0.011614846624433994,
+      "train/total_kl": 0.03409714791923761
+    },
+    {
+      "epoch": 0.8211508553654744,
+      "grad_norm": 3.03125,
+      "learning_rate": 2.3835616438356165e-05,
+      "loss": 0.4755,
+      "mean_token_accuracy": 0.8993414014577865,
+      "step": 660,
+      "train/kl_loss_qwen": 0.02683520861901343,
+      "train/kl_loss_r1": 0.011668766860384493,
+      "train/total_kl": 0.03850397523492575
+    },
+    {
+      "epoch": 0.8273716951788491,
+      "grad_norm": 2.328125,
+      "learning_rate": 2.3788916562889164e-05,
+      "loss": 0.387,
+      "mean_token_accuracy": 0.9139132559299469,
+      "step": 665,
+      "train/kl_loss_qwen": 0.01782120973803103,
+      "train/kl_loss_r1": 0.007813481765333563,
+      "train/total_kl": 0.025634691584855318
+    },
+    {
+      "epoch": 0.833592534992224,
+      "grad_norm": 2.578125,
+      "learning_rate": 2.374221668742217e-05,
+      "loss": 0.4095,
+      "mean_token_accuracy": 0.911031749844551,
+      "step": 670,
+      "train/kl_loss_qwen": 0.02113759545609355,
+      "train/kl_loss_r1": 0.01039525493979454,
+      "train/total_kl": 0.03153285039588809
+    },
+    {
+      "epoch": 0.8398133748055988,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.369551681195517e-05,
+      "loss": 0.4168,
+      "mean_token_accuracy": 0.9161271333694458,
+      "step": 675,
+      "train/kl_loss_qwen": 0.024909159727394582,
+      "train/kl_loss_r1": 0.011709271254949272,
+      "train/total_kl": 0.036618431005626915
+    },
+    {
+      "epoch": 0.8460342146189735,
+      "grad_norm": 2.46875,
+      "learning_rate": 2.364881693648817e-05,
+      "loss": 0.3758,
+      "mean_token_accuracy": 0.9198156714439392,
+      "step": 680,
+      "train/kl_loss_qwen": 0.021275783283635975,
+      "train/kl_loss_r1": 0.009512535692192615,
+      "train/total_kl": 0.03078831881284714
+    },
+    {
+      "epoch": 0.8522550544323484,
+      "grad_norm": 2.640625,
+      "learning_rate": 2.3602117061021174e-05,
+      "loss": 0.4299,
+      "mean_token_accuracy": 0.9063263684511185,
+      "step": 685,
+      "train/kl_loss_qwen": 0.021453795861452817,
+      "train/kl_loss_r1": 0.010479483660310507,
+      "train/total_kl": 0.03193327952176332
+    },
+    {
+      "epoch": 0.8584758942457231,
+      "grad_norm": 2.546875,
+      "learning_rate": 2.3555417185554173e-05,
+      "loss": 0.4041,
+      "mean_token_accuracy": 0.913576290011406,
+      "step": 690,
+      "train/kl_loss_qwen": 0.022580369981005786,
+      "train/kl_loss_r1": 0.010107158054597675,
+      "train/total_kl": 0.03268752777948976
+    },
+    {
+      "epoch": 0.864696734059098,
+      "grad_norm": 2.84375,
+      "learning_rate": 2.3508717310087175e-05,
+      "loss": 0.4291,
+      "mean_token_accuracy": 0.9059417158365249,
+      "step": 695,
+      "train/kl_loss_qwen": 0.022479857504367828,
+      "train/kl_loss_r1": 0.010261117829941213,
+      "train/total_kl": 0.03274097526445985
+    },
+    {
+      "epoch": 0.8709175738724728,
+      "grad_norm": 2.71875,
+      "learning_rate": 2.3462017434620174e-05,
+      "loss": 0.4494,
+      "mean_token_accuracy": 0.9059775233268738,
+      "step": 700,
+      "train/kl_loss_qwen": 0.027120739314705135,
+      "train/kl_loss_r1": 0.011393017042428255,
+      "train/total_kl": 0.03851375617086887
+    },
+    {
+      "epoch": 0.8771384136858476,
+      "grad_norm": 2.25,
+      "learning_rate": 2.3415317559153177e-05,
+      "loss": 0.407,
+      "mean_token_accuracy": 0.9145919442176819,
+      "step": 705,
+      "train/kl_loss_qwen": 0.021546046063303948,
+      "train/kl_loss_r1": 0.012116323586087673,
+      "train/total_kl": 0.03366236938163638
+    },
+    {
+      "epoch": 0.8833592534992224,
+      "grad_norm": 2.390625,
+      "learning_rate": 2.336861768368618e-05,
+      "loss": 0.4101,
+      "mean_token_accuracy": 0.9063404113054275,
+      "step": 710,
+      "train/kl_loss_qwen": 0.020995961781591178,
+      "train/kl_loss_r1": 0.008313580113463104,
+      "train/total_kl": 0.029309542011469604
+    },
+    {
+      "epoch": 0.8895800933125972,
+      "grad_norm": 3.0,
+      "learning_rate": 2.3321917808219178e-05,
+      "loss": 0.412,
+      "mean_token_accuracy": 0.9092709690332412,
+      "step": 715,
+      "train/kl_loss_qwen": 0.020147784356959166,
+      "train/kl_loss_r1": 0.01101614481303841,
+      "train/total_kl": 0.031163928750902414
+    },
+    {
+      "epoch": 0.895800933125972,
+      "grad_norm": 2.765625,
+      "learning_rate": 2.327521793275218e-05,
+      "loss": 0.4405,
+      "mean_token_accuracy": 0.9061545938253402,
+      "step": 720,
+      "train/kl_loss_qwen": 0.024695659801363946,
+      "train/kl_loss_r1": 0.011185528873465956,
+      "train/total_kl": 0.035881188977509734
+    },
+    {
+      "epoch": 0.9020217729393468,
+      "grad_norm": 2.15625,
+      "learning_rate": 2.322851805728518e-05,
+      "loss": 0.4035,
+      "mean_token_accuracy": 0.9125464528799057,
+      "step": 725,
+      "train/kl_loss_qwen": 0.02164424294605851,
+      "train/kl_loss_r1": 0.011241717310622335,
+      "train/total_kl": 0.03288596007041633
+    },
+    {
+      "epoch": 0.9082426127527217,
+      "grad_norm": 3.015625,
+      "learning_rate": 2.318181818181818e-05,
+      "loss": 0.4681,
+      "mean_token_accuracy": 0.9115902453660965,
+      "step": 730,
+      "train/kl_loss_qwen": 0.031023595342412592,
+      "train/kl_loss_r1": 0.013560534256976097,
+      "train/total_kl": 0.04458412951789796
+    },
+    {
+      "epoch": 0.9144634525660964,
+      "grad_norm": 2.46875,
+      "learning_rate": 2.3135118306351184e-05,
+      "loss": 0.5105,
+      "mean_token_accuracy": 0.9097710341215134,
+      "step": 735,
+      "train/kl_loss_qwen": 0.04099215737078339,
+      "train/kl_loss_r1": 0.015071367053315044,
+      "train/total_kl": 0.05606352426111698
+    },
+    {
+      "epoch": 0.9206842923794712,
+      "grad_norm": 2.5,
+      "learning_rate": 2.3088418430884183e-05,
+      "loss": 0.4504,
+      "mean_token_accuracy": 0.9054893583059311,
+      "step": 740,
+      "train/kl_loss_qwen": 0.026908464403823017,
+      "train/kl_loss_r1": 0.011770134512335061,
+      "train/total_kl": 0.038678599148988725
+    },
+    {
+      "epoch": 0.926905132192846,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.3041718555417185e-05,
+      "loss": 0.41,
+      "mean_token_accuracy": 0.9051820576190949,
+      "step": 745,
+      "train/kl_loss_qwen": 0.018081901129335165,
+      "train/kl_loss_r1": 0.008661592192947865,
+      "train/total_kl": 0.026743493136018516
+    },
+    {
+      "epoch": 0.9331259720062208,
+      "grad_norm": 2.359375,
+      "learning_rate": 2.2995018679950184e-05,
+      "loss": 0.4378,
+      "mean_token_accuracy": 0.9121837258338928,
+      "step": 750,
+      "train/kl_loss_qwen": 0.027031197026371957,
+      "train/kl_loss_r1": 0.013320808461867274,
+      "train/total_kl": 0.04035200551152229
+    },
+    {
+      "epoch": 0.9393468118195957,
+      "grad_norm": 2.453125,
+      "learning_rate": 2.2948318804483187e-05,
+      "loss": 0.4181,
+      "mean_token_accuracy": 0.9081921458244324,
+      "step": 755,
+      "train/kl_loss_qwen": 0.021465251781046392,
+      "train/kl_loss_r1": 0.011396374832838774,
+      "train/total_kl": 0.032861626520752904
+    },
+    {
+      "epoch": 0.9455676516329704,
+      "grad_norm": 2.578125,
+      "learning_rate": 2.2901618929016193e-05,
+      "loss": 0.4726,
+      "mean_token_accuracy": 0.9009045392274857,
+      "step": 760,
+      "train/kl_loss_qwen": 0.02590537634678185,
+      "train/kl_loss_r1": 0.012987637892365456,
+      "train/total_kl": 0.03889301391318441
+    },
+    {
+      "epoch": 0.9517884914463453,
+      "grad_norm": 1.890625,
+      "learning_rate": 2.285491905354919e-05,
+      "loss": 0.3837,
+      "mean_token_accuracy": 0.9149051457643509,
+      "step": 765,
+      "train/kl_loss_qwen": 0.020421567978337406,
+      "train/kl_loss_r1": 0.009222430060617626,
+      "train/total_kl": 0.029643998201936483
+    },
+    {
+      "epoch": 0.9580093312597201,
+      "grad_norm": 2.234375,
+      "learning_rate": 2.2808219178082194e-05,
+      "loss": 0.4142,
+      "mean_token_accuracy": 0.910207200050354,
+      "step": 770,
+      "train/kl_loss_qwen": 0.02117032711394131,
+      "train/kl_loss_r1": 0.010742365522310138,
+      "train/total_kl": 0.031912692729383706
+    },
+    {
+      "epoch": 0.9642301710730948,
+      "grad_norm": 2.78125,
+      "learning_rate": 2.2761519302615196e-05,
+      "loss": 0.4285,
+      "mean_token_accuracy": 0.9055554062128067,
+      "step": 775,
+      "train/kl_loss_qwen": 0.022083445498719812,
+      "train/kl_loss_r1": 0.011104172887280583,
+      "train/total_kl": 0.03318761847913265
+    },
+    {
+      "epoch": 0.9704510108864697,
+      "grad_norm": 2.0625,
+      "learning_rate": 2.2714819427148195e-05,
+      "loss": 0.4154,
+      "mean_token_accuracy": 0.9221815407276154,
+      "step": 780,
+      "train/kl_loss_qwen": 0.028531118156388403,
+      "train/kl_loss_r1": 0.01254213583888486,
+      "train/total_kl": 0.041073253750801085
+    },
+    {
+      "epoch": 0.9766718506998445,
+      "grad_norm": 2.1875,
+      "learning_rate": 2.2668119551681198e-05,
+      "loss": 0.3951,
+      "mean_token_accuracy": 0.9119761139154434,
+      "step": 785,
+      "train/kl_loss_qwen": 0.020619897544384,
+      "train/kl_loss_r1": 0.009239812684245407,
+      "train/total_kl": 0.02985971048474312
+    },
+    {
+      "epoch": 0.9828926905132193,
+      "grad_norm": 57.75,
+      "learning_rate": 2.2621419676214197e-05,
+      "loss": 0.3857,
+      "mean_token_accuracy": 0.9099031835794449,
+      "step": 790,
+      "train/kl_loss_qwen": 0.018582486687228082,
+      "train/kl_loss_r1": 0.00894598305458203,
+      "train/total_kl": 0.027528469683602452
+    },
+    {
+      "epoch": 0.9891135303265941,
+      "grad_norm": 2.21875,
+      "learning_rate": 2.25747198007472e-05,
+      "loss": 0.4272,
+      "mean_token_accuracy": 0.9072646647691727,
+      "step": 795,
+      "train/kl_loss_qwen": 0.02253551883623004,
+      "train/kl_loss_r1": 0.010769509780220687,
+      "train/total_kl": 0.033305028639733794
+    },
+    {
+      "epoch": 0.995334370139969,
+      "grad_norm": 2.859375,
+      "learning_rate": 2.25280199252802e-05,
+      "loss": 0.5008,
+      "mean_token_accuracy": 0.9021875262260437,
+      "step": 800,
+      "train/kl_loss_qwen": 0.0319652761798352,
+      "train/kl_loss_r1": 0.012161148013547063,
+      "train/total_kl": 0.04412642428651452
+    },
+    {
+      "epoch": 1.00248833592535,
+      "grad_norm": 1.609375,
+      "learning_rate": 2.24813200498132e-05,
+      "loss": 0.4557,
+      "mean_token_accuracy": 0.9248083342205394,
+      "step": 805,
+      "train/kl_loss_qwen": 0.021597513886676592,
+      "train/kl_loss_r1": 0.013360376871952956,
+      "train/total_kl": 0.03495789039880037
+    },
+    {
+      "epoch": 1.0087091757387248,
+      "grad_norm": 1.4921875,
+      "learning_rate": 2.2434620174346203e-05,
+      "loss": 0.3201,
+      "mean_token_accuracy": 0.9592776656150818,
+      "step": 810,
+      "train/kl_loss_qwen": 0.026713818963617086,
+      "train/kl_loss_r1": 0.014814306050539017,
+      "train/total_kl": 0.041528125293552876
+    },
+    {
+      "epoch": 1.0149300155520995,
+      "grad_norm": 1.65625,
+      "learning_rate": 2.2387920298879202e-05,
+      "loss": 0.3216,
+      "mean_token_accuracy": 0.9562903255224228,
+      "step": 815,
+      "train/kl_loss_qwen": 0.0231754750944674,
+      "train/kl_loss_r1": 0.018418258568271995,
+      "train/total_kl": 0.041593733709305525
+    },
+    {
+      "epoch": 1.0211508553654742,
+      "grad_norm": 1.3046875,
+      "learning_rate": 2.2341220423412204e-05,
+      "loss": 0.2895,
+      "mean_token_accuracy": 0.9590234607458115,
+      "step": 820,
+      "train/kl_loss_qwen": 0.021737991878762842,
+      "train/kl_loss_r1": 0.013914320012554526,
+      "train/total_kl": 0.035652312077581884
+    },
+    {
+      "epoch": 1.0273716951788492,
+      "grad_norm": 1.890625,
+      "learning_rate": 2.2294520547945206e-05,
+      "loss": 0.3005,
+      "mean_token_accuracy": 0.9565304934978485,
+      "step": 825,
+      "train/kl_loss_qwen": 0.02263927822932601,
+      "train/kl_loss_r1": 0.012905962206423283,
+      "train/total_kl": 0.03554524062201381
+    },
+    {
+      "epoch": 1.033592534992224,
+      "grad_norm": 1.515625,
+      "learning_rate": 2.2247820672478205e-05,
+      "loss": 0.3013,
+      "mean_token_accuracy": 0.9635909259319305,
+      "step": 830,
+      "train/kl_loss_qwen": 0.024365014489740135,
+      "train/kl_loss_r1": 0.014652392477728426,
+      "train/total_kl": 0.03901740722358227
+    },
+    {
+      "epoch": 1.0398133748055987,
+      "grad_norm": 1.6796875,
+      "learning_rate": 2.2201120797011208e-05,
+      "loss": 0.316,
+      "mean_token_accuracy": 0.9590081989765167,
+      "step": 835,
+      "train/kl_loss_qwen": 0.023819171031937002,
+      "train/kl_loss_r1": 0.016059246635995804,
+      "train/total_kl": 0.03987841745838523
+    },
+    {
+      "epoch": 1.0460342146189736,
+      "grad_norm": 1.3828125,
+      "learning_rate": 2.2154420921544207e-05,
+      "loss": 0.3033,
+      "mean_token_accuracy": 0.9596952319145202,
+      "step": 840,
+      "train/kl_loss_qwen": 0.02370280809700489,
+      "train/kl_loss_r1": 0.014899721695110201,
+      "train/total_kl": 0.03860252983868122
+    },
+    {
+      "epoch": 1.0522550544323483,
+      "grad_norm": 1.234375,
+      "learning_rate": 2.210772104607721e-05,
+      "loss": 0.2822,
+      "mean_token_accuracy": 0.961972963809967,
+      "step": 845,
+      "train/kl_loss_qwen": 0.021481084870174527,
+      "train/kl_loss_r1": 0.01311062837485224,
+      "train/total_kl": 0.03459171336144209
+    },
+    {
+      "epoch": 1.058475894245723,
+      "grad_norm": 1.484375,
+      "learning_rate": 2.2061021170610215e-05,
+      "loss": 0.2895,
+      "mean_token_accuracy": 0.959705936908722,
+      "step": 850,
+      "train/kl_loss_qwen": 0.022235888708382844,
+      "train/kl_loss_r1": 0.013539392384700477,
+      "train/total_kl": 0.03577528093010187
+    },
+    {
+      "epoch": 1.064696734059098,
+      "grad_norm": 1.890625,
+      "learning_rate": 2.2014321295143214e-05,
+      "loss": 0.2909,
+      "mean_token_accuracy": 0.960792264342308,
+      "step": 855,
+      "train/kl_loss_qwen": 0.020066874427720906,
+      "train/kl_loss_r1": 0.014030058751814067,
+      "train/total_kl": 0.034096932876855134
+    },
+    {
+      "epoch": 1.0709175738724728,
+      "grad_norm": 1.6171875,
+      "learning_rate": 2.1967621419676216e-05,
+      "loss": 0.3042,
+      "mean_token_accuracy": 0.9614545047283173,
+      "step": 860,
+      "train/kl_loss_qwen": 0.023757481807842852,
+      "train/kl_loss_r1": 0.015206769714131952,
+      "train/total_kl": 0.03896425180137157
+    },
+    {
+      "epoch": 1.0771384136858475,
+      "grad_norm": 1.28125,
+      "learning_rate": 2.192092154420922e-05,
+      "loss": 0.3703,
+      "mean_token_accuracy": 0.955784410238266,
+      "step": 865,
+      "train/kl_loss_qwen": 0.03151101716794073,
+      "train/kl_loss_r1": 0.01914216240402311,
+      "train/total_kl": 0.050653179083019494
+    },
+    {
+      "epoch": 1.0833592534992225,
+      "grad_norm": 5.53125,
+      "learning_rate": 2.1874221668742218e-05,
+      "loss": 0.4169,
+      "mean_token_accuracy": 0.9540597677230835,
+      "step": 870,
+      "train/kl_loss_qwen": 0.03798724971711635,
+      "train/kl_loss_r1": 0.021596997044980526,
+      "train/total_kl": 0.05958424676209688
+    },
+    {
+      "epoch": 1.0895800933125972,
+      "grad_norm": 2.265625,
+      "learning_rate": 2.182752179327522e-05,
+      "loss": 0.3297,
+      "mean_token_accuracy": 0.9567724496126175,
+      "step": 875,
+      "train/kl_loss_qwen": 0.02518448540940881,
+      "train/kl_loss_r1": 0.015798166557215155,
+      "train/total_kl": 0.04098265203647315
+    },
+    {
+      "epoch": 1.095800933125972,
+      "grad_norm": 1.53125,
+      "learning_rate": 2.178082191780822e-05,
+      "loss": 0.2776,
+      "mean_token_accuracy": 0.96037577688694,
+      "step": 880,
+      "train/kl_loss_qwen": 0.019977604039013386,
+      "train/kl_loss_r1": 0.012469864799641073,
+      "train/total_kl": 0.032447469513863327
+    },
+    {
+      "epoch": 1.1020217729393469,
+      "grad_norm": 1.6796875,
+      "learning_rate": 2.173412204234122e-05,
+      "loss": 0.3031,
+      "mean_token_accuracy": 0.9599328011274337,
+      "step": 885,
+      "train/kl_loss_qwen": 0.02527875336818397,
+      "train/kl_loss_r1": 0.014116847841069102,
+      "train/total_kl": 0.0393956009298563
+    },
+    {
+      "epoch": 1.1082426127527216,
+      "grad_norm": 1.4921875,
+      "learning_rate": 2.1687422166874224e-05,
+      "loss": 0.3096,
+      "mean_token_accuracy": 0.9562761545181274,
+      "step": 890,
+      "train/kl_loss_qwen": 0.02247354043647647,
+      "train/kl_loss_r1": 0.01520557776093483,
+      "train/total_kl": 0.03767911838367581
+    },
+    {
+      "epoch": 1.1144634525660964,
+      "grad_norm": 1.5078125,
+      "learning_rate": 2.1640722291407223e-05,
+      "loss": 0.277,
+      "mean_token_accuracy": 0.9599349290132523,
+      "step": 895,
+      "train/kl_loss_qwen": 0.01961457757279277,
+      "train/kl_loss_r1": 0.012993672012817115,
+      "train/total_kl": 0.0326082497369498
+    },
+    {
+      "epoch": 1.1206842923794713,
+      "grad_norm": 1.546875,
+      "learning_rate": 2.1594022415940225e-05,
+      "loss": 0.3727,
+      "mean_token_accuracy": 0.9548643559217453,
+      "step": 900,
+      "train/kl_loss_qwen": 0.0343707412481308,
+      "train/kl_loss_r1": 0.017095418740063907,
+      "train/total_kl": 0.051466159708797934
+    },
+    {
+      "epoch": 1.126905132192846,
+      "grad_norm": 1.375,
+      "learning_rate": 2.1547322540473224e-05,
+      "loss": 0.3068,
+      "mean_token_accuracy": 0.9591044306755065,
+      "step": 905,
+      "train/kl_loss_qwen": 0.02452373835258186,
+      "train/kl_loss_r1": 0.01449969953391701,
+      "train/total_kl": 0.039023438654839995
+    },
+    {
+      "epoch": 1.1331259720062208,
+      "grad_norm": 1.3828125,
+      "learning_rate": 2.1500622665006227e-05,
+      "loss": 0.2977,
+      "mean_token_accuracy": 0.9605327278375626,
+      "step": 910,
+      "train/kl_loss_qwen": 0.023107501212507486,
+      "train/kl_loss_r1": 0.015223974105902016,
+      "train/total_kl": 0.038331475015729664
+    },
+    {
+      "epoch": 1.1393468118195957,
+      "grad_norm": 1.4609375,
+      "learning_rate": 2.145392278953923e-05,
+      "loss": 0.3055,
+      "mean_token_accuracy": 0.9603873282670975,
+      "step": 915,
+      "train/kl_loss_qwen": 0.023001648485660553,
+      "train/kl_loss_r1": 0.015112580894492567,
+      "train/total_kl": 0.0381142295897007
+    },
+    {
+      "epoch": 1.1455676516329705,
+      "grad_norm": 1.5703125,
+      "learning_rate": 2.1407222914072228e-05,
+      "loss": 0.3037,
+      "mean_token_accuracy": 0.9607974886894226,
+      "step": 920,
+      "train/kl_loss_qwen": 0.023130306228995322,
+      "train/kl_loss_r1": 0.013983825081959366,
+      "train/total_kl": 0.03711413135752082
+    },
+    {
+      "epoch": 1.1517884914463452,
+      "grad_norm": 1.203125,
+      "learning_rate": 2.136052303860523e-05,
+      "loss": 0.288,
+      "mean_token_accuracy": 0.9628516793251037,
+      "step": 925,
+      "train/kl_loss_qwen": 0.022921944176778196,
+      "train/kl_loss_r1": 0.01405964158475399,
+      "train/total_kl": 0.036981585528701545
+    },
+    {
+      "epoch": 1.15800933125972,
+      "grad_norm": 1.4375,
+      "learning_rate": 2.131382316313823e-05,
+      "loss": 0.3152,
+      "mean_token_accuracy": 0.9601940363645554,
+      "step": 930,
+      "train/kl_loss_qwen": 0.026545584946870805,
+      "train/kl_loss_r1": 0.014174379943870007,
+      "train/total_kl": 0.040719965007156134
+    },
+    {
+      "epoch": 1.164230171073095,
+      "grad_norm": 1.2734375,
+      "learning_rate": 2.126712328767123e-05,
+      "loss": 0.2835,
+      "mean_token_accuracy": 0.9574981659650803,
+      "step": 935,
+      "train/kl_loss_qwen": 0.019981388468295336,
+      "train/kl_loss_r1": 0.012800442287698388,
+      "train/total_kl": 0.032781831081956626
+    },
+    {
+      "epoch": 1.1704510108864696,
+      "grad_norm": 1.484375,
+      "learning_rate": 2.1220423412204237e-05,
+      "loss": 0.2909,
+      "mean_token_accuracy": 0.958210039138794,
+      "step": 940,
+      "train/kl_loss_qwen": 0.0211213490460068,
+      "train/kl_loss_r1": 0.012211221340112387,
+      "train/total_kl": 0.03333257073536515
+    },
+    {
+      "epoch": 1.1766718506998446,
+      "grad_norm": 1.7578125,
+      "learning_rate": 2.1173723536737236e-05,
+      "loss": 0.3088,
+      "mean_token_accuracy": 0.9566162914037705,
+      "step": 945,
+      "train/kl_loss_qwen": 0.023499226197600365,
+      "train/kl_loss_r1": 0.014358895621262491,
+      "train/total_kl": 0.037858121935278176
+    },
+    {
+      "epoch": 1.1828926905132193,
+      "grad_norm": 2.890625,
+      "learning_rate": 2.112702366127024e-05,
+      "loss": 0.3129,
+      "mean_token_accuracy": 0.9563288688659668,
+      "step": 950,
+      "train/kl_loss_qwen": 0.023788028536364435,
+      "train/kl_loss_r1": 0.015399044775404037,
+      "train/total_kl": 0.03918707333505154
+    },
+    {
+      "epoch": 1.189113530326594,
+      "grad_norm": 1.3515625,
+      "learning_rate": 2.1080323785803238e-05,
+      "loss": 0.2965,
+      "mean_token_accuracy": 0.9597578674554825,
+      "step": 955,
+      "train/kl_loss_qwen": 0.023407862475141882,
+      "train/kl_loss_r1": 0.013951084460131824,
+      "train/total_kl": 0.03735894681885839
+    },
+    {
+      "epoch": 1.1953343701399688,
+      "grad_norm": 1.546875,
+      "learning_rate": 2.103362391033624e-05,
+      "loss": 0.3034,
+      "mean_token_accuracy": 0.952317300438881,
+      "step": 960,
+      "train/kl_loss_qwen": 0.02142586586996913,
+      "train/kl_loss_r1": 0.013223635125905275,
+      "train/total_kl": 0.03464950108900666
+    },
+    {
+      "epoch": 1.2015552099533438,
+      "grad_norm": 1.7578125,
+      "learning_rate": 2.0986924034869242e-05,
+      "loss": 0.2921,
+      "mean_token_accuracy": 0.9569721490144729,
+      "step": 965,
+      "train/kl_loss_qwen": 0.020508702797815203,
+      "train/kl_loss_r1": 0.013173141260631382,
+      "train/total_kl": 0.033681843988597394
+    },
+    {
+      "epoch": 1.2077760497667185,
+      "grad_norm": 1.4921875,
+      "learning_rate": 2.094022415940224e-05,
+      "loss": 0.3207,
+      "mean_token_accuracy": 0.9579594194889068,
+      "step": 970,
+      "train/kl_loss_qwen": 0.026407206989824773,
+      "train/kl_loss_r1": 0.01604833439923823,
+      "train/total_kl": 0.04245554124936461
+    },
+    {
+      "epoch": 1.2139968895800932,
+      "grad_norm": 1.40625,
+      "learning_rate": 2.0893524283935244e-05,
+      "loss": 0.2691,
+      "mean_token_accuracy": 0.9618915826082229,
+      "step": 975,
+      "train/kl_loss_qwen": 0.020143463788554073,
+      "train/kl_loss_r1": 0.011573326494544744,
+      "train/total_kl": 0.031716789770871404
+    },
+    {
+      "epoch": 1.2202177293934682,
+      "grad_norm": 1.71875,
+      "learning_rate": 2.0846824408468246e-05,
+      "loss": 0.2878,
+      "mean_token_accuracy": 0.9602339863777161,
+      "step": 980,
+      "train/kl_loss_qwen": 0.021193863078951835,
+      "train/kl_loss_r1": 0.014284224039874971,
+      "train/total_kl": 0.03547808742150664
+    },
+    {
+      "epoch": 1.226438569206843,
+      "grad_norm": 1.546875,
+      "learning_rate": 2.0800124533001245e-05,
+      "loss": 0.2961,
+      "mean_token_accuracy": 0.9568452388048172,
+      "step": 985,
+      "train/kl_loss_qwen": 0.021635016007348896,
+      "train/kl_loss_r1": 0.012491077953018247,
+      "train/total_kl": 0.034126093704253437
+    },
+    {
+      "epoch": 1.2326594090202176,
+      "grad_norm": 1.4609375,
+      "learning_rate": 2.0753424657534248e-05,
+      "loss": 0.2762,
+      "mean_token_accuracy": 0.9612568289041519,
+      "step": 990,
+      "train/kl_loss_qwen": 0.020384321454912424,
+      "train/kl_loss_r1": 0.012731389002874493,
+      "train/total_kl": 0.03311571013182402
+    },
+    {
+      "epoch": 1.2388802488335926,
+      "grad_norm": 1.6484375,
+      "learning_rate": 2.0706724782067247e-05,
+      "loss": 0.298,
+      "mean_token_accuracy": 0.9548188507556915,
+      "step": 995,
+      "train/kl_loss_qwen": 0.02017202479764819,
+      "train/kl_loss_r1": 0.013068727403879165,
+      "train/total_kl": 0.0332407521083951
+    },
+    {
+      "epoch": 1.2451010886469673,
+      "grad_norm": 1.4140625,
+      "learning_rate": 2.066002490660025e-05,
+      "loss": 0.3913,
+      "mean_token_accuracy": 0.9523698449134826,
+      "step": 1000,
+      "train/kl_loss_qwen": 0.0328057702165097,
+      "train/kl_loss_r1": 0.017370638553984463,
+      "train/total_kl": 0.05017640832811594
+    },
+    {
+      "epoch": 1.251321928460342,
+      "grad_norm": 1.421875,
+      "learning_rate": 2.061332503113325e-05,
+      "loss": 0.2838,
+      "mean_token_accuracy": 0.9600636452436447,
+      "step": 1005,
+      "train/kl_loss_qwen": 0.02112743421457708,
+      "train/kl_loss_r1": 0.012237887969240547,
+      "train/total_kl": 0.033365322183817624
+    },
+    {
+      "epoch": 1.257542768273717,
+      "grad_norm": 1.453125,
+      "learning_rate": 2.056662515566625e-05,
+      "loss": 0.2888,
+      "mean_token_accuracy": 0.9591195553541183,
+      "step": 1010,
+      "train/kl_loss_qwen": 0.020449397200718523,
+      "train/kl_loss_r1": 0.013287889072671533,
+      "train/total_kl": 0.03373728627339005
+    },
+    {
+      "epoch": 1.2637636080870918,
+      "grad_norm": 1.875,
+      "learning_rate": 2.0519925280199253e-05,
+      "loss": 0.334,
+      "mean_token_accuracy": 0.9558851152658463,
+      "step": 1015,
+      "train/kl_loss_qwen": 0.02765529789030552,
+      "train/kl_loss_r1": 0.01525876906234771,
+      "train/total_kl": 0.04291406646370888
+    },
+    {
+      "epoch": 1.2699844479004665,
+      "grad_norm": 1.390625,
+      "learning_rate": 2.047322540473225e-05,
+      "loss": 0.2733,
+      "mean_token_accuracy": 0.9612768888473511,
+      "step": 1020,
+      "train/kl_loss_qwen": 0.02055009249597788,
+      "train/kl_loss_r1": 0.01219199935439974,
+      "train/total_kl": 0.03274209164083004
+    },
+    {
+      "epoch": 1.2762052877138412,
+      "grad_norm": 1.484375,
+      "learning_rate": 2.0426525529265254e-05,
+      "loss": 0.3006,
+      "mean_token_accuracy": 0.9604948818683624,
+      "step": 1025,
+      "train/kl_loss_qwen": 0.023509953077882527,
+      "train/kl_loss_r1": 0.013066381262615324,
+      "train/total_kl": 0.0365763345733285
+    },
+    {
+      "epoch": 1.2824261275272162,
+      "grad_norm": 1.234375,
+      "learning_rate": 2.037982565379826e-05,
+      "loss": 0.3579,
+      "mean_token_accuracy": 0.9586360841989517,
+      "step": 1030,
+      "train/kl_loss_qwen": 0.03005623547360301,
+      "train/kl_loss_r1": 0.018516876851208508,
+      "train/total_kl": 0.048573113046586514
+    },
+    {
+      "epoch": 1.288646967340591,
+      "grad_norm": 1.75,
+      "learning_rate": 2.033312577833126e-05,
+      "loss": 0.3284,
+      "mean_token_accuracy": 0.9601274847984314,
+      "step": 1035,
+      "train/kl_loss_qwen": 0.029562953114509582,
+      "train/kl_loss_r1": 0.014028432802297175,
+      "train/total_kl": 0.04359138542786241
+    },
+    {
+      "epoch": 1.2948678071539659,
+      "grad_norm": 1.4375,
+      "learning_rate": 2.028642590286426e-05,
+      "loss": 0.3047,
+      "mean_token_accuracy": 0.957324641942978,
+      "step": 1040,
+      "train/kl_loss_qwen": 0.02176124998368323,
+      "train/kl_loss_r1": 0.01420491524040699,
+      "train/total_kl": 0.03596616517752409
+    },
+    {
+      "epoch": 1.3010886469673406,
+      "grad_norm": 1.5390625,
+      "learning_rate": 2.023972602739726e-05,
+      "loss": 0.2959,
+      "mean_token_accuracy": 0.9622577458620072,
+      "step": 1045,
+      "train/kl_loss_qwen": 0.023116547102108597,
+      "train/kl_loss_r1": 0.014679647493176162,
+      "train/total_kl": 0.03779619401320815
+    },
+    {
+      "epoch": 1.3073094867807153,
+      "grad_norm": 1.3828125,
+      "learning_rate": 2.0193026151930263e-05,
+      "loss": 0.2817,
+      "mean_token_accuracy": 0.9572558134794236,
+      "step": 1050,
+      "train/kl_loss_qwen": 0.02008639704436064,
+      "train/kl_loss_r1": 0.012323370180092752,
+      "train/total_kl": 0.03240976668894291
+    },
+    {
+      "epoch": 1.31353032659409,
+      "grad_norm": 1.953125,
+      "learning_rate": 2.0146326276463265e-05,
+      "loss": 0.2899,
+      "mean_token_accuracy": 0.959894523024559,
+      "step": 1055,
+      "train/kl_loss_qwen": 0.022242504497990014,
+      "train/kl_loss_r1": 0.012922527687624096,
+      "train/total_kl": 0.03516503153368831
+    },
+    {
+      "epoch": 1.319751166407465,
+      "grad_norm": 1.5390625,
+      "learning_rate": 2.0099626400996264e-05,
+      "loss": 0.275,
+      "mean_token_accuracy": 0.9640595138072967,
+      "step": 1060,
+      "train/kl_loss_qwen": 0.020678923232480884,
+      "train/kl_loss_r1": 0.013264290383085608,
+      "train/total_kl": 0.033943213801831004
+    },
+    {
+      "epoch": 1.3259720062208398,
+      "grad_norm": 1.453125,
+      "learning_rate": 2.0052926525529266e-05,
+      "loss": 0.2958,
+      "mean_token_accuracy": 0.9585067182779312,
+      "step": 1065,
+      "train/kl_loss_qwen": 0.020578745240345597,
+      "train/kl_loss_r1": 0.013947037351317703,
+      "train/total_kl": 0.03452578242868185
+    },
+    {
+      "epoch": 1.3321928460342147,
+      "grad_norm": 1.46875,
+      "learning_rate": 2.000622665006227e-05,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.9577532559633255,
+      "step": 1070,
+      "train/kl_loss_qwen": 0.02328689182177186,
+      "train/kl_loss_r1": 0.01395503762178123,
+      "train/total_kl": 0.037241929210722446
+    },
+    {
+      "epoch": 1.3384136858475895,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.9959526774595268e-05,
+      "loss": 0.2788,
+      "mean_token_accuracy": 0.9609476655721665,
+      "step": 1075,
+      "train/kl_loss_qwen": 0.020258180052042007,
+      "train/kl_loss_r1": 0.013444960676133632,
+      "train/total_kl": 0.03370314054191113
+    },
+    {
+      "epoch": 1.3446345256609642,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.991282689912827e-05,
+      "loss": 0.2978,
+      "mean_token_accuracy": 0.9590980023145675,
+      "step": 1080,
+      "train/kl_loss_qwen": 0.02293569967150688,
+      "train/kl_loss_r1": 0.014317399612627924,
+      "train/total_kl": 0.037253099400550126
+    },
+    {
+      "epoch": 1.350855365474339,
+      "grad_norm": 1.359375,
+      "learning_rate": 1.986612702366127e-05,
+      "loss": 0.269,
+      "mean_token_accuracy": 0.9612076997756958,
+      "step": 1085,
+      "train/kl_loss_qwen": 0.020363600039854646,
+      "train/kl_loss_r1": 0.011795403668656945,
+      "train/total_kl": 0.032159003615379336
+    },
+    {
+      "epoch": 1.3570762052877139,
+      "grad_norm": 1.46875,
+      "learning_rate": 1.981942714819427e-05,
+      "loss": 0.3482,
+      "mean_token_accuracy": 0.9550175487995147,
+      "step": 1090,
+      "train/kl_loss_qwen": 0.029534393968060613,
+      "train/kl_loss_r1": 0.01619353631976992,
+      "train/total_kl": 0.04572793049737811
+    },
+    {
+      "epoch": 1.3632970451010886,
+      "grad_norm": 1.6171875,
+      "learning_rate": 1.9772727272727274e-05,
+      "loss": 0.2874,
+      "mean_token_accuracy": 0.9602297455072403,
+      "step": 1095,
+      "train/kl_loss_qwen": 0.020855060778558254,
+      "train/kl_loss_r1": 0.01363906858023256,
+      "train/total_kl": 0.03449412910267711
+    },
+    {
+      "epoch": 1.3695178849144636,
+      "grad_norm": 1.578125,
+      "learning_rate": 1.9726027397260273e-05,
+      "loss": 0.3941,
+      "mean_token_accuracy": 0.9562926113605499,
+      "step": 1100,
+      "train/kl_loss_qwen": 0.03488424099050462,
+      "train/kl_loss_r1": 0.01934248344041407,
+      "train/total_kl": 0.05422672387212515
+    },
+    {
+      "epoch": 1.3757387247278383,
+      "grad_norm": 1.5546875,
+      "learning_rate": 1.9679327521793275e-05,
+      "loss": 0.2959,
+      "mean_token_accuracy": 0.956820473074913,
+      "step": 1105,
+      "train/kl_loss_qwen": 0.021371696330606937,
+      "train/kl_loss_r1": 0.012425292772240937,
+      "train/total_kl": 0.033796988613903525
+    },
+    {
+      "epoch": 1.381959564541213,
+      "grad_norm": 1.6484375,
+      "learning_rate": 1.9632627646326274e-05,
+      "loss": 0.3097,
+      "mean_token_accuracy": 0.9579416275024414,
+      "step": 1110,
+      "train/kl_loss_qwen": 0.023708021268248558,
+      "train/kl_loss_r1": 0.015187849942594766,
+      "train/total_kl": 0.03889587111771107
+    },
+    {
+      "epoch": 1.3881804043545878,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.9585927770859276e-05,
+      "loss": 0.2784,
+      "mean_token_accuracy": 0.9591321825981141,
+      "step": 1115,
+      "train/kl_loss_qwen": 0.02066819160245359,
+      "train/kl_loss_r1": 0.011791627784259618,
+      "train/total_kl": 0.032459819409996274
+    },
+    {
+      "epoch": 1.3944012441679627,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.9539227895392282e-05,
+      "loss": 0.2905,
+      "mean_token_accuracy": 0.9603239595890045,
+      "step": 1120,
+      "train/kl_loss_qwen": 0.021381384460255504,
+      "train/kl_loss_r1": 0.014689616672694684,
+      "train/total_kl": 0.03607100090011954
+    },
+    {
+      "epoch": 1.4006220839813375,
+      "grad_norm": 3.125,
+      "learning_rate": 1.949252801992528e-05,
+      "loss": 0.39,
+      "mean_token_accuracy": 0.9580953747034073,
+      "step": 1125,
+      "train/kl_loss_qwen": 0.03591306242160499,
+      "train/kl_loss_r1": 0.018943295208737256,
+      "train/total_kl": 0.054856357537209986
+    },
+    {
+      "epoch": 1.4068429237947122,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.9445828144458284e-05,
+      "loss": 0.3111,
+      "mean_token_accuracy": 0.9572787255048751,
+      "step": 1130,
+      "train/kl_loss_qwen": 0.023058245377615093,
+      "train/kl_loss_r1": 0.01515980097465217,
+      "train/total_kl": 0.03821804653853178
+    },
+    {
+      "epoch": 1.4130637636080872,
+      "grad_norm": 1.375,
+      "learning_rate": 1.9399128268991283e-05,
+      "loss": 0.3609,
+      "mean_token_accuracy": 0.9568741142749786,
+      "step": 1135,
+      "train/kl_loss_qwen": 0.03462322899140417,
+      "train/kl_loss_r1": 0.015330694918520748,
+      "train/total_kl": 0.049953922908753154
+    },
+    {
+      "epoch": 1.419284603421462,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.9352428393524285e-05,
+      "loss": 0.2859,
+      "mean_token_accuracy": 0.9596155285835266,
+      "step": 1140,
+      "train/kl_loss_qwen": 0.019285118207335472,
+      "train/kl_loss_r1": 0.013246331200934946,
+      "train/total_kl": 0.032531449338421226
+    },
+    {
+      "epoch": 1.4255054432348366,
+      "grad_norm": 1.6328125,
+      "learning_rate": 1.9305728518057287e-05,
+      "loss": 0.3679,
+      "mean_token_accuracy": 0.9594572901725769,
+      "step": 1145,
+      "train/kl_loss_qwen": 0.03480704687535763,
+      "train/kl_loss_r1": 0.01769161957781762,
+      "train/total_kl": 0.0524986662901938
+    },
+    {
+      "epoch": 1.4317262830482116,
+      "grad_norm": 1.6015625,
+      "learning_rate": 1.9259028642590286e-05,
+      "loss": 0.3798,
+      "mean_token_accuracy": 0.9549993693828582,
+      "step": 1150,
+      "train/kl_loss_qwen": 0.033570041367784145,
+      "train/kl_loss_r1": 0.01750114664901048,
+      "train/total_kl": 0.0510711875744164
+    },
+    {
+      "epoch": 1.4379471228615863,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.921232876712329e-05,
+      "loss": 0.2985,
+      "mean_token_accuracy": 0.9594900667667389,
+      "step": 1155,
+      "train/kl_loss_qwen": 0.021839511627331377,
+      "train/kl_loss_r1": 0.014621772128157318,
+      "train/total_kl": 0.036461283918470144
+    },
+    {
+      "epoch": 1.444167962674961,
+      "grad_norm": 1.375,
+      "learning_rate": 1.9165628891656288e-05,
+      "loss": 0.291,
+      "mean_token_accuracy": 0.9573999226093293,
+      "step": 1160,
+      "train/kl_loss_qwen": 0.021014203736558556,
+      "train/kl_loss_r1": 0.013348353654146194,
+      "train/total_kl": 0.03436255753040314
+    },
+    {
+      "epoch": 1.450388802488336,
+      "grad_norm": 4.03125,
+      "learning_rate": 1.911892901618929e-05,
+      "loss": 0.4053,
+      "mean_token_accuracy": 0.9548583000898361,
+      "step": 1165,
+      "train/kl_loss_qwen": 0.03519875258207321,
+      "train/kl_loss_r1": 0.022582641919143498,
+      "train/total_kl": 0.0577813945710659
+    },
+    {
+      "epoch": 1.4566096423017107,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.9072229140722292e-05,
+      "loss": 0.3189,
+      "mean_token_accuracy": 0.9598561078310013,
+      "step": 1170,
+      "train/kl_loss_qwen": 0.02605470730923116,
+      "train/kl_loss_r1": 0.016693935636430977,
+      "train/total_kl": 0.04274864299222827
+    },
+    {
+      "epoch": 1.4628304821150855,
+      "grad_norm": 1.3984375,
+      "learning_rate": 1.902552926525529e-05,
+      "loss": 0.2911,
+      "mean_token_accuracy": 0.9594184875488281,
+      "step": 1175,
+      "train/kl_loss_qwen": 0.022187241865321995,
+      "train/kl_loss_r1": 0.013874154957011341,
+      "train/total_kl": 0.036061396915465595
+    },
+    {
+      "epoch": 1.4690513219284602,
+      "grad_norm": 2.328125,
+      "learning_rate": 1.8978829389788294e-05,
+      "loss": 0.3264,
+      "mean_token_accuracy": 0.9572916924953461,
+      "step": 1180,
+      "train/kl_loss_qwen": 0.0274376577232033,
+      "train/kl_loss_r1": 0.01724695498123765,
+      "train/total_kl": 0.04468461312353611
+    },
+    {
+      "epoch": 1.4752721617418352,
+      "grad_norm": 1.65625,
+      "learning_rate": 1.8932129514321296e-05,
+      "loss": 0.399,
+      "mean_token_accuracy": 0.956961777806282,
+      "step": 1185,
+      "train/kl_loss_qwen": 0.037932181172072886,
+      "train/kl_loss_r1": 0.02000022241845727,
+      "train/total_kl": 0.057932402938604355
+    },
+    {
+      "epoch": 1.48149300155521,
+      "grad_norm": 1.28125,
+      "learning_rate": 1.8885429638854295e-05,
+      "loss": 0.2916,
+      "mean_token_accuracy": 0.961290568113327,
+      "step": 1190,
+      "train/kl_loss_qwen": 0.02299168729223311,
+      "train/kl_loss_r1": 0.014748373185284436,
+      "train/total_kl": 0.03774006050080061
+    },
+    {
+      "epoch": 1.4877138413685849,
+      "grad_norm": 1.21875,
+      "learning_rate": 1.8838729763387298e-05,
+      "loss": 0.284,
+      "mean_token_accuracy": 0.9619271278381347,
+      "step": 1195,
+      "train/kl_loss_qwen": 0.021989288041368127,
+      "train/kl_loss_r1": 0.013442329177632929,
+      "train/total_kl": 0.03543161693960428
+    },
+    {
+      "epoch": 1.4939346811819596,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.8792029887920297e-05,
+      "loss": 0.2754,
+      "mean_token_accuracy": 0.9628610700368881,
+      "step": 1200,
+      "train/kl_loss_qwen": 0.01976469177752733,
+      "train/kl_loss_r1": 0.013379984023049473,
+      "train/total_kl": 0.03314467594027519
+    },
+    {
+      "epoch": 1.5001555209953343,
+      "grad_norm": 1.8671875,
+      "learning_rate": 1.8745330012453302e-05,
+      "loss": 0.3202,
+      "mean_token_accuracy": 0.9567913293838501,
+      "step": 1205,
+      "train/kl_loss_qwen": 0.02559811775572598,
+      "train/kl_loss_r1": 0.01404339938890189,
+      "train/total_kl": 0.03964151693508029
+    },
+    {
+      "epoch": 1.506376360808709,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.8698630136986305e-05,
+      "loss": 0.2755,
+      "mean_token_accuracy": 0.9610955238342285,
+      "step": 1210,
+      "train/kl_loss_qwen": 0.020736100152134895,
+      "train/kl_loss_r1": 0.012843929487280548,
+      "train/total_kl": 0.03358002956956625
+    },
+    {
+      "epoch": 1.512597200622084,
+      "grad_norm": 1.7109375,
+      "learning_rate": 1.8651930261519304e-05,
+      "loss": 0.3054,
+      "mean_token_accuracy": 0.9547863215208053,
+      "step": 1215,
+      "train/kl_loss_qwen": 0.022321202885359527,
+      "train/kl_loss_r1": 0.015055784210562705,
+      "train/total_kl": 0.03737698718905449
+    },
+    {
+      "epoch": 1.5188180404354588,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.8605230386052306e-05,
+      "loss": 0.2832,
+      "mean_token_accuracy": 0.9598489284515381,
+      "step": 1220,
+      "train/kl_loss_qwen": 0.02179465526714921,
+      "train/kl_loss_r1": 0.013072352437302471,
+      "train/total_kl": 0.03486700775101781
+    },
+    {
+      "epoch": 1.5250388802488337,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.8558530510585305e-05,
+      "loss": 0.2928,
+      "mean_token_accuracy": 0.9627665460109711,
+      "step": 1225,
+      "train/kl_loss_qwen": 0.022760304110124707,
+      "train/kl_loss_r1": 0.014947098214179277,
+      "train/total_kl": 0.037707402184605596
+    },
+    {
+      "epoch": 1.5312597200622085,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.8511830635118307e-05,
+      "loss": 0.2857,
+      "mean_token_accuracy": 0.9622293323278427,
+      "step": 1230,
+      "train/kl_loss_qwen": 0.021702085016295315,
+      "train/kl_loss_r1": 0.014505600836127997,
+      "train/total_kl": 0.03620768608525395
+    },
+    {
+      "epoch": 1.5374805598755832,
+      "grad_norm": 1.4375,
+      "learning_rate": 1.846513075965131e-05,
+      "loss": 0.3007,
+      "mean_token_accuracy": 0.9588985919952393,
+      "step": 1235,
+      "train/kl_loss_qwen": 0.023196784174069764,
+      "train/kl_loss_r1": 0.015077767241746188,
+      "train/total_kl": 0.03827455164864659
+    },
+    {
+      "epoch": 1.543701399688958,
+      "grad_norm": 1.3671875,
+      "learning_rate": 1.841843088418431e-05,
+      "loss": 0.3002,
+      "mean_token_accuracy": 0.9607412844896317,
+      "step": 1240,
+      "train/kl_loss_qwen": 0.02430763831362128,
+      "train/kl_loss_r1": 0.013863891735672951,
+      "train/total_kl": 0.03817152995616198
+    },
+    {
+      "epoch": 1.5499222395023327,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.837173100871731e-05,
+      "loss": 0.2951,
+      "mean_token_accuracy": 0.9558100968599319,
+      "step": 1245,
+      "train/kl_loss_qwen": 0.020561574772000313,
+      "train/kl_loss_r1": 0.014364191447384655,
+      "train/total_kl": 0.03492576666176319
+    },
+    {
+      "epoch": 1.5561430793157076,
+      "grad_norm": 1.5703125,
+      "learning_rate": 1.832503113325031e-05,
+      "loss": 0.2871,
+      "mean_token_accuracy": 0.9567299842834472,
+      "step": 1250,
+      "train/kl_loss_qwen": 0.02099115620367229,
+      "train/kl_loss_r1": 0.01232013343833387,
+      "train/total_kl": 0.03331128945574165
+    },
+    {
+      "epoch": 1.5623639191290826,
+      "grad_norm": 1.859375,
+      "learning_rate": 1.8278331257783313e-05,
+      "loss": 0.2965,
+      "mean_token_accuracy": 0.9580708712339401,
+      "step": 1255,
+      "train/kl_loss_qwen": 0.019678771495819092,
+      "train/kl_loss_r1": 0.013691711169667543,
+      "train/total_kl": 0.03337048254907131
+    },
+    {
+      "epoch": 1.5685847589424573,
+      "grad_norm": 1.625,
+      "learning_rate": 1.8231631382316315e-05,
+      "loss": 0.3153,
+      "mean_token_accuracy": 0.9576340615749359,
+      "step": 1260,
+      "train/kl_loss_qwen": 0.023866409715265036,
+      "train/kl_loss_r1": 0.014706146181561052,
+      "train/total_kl": 0.03857255624607205
+    },
+    {
+      "epoch": 1.574805598755832,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.8184931506849314e-05,
+      "loss": 0.2907,
+      "mean_token_accuracy": 0.9617926776409149,
+      "step": 1265,
+      "train/kl_loss_qwen": 0.021931132208555935,
+      "train/kl_loss_r1": 0.014819200243800878,
+      "train/total_kl": 0.036750332452356815
+    },
+    {
+      "epoch": 1.5810264385692068,
+      "grad_norm": 1.28125,
+      "learning_rate": 1.8138231631382316e-05,
+      "loss": 0.2851,
+      "mean_token_accuracy": 0.9603859275579453,
+      "step": 1270,
+      "train/kl_loss_qwen": 0.02175323711708188,
+      "train/kl_loss_r1": 0.01273711076937616,
+      "train/total_kl": 0.034490348072722554
+    },
+    {
+      "epoch": 1.5872472783825815,
+      "grad_norm": 1.578125,
+      "learning_rate": 1.809153175591532e-05,
+      "loss": 0.4085,
+      "mean_token_accuracy": 0.9475253760814667,
+      "step": 1275,
+      "train/kl_loss_qwen": 0.03326868480071425,
+      "train/kl_loss_r1": 0.017885031923651696,
+      "train/total_kl": 0.05115371756255627
+    },
+    {
+      "epoch": 1.5934681181959565,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.8044831880448318e-05,
+      "loss": 0.3215,
+      "mean_token_accuracy": 0.9612332046031952,
+      "step": 1280,
+      "train/kl_loss_qwen": 0.027783603593707084,
+      "train/kl_loss_r1": 0.01593417483381927,
+      "train/total_kl": 0.04371777819469571
+    },
+    {
+      "epoch": 1.5996889580093314,
+      "grad_norm": 1.921875,
+      "learning_rate": 1.799813200498132e-05,
+      "loss": 0.2871,
+      "mean_token_accuracy": 0.9571006387472153,
+      "step": 1285,
+      "train/kl_loss_qwen": 0.018922175094485284,
+      "train/kl_loss_r1": 0.011591637996025384,
+      "train/total_kl": 0.030513812974095344
+    },
+    {
+      "epoch": 1.6059097978227062,
+      "grad_norm": 1.3046875,
+      "learning_rate": 1.795143212951432e-05,
+      "loss": 0.3004,
+      "mean_token_accuracy": 0.9639281570911408,
+      "step": 1290,
+      "train/kl_loss_qwen": 0.025467291148379444,
+      "train/kl_loss_r1": 0.014465117454528808,
+      "train/total_kl": 0.039932408928871156
+    },
+    {
+      "epoch": 1.6121306376360809,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.7904732254047325e-05,
+      "loss": 0.3065,
+      "mean_token_accuracy": 0.957399845123291,
+      "step": 1295,
+      "train/kl_loss_qwen": 0.024825150426477195,
+      "train/kl_loss_r1": 0.013710612105205655,
+      "train/total_kl": 0.03853576248511672
+    },
+    {
+      "epoch": 1.6183514774494556,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.7858032378580327e-05,
+      "loss": 0.2833,
+      "mean_token_accuracy": 0.9580383241176605,
+      "step": 1300,
+      "train/kl_loss_qwen": 0.020569146750494836,
+      "train/kl_loss_r1": 0.012861439283005894,
+      "train/total_kl": 0.033430585730820896
+    },
+    {
+      "epoch": 1.6245723172628304,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.7811332503113326e-05,
+      "loss": 0.3294,
+      "mean_token_accuracy": 0.9609451532363892,
+      "step": 1305,
+      "train/kl_loss_qwen": 0.02771374033764005,
+      "train/kl_loss_r1": 0.01762062469497323,
+      "train/total_kl": 0.045334365032613275
+    },
+    {
+      "epoch": 1.6307931570762053,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.776463262764633e-05,
+      "loss": 0.2977,
+      "mean_token_accuracy": 0.9565259754657746,
+      "step": 1310,
+      "train/kl_loss_qwen": 0.022654768778011203,
+      "train/kl_loss_r1": 0.01242492652963847,
+      "train/total_kl": 0.03507969556376338
+    },
+    {
+      "epoch": 1.63701399688958,
+      "grad_norm": 1.75,
+      "learning_rate": 1.7717932752179327e-05,
+      "loss": 0.3123,
+      "mean_token_accuracy": 0.9604735493659973,
+      "step": 1315,
+      "train/kl_loss_qwen": 0.024692377913743256,
+      "train/kl_loss_r1": 0.015999246342107653,
+      "train/total_kl": 0.04069162430241704
+    },
+    {
+      "epoch": 1.643234836702955,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.767123287671233e-05,
+      "loss": 0.3212,
+      "mean_token_accuracy": 0.9578433007001876,
+      "step": 1320,
+      "train/kl_loss_qwen": 0.025619086902588607,
+      "train/kl_loss_r1": 0.015220096544362604,
+      "train/total_kl": 0.04083918360993266
+    },
+    {
+      "epoch": 1.6494556765163297,
+      "grad_norm": 1.2734375,
+      "learning_rate": 1.7624533001245332e-05,
+      "loss": 0.2734,
+      "mean_token_accuracy": 0.9615452647209167,
+      "step": 1325,
+      "train/kl_loss_qwen": 0.020037101954221724,
+      "train/kl_loss_r1": 0.01251753638498485,
+      "train/total_kl": 0.032554637920111415
+    },
+    {
+      "epoch": 1.6556765163297045,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.757783312577833e-05,
+      "loss": 0.3049,
+      "mean_token_accuracy": 0.9566170126199722,
+      "step": 1330,
+      "train/kl_loss_qwen": 0.022488067951053382,
+      "train/kl_loss_r1": 0.014429900841787458,
+      "train/total_kl": 0.03691796939820051
+    },
+    {
+      "epoch": 1.6618973561430792,
+      "grad_norm": 1.28125,
+      "learning_rate": 1.7531133250311334e-05,
+      "loss": 0.2719,
+      "mean_token_accuracy": 0.9647402167320251,
+      "step": 1335,
+      "train/kl_loss_qwen": 0.02126315711066127,
+      "train/kl_loss_r1": 0.013214132492430508,
+      "train/total_kl": 0.03447728957980871
+    },
+    {
+      "epoch": 1.6681181959564542,
+      "grad_norm": 1.2265625,
+      "learning_rate": 1.7484433374844333e-05,
+      "loss": 0.2843,
+      "mean_token_accuracy": 0.9628708630800247,
+      "step": 1340,
+      "train/kl_loss_qwen": 0.022394176525995135,
+      "train/kl_loss_r1": 0.012963168020360172,
+      "train/total_kl": 0.035357344802469014
+    },
+    {
+      "epoch": 1.674339035769829,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.7437733499377335e-05,
+      "loss": 0.3063,
+      "mean_token_accuracy": 0.95919728577137,
+      "step": 1345,
+      "train/kl_loss_qwen": 0.024591578030958773,
+      "train/kl_loss_r1": 0.014588873228058219,
+      "train/total_kl": 0.03918045153841376
+    },
+    {
+      "epoch": 1.6805598755832039,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.7391033623910337e-05,
+      "loss": 0.2814,
+      "mean_token_accuracy": 0.9604864865541458,
+      "step": 1350,
+      "train/kl_loss_qwen": 0.021181778702884912,
+      "train/kl_loss_r1": 0.012493171030655503,
+      "train/total_kl": 0.03367494996637106
+    },
+    {
+      "epoch": 1.6867807153965786,
+      "grad_norm": 1.734375,
+      "learning_rate": 1.7344333748443336e-05,
+      "loss": 0.2888,
+      "mean_token_accuracy": 0.962582352757454,
+      "step": 1355,
+      "train/kl_loss_qwen": 0.022155422298237682,
+      "train/kl_loss_r1": 0.01366903679445386,
+      "train/total_kl": 0.03582445904612541
+    },
+    {
+      "epoch": 1.6930015552099533,
+      "grad_norm": 1.640625,
+      "learning_rate": 1.729763387297634e-05,
+      "loss": 0.3055,
+      "mean_token_accuracy": 0.9639359444379807,
+      "step": 1360,
+      "train/kl_loss_qwen": 0.025513780070468783,
+      "train/kl_loss_r1": 0.01692169248126447,
+      "train/total_kl": 0.04243547255173326
+    },
+    {
+      "epoch": 1.699222395023328,
+      "grad_norm": 1.6171875,
+      "learning_rate": 1.725093399750934e-05,
+      "loss": 0.2947,
+      "mean_token_accuracy": 0.9534312933683395,
+      "step": 1365,
+      "train/kl_loss_qwen": 0.020309114549309015,
+      "train/kl_loss_r1": 0.01252038376405835,
+      "train/total_kl": 0.03282949859276414
+    },
+    {
+      "epoch": 1.7054432348367028,
+      "grad_norm": 1.3671875,
+      "learning_rate": 1.720423412204234e-05,
+      "loss": 0.2779,
+      "mean_token_accuracy": 0.9649826675653458,
+      "step": 1370,
+      "train/kl_loss_qwen": 0.022552193282172082,
+      "train/kl_loss_r1": 0.014152583619579672,
+      "train/total_kl": 0.03670477671548724
+    },
+    {
+      "epoch": 1.7116640746500777,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.7157534246575342e-05,
+      "loss": 0.2878,
+      "mean_token_accuracy": 0.9584995239973069,
+      "step": 1375,
+      "train/kl_loss_qwen": 0.022065887739881874,
+      "train/kl_loss_r1": 0.012487068446353078,
+      "train/total_kl": 0.034552955999970436
+    },
+    {
+      "epoch": 1.7178849144634527,
+      "grad_norm": 1.1328125,
+      "learning_rate": 1.711083437110834e-05,
+      "loss": 0.2609,
+      "mean_token_accuracy": 0.9647163540124893,
+      "step": 1380,
+      "train/kl_loss_qwen": 0.019539849366992712,
+      "train/kl_loss_r1": 0.011925394786521793,
+      "train/total_kl": 0.03146524429321289
+    },
+    {
+      "epoch": 1.7241057542768274,
+      "grad_norm": 1.7109375,
+      "learning_rate": 1.7064134495641347e-05,
+      "loss": 0.3011,
+      "mean_token_accuracy": 0.9581982225179673,
+      "step": 1385,
+      "train/kl_loss_qwen": 0.02257566642947495,
+      "train/kl_loss_r1": 0.013715238706208766,
+      "train/total_kl": 0.03629090515896678
+    },
+    {
+      "epoch": 1.7303265940902022,
+      "grad_norm": 1.328125,
+      "learning_rate": 1.701743462017435e-05,
+      "loss": 0.285,
+      "mean_token_accuracy": 0.9621599823236465,
+      "step": 1390,
+      "train/kl_loss_qwen": 0.022289714217185973,
+      "train/kl_loss_r1": 0.01341030071489513,
+      "train/total_kl": 0.035700014885514976
+    },
+    {
+      "epoch": 1.736547433903577,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.697073474470735e-05,
+      "loss": 0.2936,
+      "mean_token_accuracy": 0.9583886325359344,
+      "step": 1395,
+      "train/kl_loss_qwen": 0.021757268300279974,
+      "train/kl_loss_r1": 0.013224484142847359,
+      "train/total_kl": 0.03498175237327814
+    },
+    {
+      "epoch": 1.7427682737169516,
+      "grad_norm": 1.4765625,
+      "learning_rate": 1.692403486924035e-05,
+      "loss": 0.3241,
+      "mean_token_accuracy": 0.9611039608716965,
+      "step": 1400,
+      "train/kl_loss_qwen": 0.027939339494332673,
+      "train/kl_loss_r1": 0.015984538733027874,
+      "train/total_kl": 0.043923878204077484
+    },
+    {
+      "epoch": 1.7489891135303266,
+      "grad_norm": 1.6015625,
+      "learning_rate": 1.687733499377335e-05,
+      "loss": 0.3045,
+      "mean_token_accuracy": 0.9567447185516358,
+      "step": 1405,
+      "train/kl_loss_qwen": 0.02204110100865364,
+      "train/kl_loss_r1": 0.01382591521833092,
+      "train/total_kl": 0.03586701643653214
+    },
+    {
+      "epoch": 1.7552099533437016,
+      "grad_norm": 1.84375,
+      "learning_rate": 1.6830635118306352e-05,
+      "loss": 0.2962,
+      "mean_token_accuracy": 0.9537202090024948,
+      "step": 1410,
+      "train/kl_loss_qwen": 0.021085012378171085,
+      "train/kl_loss_r1": 0.01382909684907645,
+      "train/total_kl": 0.03491410920396447
+    },
+    {
+      "epoch": 1.7614307931570763,
+      "grad_norm": 1.5078125,
+      "learning_rate": 1.6783935242839355e-05,
+      "loss": 0.2926,
+      "mean_token_accuracy": 0.9642517894506455,
+      "step": 1415,
+      "train/kl_loss_qwen": 0.023424046859145165,
+      "train/kl_loss_r1": 0.014761518430896103,
+      "train/total_kl": 0.03818556563928723
+    },
+    {
+      "epoch": 1.767651632970451,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.6737235367372354e-05,
+      "loss": 0.3051,
+      "mean_token_accuracy": 0.9608754545450211,
+      "step": 1420,
+      "train/kl_loss_qwen": 0.024306372087448836,
+      "train/kl_loss_r1": 0.014464322733692825,
+      "train/total_kl": 0.03877069475129247
+    },
+    {
+      "epoch": 1.7738724727838258,
+      "grad_norm": 1.734375,
+      "learning_rate": 1.6690535491905356e-05,
+      "loss": 0.2896,
+      "mean_token_accuracy": 0.9598593056201935,
+      "step": 1425,
+      "train/kl_loss_qwen": 0.020282325707376002,
+      "train/kl_loss_r1": 0.012976178899407387,
+      "train/total_kl": 0.03325850497931242
+    },
+    {
+      "epoch": 1.7800933125972005,
+      "grad_norm": 1.4921875,
+      "learning_rate": 1.6643835616438355e-05,
+      "loss": 0.4165,
+      "mean_token_accuracy": 0.9545033365488053,
+      "step": 1430,
+      "train/kl_loss_qwen": 0.03436682634055614,
+      "train/kl_loss_r1": 0.0241848505102098,
+      "train/total_kl": 0.05855167773552239
+    },
+    {
+      "epoch": 1.7863141524105755,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.6597135740971357e-05,
+      "loss": 0.2962,
+      "mean_token_accuracy": 0.9586415469646454,
+      "step": 1435,
+      "train/kl_loss_qwen": 0.022492232220247387,
+      "train/kl_loss_r1": 0.013683093525469304,
+      "train/total_kl": 0.036175326071679594
+    },
+    {
+      "epoch": 1.7925349922239502,
+      "grad_norm": 1.6953125,
+      "learning_rate": 1.655043586550436e-05,
+      "loss": 0.3056,
+      "mean_token_accuracy": 0.9552174896001816,
+      "step": 1440,
+      "train/kl_loss_qwen": 0.021061650943011045,
+      "train/kl_loss_r1": 0.014696670183911919,
+      "train/total_kl": 0.035758321080356835
+    },
+    {
+      "epoch": 1.7987558320373251,
+      "grad_norm": 1.3125,
+      "learning_rate": 1.650373599003736e-05,
+      "loss": 0.3284,
+      "mean_token_accuracy": 0.9582234472036362,
+      "step": 1445,
+      "train/kl_loss_qwen": 0.026695101195946334,
+      "train/kl_loss_r1": 0.015962717402726413,
+      "train/total_kl": 0.042657819017767905
+    },
+    {
+      "epoch": 1.8049766718506999,
+      "grad_norm": 1.5234375,
+      "learning_rate": 1.645703611457036e-05,
+      "loss": 0.3002,
+      "mean_token_accuracy": 0.9622107416391372,
+      "step": 1450,
+      "train/kl_loss_qwen": 0.02437539123930037,
+      "train/kl_loss_r1": 0.014243083982728421,
+      "train/total_kl": 0.03861847519874573
+    },
+    {
+      "epoch": 1.8111975116640746,
+      "grad_norm": 1.2421875,
+      "learning_rate": 1.641033623910336e-05,
+      "loss": 0.2928,
+      "mean_token_accuracy": 0.9614711910486221,
+      "step": 1455,
+      "train/kl_loss_qwen": 0.02260896023362875,
+      "train/kl_loss_r1": 0.012275043106637896,
+      "train/total_kl": 0.03488400299102068
+    },
+    {
+      "epoch": 1.8174183514774493,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.6363636363636363e-05,
+      "loss": 0.2943,
+      "mean_token_accuracy": 0.9561317473649978,
+      "step": 1460,
+      "train/kl_loss_qwen": 0.020934791956096887,
+      "train/kl_loss_r1": 0.013324381457641721,
+      "train/total_kl": 0.034259173553436995
+    },
+    {
+      "epoch": 1.8236391912908243,
+      "grad_norm": 1.484375,
+      "learning_rate": 1.6316936488169365e-05,
+      "loss": 0.2998,
+      "mean_token_accuracy": 0.9572570383548736,
+      "step": 1465,
+      "train/kl_loss_qwen": 0.019881926383823155,
+      "train/kl_loss_r1": 0.013012451119720936,
+      "train/total_kl": 0.03289437731727958
+    },
+    {
+      "epoch": 1.829860031104199,
+      "grad_norm": 1.28125,
+      "learning_rate": 1.6270236612702364e-05,
+      "loss": 0.2816,
+      "mean_token_accuracy": 0.9600322395563126,
+      "step": 1470,
+      "train/kl_loss_qwen": 0.020596644142642617,
+      "train/kl_loss_r1": 0.01168474464211613,
+      "train/total_kl": 0.032281389273703096
+    },
+    {
+      "epoch": 1.836080870917574,
+      "grad_norm": 6.1875,
+      "learning_rate": 1.622353673723537e-05,
+      "loss": 0.4247,
+      "mean_token_accuracy": 0.9523273229598999,
+      "step": 1475,
+      "train/kl_loss_qwen": 0.04184676762670279,
+      "train/kl_loss_r1": 0.02051756768487394,
+      "train/total_kl": 0.06236433489248157
+    },
+    {
+      "epoch": 1.8423017107309487,
+      "grad_norm": 1.6015625,
+      "learning_rate": 1.6176836861768372e-05,
+      "loss": 0.277,
+      "mean_token_accuracy": 0.9600133568048477,
+      "step": 1480,
+      "train/kl_loss_qwen": 0.020399686740711333,
+      "train/kl_loss_r1": 0.012775455461815,
+      "train/total_kl": 0.03317514192312956
+    },
+    {
+      "epoch": 1.8485225505443235,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.613013698630137e-05,
+      "loss": 0.2965,
+      "mean_token_accuracy": 0.9654383957386017,
+      "step": 1485,
+      "train/kl_loss_qwen": 0.02537273187190294,
+      "train/kl_loss_r1": 0.015087466267868877,
+      "train/total_kl": 0.040460198651999235
+    },
+    {
+      "epoch": 1.8547433903576982,
+      "grad_norm": 1.4296875,
+      "learning_rate": 1.6083437110834373e-05,
+      "loss": 0.2804,
+      "mean_token_accuracy": 0.9586332231760025,
+      "step": 1490,
+      "train/kl_loss_qwen": 0.01983982524834573,
+      "train/kl_loss_r1": 0.011603046138770879,
+      "train/total_kl": 0.031442871689796446
+    },
+    {
+      "epoch": 1.860964230171073,
+      "grad_norm": 1.625,
+      "learning_rate": 1.6036737235367372e-05,
+      "loss": 0.2797,
+      "mean_token_accuracy": 0.9595640987157822,
+      "step": 1495,
+      "train/kl_loss_qwen": 0.019799032853916286,
+      "train/kl_loss_r1": 0.012844469002448022,
+      "train/total_kl": 0.03264350155368447
+    },
+    {
+      "epoch": 1.8671850699844479,
+      "grad_norm": 1.2734375,
+      "learning_rate": 1.5990037359900375e-05,
+      "loss": 0.3118,
+      "mean_token_accuracy": 0.9638541102409363,
+      "step": 1500,
+      "train/kl_loss_qwen": 0.026107014529407023,
+      "train/kl_loss_r1": 0.016707474808208646,
+      "train/total_kl": 0.04281448973342776
+    },
+    {
+      "epoch": 1.8734059097978228,
+      "grad_norm": 1.59375,
+      "learning_rate": 1.5943337484433377e-05,
+      "loss": 0.2922,
+      "mean_token_accuracy": 0.9552089840173721,
+      "step": 1505,
+      "train/kl_loss_qwen": 0.0205880144611001,
+      "train/kl_loss_r1": 0.01196823725476861,
+      "train/total_kl": 0.03255625148303807
+    },
+    {
+      "epoch": 1.8796267496111976,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.5896637608966376e-05,
+      "loss": 0.3049,
+      "mean_token_accuracy": 0.9595163345336915,
+      "step": 1510,
+      "train/kl_loss_qwen": 0.024258351232856513,
+      "train/kl_loss_r1": 0.01617993521504104,
+      "train/total_kl": 0.04043828658759594
+    },
+    {
+      "epoch": 1.8858475894245723,
+      "grad_norm": 1.5,
+      "learning_rate": 1.584993773349938e-05,
+      "loss": 0.2934,
+      "mean_token_accuracy": 0.9597264170646668,
+      "step": 1515,
+      "train/kl_loss_qwen": 0.023380379332229494,
+      "train/kl_loss_r1": 0.013724281871691345,
+      "train/total_kl": 0.03710466120392084
+    },
+    {
+      "epoch": 1.892068429237947,
+      "grad_norm": 1.375,
+      "learning_rate": 1.5803237858032377e-05,
+      "loss": 0.2886,
+      "mean_token_accuracy": 0.9607859402894974,
+      "step": 1520,
+      "train/kl_loss_qwen": 0.021867345878854395,
+      "train/kl_loss_r1": 0.014179157582111657,
+      "train/total_kl": 0.036046503484249114
+    },
+    {
+      "epoch": 1.8982892690513218,
+      "grad_norm": 1.6484375,
+      "learning_rate": 1.575653798256538e-05,
+      "loss": 0.2992,
+      "mean_token_accuracy": 0.9605538129806519,
+      "step": 1525,
+      "train/kl_loss_qwen": 0.023080370994284748,
+      "train/kl_loss_r1": 0.01472889562137425,
+      "train/total_kl": 0.03780926624312997
+    },
+    {
+      "epoch": 1.9045101088646967,
+      "grad_norm": 1.875,
+      "learning_rate": 1.5709838107098382e-05,
+      "loss": 0.3005,
+      "mean_token_accuracy": 0.9580298513174057,
+      "step": 1530,
+      "train/kl_loss_qwen": 0.02314431550912559,
+      "train/kl_loss_r1": 0.013429639278911054,
+      "train/total_kl": 0.036573955044150355
+    },
+    {
+      "epoch": 1.9107309486780717,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.566313823163138e-05,
+      "loss": 0.2749,
+      "mean_token_accuracy": 0.9649899780750275,
+      "step": 1535,
+      "train/kl_loss_qwen": 0.02057855702005327,
+      "train/kl_loss_r1": 0.014282949059270323,
+      "train/total_kl": 0.03486150596290827
+    },
+    {
+      "epoch": 1.9169517884914464,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.5616438356164384e-05,
+      "loss": 0.2972,
+      "mean_token_accuracy": 0.9626806735992431,
+      "step": 1540,
+      "train/kl_loss_qwen": 0.025072267558425666,
+      "train/kl_loss_r1": 0.015387872420251369,
+      "train/total_kl": 0.04046014007180929
+    },
+    {
+      "epoch": 1.9231726283048212,
+      "grad_norm": 1.6015625,
+      "learning_rate": 1.5569738480697383e-05,
+      "loss": 0.2917,
+      "mean_token_accuracy": 0.9622645407915116,
+      "step": 1545,
+      "train/kl_loss_qwen": 0.022078696498647334,
+      "train/kl_loss_r1": 0.014551608706824481,
+      "train/total_kl": 0.036630304995924234
+    },
+    {
+      "epoch": 1.929393468118196,
+      "grad_norm": 1.625,
+      "learning_rate": 1.5523038605230385e-05,
+      "loss": 0.3041,
+      "mean_token_accuracy": 0.9566681832075119,
+      "step": 1550,
+      "train/kl_loss_qwen": 0.02196988635696471,
+      "train/kl_loss_r1": 0.014778985898010433,
+      "train/total_kl": 0.03674887223169208
+    },
+    {
+      "epoch": 1.9356143079315706,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.5476338729763387e-05,
+      "loss": 0.3967,
+      "mean_token_accuracy": 0.9541066825389862,
+      "step": 1555,
+      "train/kl_loss_qwen": 0.03710188092663884,
+      "train/kl_loss_r1": 0.018790926854126155,
+      "train/total_kl": 0.055892806779593227
+    },
+    {
+      "epoch": 1.9418351477449456,
+      "grad_norm": 1.4296875,
+      "learning_rate": 1.5429638854296386e-05,
+      "loss": 0.3105,
+      "mean_token_accuracy": 0.9602783590555191,
+      "step": 1560,
+      "train/kl_loss_qwen": 0.02596303396858275,
+      "train/kl_loss_r1": 0.0147313819732517,
+      "train/total_kl": 0.040694416221231225
+    },
+    {
+      "epoch": 1.9480559875583203,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.5382938978829392e-05,
+      "loss": 0.2919,
+      "mean_token_accuracy": 0.95930115878582,
+      "step": 1565,
+      "train/kl_loss_qwen": 0.02266333531588316,
+      "train/kl_loss_r1": 0.013036081078462302,
+      "train/total_kl": 0.03569941623136401
+    },
+    {
+      "epoch": 1.9542768273716953,
+      "grad_norm": 1.2421875,
+      "learning_rate": 1.5336239103362394e-05,
+      "loss": 0.3061,
+      "mean_token_accuracy": 0.9625548154115677,
+      "step": 1570,
+      "train/kl_loss_qwen": 0.026318618888035418,
+      "train/kl_loss_r1": 0.015444871853105724,
+      "train/total_kl": 0.04176349071785808
+    },
+    {
+      "epoch": 1.96049766718507,
+      "grad_norm": 1.515625,
+      "learning_rate": 1.5289539227895393e-05,
+      "loss": 0.2886,
+      "mean_token_accuracy": 0.9617161154747009,
+      "step": 1575,
+      "train/kl_loss_qwen": 0.02043143748305738,
+      "train/kl_loss_r1": 0.014007525471970438,
+      "train/total_kl": 0.03443896248936653
+    },
+    {
+      "epoch": 1.9667185069984447,
+      "grad_norm": 1.4453125,
+      "learning_rate": 1.5242839352428394e-05,
+      "loss": 0.296,
+      "mean_token_accuracy": 0.958229985833168,
+      "step": 1580,
+      "train/kl_loss_qwen": 0.020992166455835105,
+      "train/kl_loss_r1": 0.013891345215961337,
+      "train/total_kl": 0.03488351134583354
+    },
+    {
+      "epoch": 1.9729393468118195,
+      "grad_norm": 1.7109375,
+      "learning_rate": 1.5196139476961396e-05,
+      "loss": 0.303,
+      "mean_token_accuracy": 0.955667045712471,
+      "step": 1585,
+      "train/kl_loss_qwen": 0.021437043463811277,
+      "train/kl_loss_r1": 0.013299981201998889,
+      "train/total_kl": 0.03473702482879162
+    },
+    {
+      "epoch": 1.9791601866251944,
+      "grad_norm": 3.46875,
+      "learning_rate": 1.5149439601494397e-05,
+      "loss": 0.3863,
+      "mean_token_accuracy": 0.9547875672578812,
+      "step": 1590,
+      "train/kl_loss_qwen": 0.03339828597381711,
+      "train/kl_loss_r1": 0.019195286463946103,
+      "train/total_kl": 0.05259357215836644
+    },
+    {
+      "epoch": 1.9853810264385692,
+      "grad_norm": 1.40625,
+      "learning_rate": 1.5102739726027398e-05,
+      "loss": 0.302,
+      "mean_token_accuracy": 0.9607174038887024,
+      "step": 1595,
+      "train/kl_loss_qwen": 0.024118094984441996,
+      "train/kl_loss_r1": 0.015118178050033747,
+      "train/total_kl": 0.039236273430287835
+    },
+    {
+      "epoch": 1.9916018662519441,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.5056039850560399e-05,
+      "loss": 0.2702,
+      "mean_token_accuracy": 0.9614597886800766,
+      "step": 1600,
+      "train/kl_loss_qwen": 0.02008180283010006,
+      "train/kl_loss_r1": 0.011453768424689769,
+      "train/total_kl": 0.031535571161657575
+    },
+    {
+      "epoch": 1.9978227060653189,
+      "grad_norm": 1.1640625,
+      "learning_rate": 1.5009339975093401e-05,
+      "loss": 0.2883,
+      "mean_token_accuracy": 0.9607007771730423,
+      "step": 1605,
+      "train/kl_loss_qwen": 0.023967621568590402,
+      "train/kl_loss_r1": 0.01288898903876543,
+      "train/total_kl": 0.03685661060735583
+    },
+    {
+      "epoch": 2.0049766718507,
+      "grad_norm": 1.3515625,
+      "learning_rate": 1.4962640099626402e-05,
+      "loss": 0.2945,
+      "mean_token_accuracy": 0.9731682376428084,
+      "step": 1610,
+      "train/kl_loss_qwen": 0.021845903649756856,
+      "train/kl_loss_r1": 0.012560325928709724,
+      "train/total_kl": 0.034406229599633
+    },
+    {
+      "epoch": 2.011197511664075,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.4915940224159402e-05,
+      "loss": 0.2426,
+      "mean_token_accuracy": 0.9784580409526825,
+      "step": 1615,
+      "train/kl_loss_qwen": 0.02075221575796604,
+      "train/kl_loss_r1": 0.012880782317370176,
+      "train/total_kl": 0.03363299807533622
+    },
+    {
+      "epoch": 2.0174183514774495,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.4869240348692403e-05,
+      "loss": 0.2472,
+      "mean_token_accuracy": 0.9778133362531662,
+      "step": 1620,
+      "train/kl_loss_qwen": 0.02218270841985941,
+      "train/kl_loss_r1": 0.012616124283522368,
+      "train/total_kl": 0.03479883261024952
+    },
+    {
+      "epoch": 2.0236391912908243,
+      "grad_norm": 0.9296875,
+      "learning_rate": 1.4822540473225404e-05,
+      "loss": 0.2411,
+      "mean_token_accuracy": 0.9795323252677918,
+      "step": 1625,
+      "train/kl_loss_qwen": 0.021198717644438148,
+      "train/kl_loss_r1": 0.013453285675495863,
+      "train/total_kl": 0.034652003180235626
+    },
+    {
+      "epoch": 2.029860031104199,
+      "grad_norm": 0.9765625,
+      "learning_rate": 1.4775840597758408e-05,
+      "loss": 0.2423,
+      "mean_token_accuracy": 0.9800434708595276,
+      "step": 1630,
+      "train/kl_loss_qwen": 0.02066632085479796,
+      "train/kl_loss_r1": 0.014415074535645545,
+      "train/total_kl": 0.03508139550685883
+    },
+    {
+      "epoch": 2.0360808709175737,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.4729140722291408e-05,
+      "loss": 0.2389,
+      "mean_token_accuracy": 0.9806335180997848,
+      "step": 1635,
+      "train/kl_loss_qwen": 0.02023201258853078,
+      "train/kl_loss_r1": 0.01314310000743717,
+      "train/total_kl": 0.0333751130849123
+    },
+    {
+      "epoch": 2.0423017107309485,
+      "grad_norm": 1.0078125,
+      "learning_rate": 1.4682440846824409e-05,
+      "loss": 0.2651,
+      "mean_token_accuracy": 0.9811090856790543,
+      "step": 1640,
+      "train/kl_loss_qwen": 0.024384100595489143,
+      "train/kl_loss_r1": 0.016845971532166003,
+      "train/total_kl": 0.04123007152229548
+    },
+    {
+      "epoch": 2.0485225505443236,
+      "grad_norm": 0.828125,
+      "learning_rate": 1.463574097135741e-05,
+      "loss": 0.244,
+      "mean_token_accuracy": 0.9806887865066528,
+      "step": 1645,
+      "train/kl_loss_qwen": 0.02161189350299537,
+      "train/kl_loss_r1": 0.01435529210139066,
+      "train/total_kl": 0.035967185720801355
+    },
+    {
+      "epoch": 2.0547433903576984,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.4589041095890412e-05,
+      "loss": 0.2589,
+      "mean_token_accuracy": 0.9809038013219833,
+      "step": 1650,
+      "train/kl_loss_qwen": 0.023991528805345295,
+      "train/kl_loss_r1": 0.015264363773167134,
+      "train/total_kl": 0.039255892764776944
+    },
+    {
+      "epoch": 2.060964230171073,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.4542341220423413e-05,
+      "loss": 0.2429,
+      "mean_token_accuracy": 0.9789062738418579,
+      "step": 1655,
+      "train/kl_loss_qwen": 0.020654200948774813,
+      "train/kl_loss_r1": 0.013649226189590991,
+      "train/total_kl": 0.03430342748761177
+    },
+    {
+      "epoch": 2.067185069984448,
+      "grad_norm": 2.1875,
+      "learning_rate": 1.4495641344956414e-05,
+      "loss": 0.3744,
+      "mean_token_accuracy": 0.9751468390226364,
+      "step": 1660,
+      "train/kl_loss_qwen": 0.03906259112991393,
+      "train/kl_loss_r1": 0.020116307865828277,
+      "train/total_kl": 0.05917889960110188
+    },
+    {
+      "epoch": 2.0734059097978226,
+      "grad_norm": 1.1015625,
+      "learning_rate": 1.4448941469489414e-05,
+      "loss": 0.2823,
+      "mean_token_accuracy": 0.977814581990242,
+      "step": 1665,
+      "train/kl_loss_qwen": 0.02684653955511749,
+      "train/kl_loss_r1": 0.016426597093231975,
+      "train/total_kl": 0.04327313648536801
+    },
+    {
+      "epoch": 2.0796267496111973,
+      "grad_norm": 0.890625,
+      "learning_rate": 1.4402241594022415e-05,
+      "loss": 0.241,
+      "mean_token_accuracy": 0.978304636478424,
+      "step": 1670,
+      "train/kl_loss_qwen": 0.01918859565630555,
+      "train/kl_loss_r1": 0.012731851963326334,
+      "train/total_kl": 0.03192044720053673
+    },
+    {
+      "epoch": 2.0858475894245725,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.4355541718555419e-05,
+      "loss": 0.2495,
+      "mean_token_accuracy": 0.9780581414699554,
+      "step": 1675,
+      "train/kl_loss_qwen": 0.022329255240038037,
+      "train/kl_loss_r1": 0.013668485963717104,
+      "train/total_kl": 0.035997741110622886
+    },
+    {
+      "epoch": 2.0920684292379472,
+      "grad_norm": 2.0,
+      "learning_rate": 1.430884184308842e-05,
+      "loss": 0.3164,
+      "mean_token_accuracy": 0.973807492852211,
+      "step": 1680,
+      "train/kl_loss_qwen": 0.0302242751698941,
+      "train/kl_loss_r1": 0.01509574861265719,
+      "train/total_kl": 0.045320024248212576
+    },
+    {
+      "epoch": 2.098289269051322,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.426214196762142e-05,
+      "loss": 0.2657,
+      "mean_token_accuracy": 0.9796731650829316,
+      "step": 1685,
+      "train/kl_loss_qwen": 0.024477861635386945,
+      "train/kl_loss_r1": 0.016039001010358333,
+      "train/total_kl": 0.04051686236634851
+    },
+    {
+      "epoch": 2.1045101088646967,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.4215442092154421e-05,
+      "loss": 0.2612,
+      "mean_token_accuracy": 0.9790675967931748,
+      "step": 1690,
+      "train/kl_loss_qwen": 0.02389554716646671,
+      "train/kl_loss_r1": 0.01507699298672378,
+      "train/total_kl": 0.03897254019975662
+    },
+    {
+      "epoch": 2.1107309486780714,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.4168742216687423e-05,
+      "loss": 0.3605,
+      "mean_token_accuracy": 0.9704611152410507,
+      "step": 1695,
+      "train/kl_loss_qwen": 0.03224745257757604,
+      "train/kl_loss_r1": 0.017923418898135424,
+      "train/total_kl": 0.05017087059095502
+    },
+    {
+      "epoch": 2.116951788491446,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.4122042341220424e-05,
+      "loss": 0.2689,
+      "mean_token_accuracy": 0.9798552840948105,
+      "step": 1700,
+      "train/kl_loss_qwen": 0.023866656003519893,
+      "train/kl_loss_r1": 0.01644910844042897,
+      "train/total_kl": 0.040315764397382735
+    },
+    {
+      "epoch": 2.1231726283048213,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.4075342465753425e-05,
+      "loss": 0.2528,
+      "mean_token_accuracy": 0.9789301216602325,
+      "step": 1705,
+      "train/kl_loss_qwen": 0.022611541347578167,
+      "train/kl_loss_r1": 0.014065819093957543,
+      "train/total_kl": 0.0366773602552712
+    },
+    {
+      "epoch": 2.129393468118196,
+      "grad_norm": 0.921875,
+      "learning_rate": 1.4028642590286425e-05,
+      "loss": 0.2394,
+      "mean_token_accuracy": 0.9789743661880493,
+      "step": 1710,
+      "train/kl_loss_qwen": 0.020024046860635282,
+      "train/kl_loss_r1": 0.012307545309886337,
+      "train/total_kl": 0.03233159258961678
+    },
+    {
+      "epoch": 2.135614307931571,
+      "grad_norm": 1.0390625,
+      "learning_rate": 1.3981942714819426e-05,
+      "loss": 0.2393,
+      "mean_token_accuracy": 0.9808943539857864,
+      "step": 1715,
+      "train/kl_loss_qwen": 0.022391148284077643,
+      "train/kl_loss_r1": 0.013058097870089113,
+      "train/total_kl": 0.03544924585148692
+    },
+    {
+      "epoch": 2.1418351477449455,
+      "grad_norm": 1.734375,
+      "learning_rate": 1.393524283935243e-05,
+      "loss": 0.2753,
+      "mean_token_accuracy": 0.9771335572004318,
+      "step": 1720,
+      "train/kl_loss_qwen": 0.024992619268596174,
+      "train/kl_loss_r1": 0.01562864559236914,
+      "train/total_kl": 0.04062126437202096
+    },
+    {
+      "epoch": 2.1480559875583203,
+      "grad_norm": 1.34375,
+      "learning_rate": 1.388854296388543e-05,
+      "loss": 0.242,
+      "mean_token_accuracy": 0.9794571280479432,
+      "step": 1725,
+      "train/kl_loss_qwen": 0.02109189019538462,
+      "train/kl_loss_r1": 0.013026425126008689,
+      "train/total_kl": 0.03411831529811025
+    },
+    {
+      "epoch": 2.154276827371695,
+      "grad_norm": 1.1171875,
+      "learning_rate": 1.3841843088418432e-05,
+      "loss": 0.2591,
+      "mean_token_accuracy": 0.9810844957828522,
+      "step": 1730,
+      "train/kl_loss_qwen": 0.024620172381401063,
+      "train/kl_loss_r1": 0.01482275347225368,
+      "train/total_kl": 0.039442925900220874
+    },
+    {
+      "epoch": 2.16049766718507,
+      "grad_norm": 1.1640625,
+      "learning_rate": 1.3795143212951432e-05,
+      "loss": 0.2633,
+      "mean_token_accuracy": 0.9778509587049484,
+      "step": 1735,
+      "train/kl_loss_qwen": 0.024361231364309788,
+      "train/kl_loss_r1": 0.01404720451682806,
+      "train/total_kl": 0.03840843569487333
+    },
+    {
+      "epoch": 2.166718506998445,
+      "grad_norm": 0.97265625,
+      "learning_rate": 1.3748443337484433e-05,
+      "loss": 0.2285,
+      "mean_token_accuracy": 0.9812567800283432,
+      "step": 1740,
+      "train/kl_loss_qwen": 0.02088723029009998,
+      "train/kl_loss_r1": 0.011959241493605078,
+      "train/total_kl": 0.032846471574157474
+    },
+    {
+      "epoch": 2.1729393468118197,
+      "grad_norm": 0.9765625,
+      "learning_rate": 1.3701743462017435e-05,
+      "loss": 0.2489,
+      "mean_token_accuracy": 0.9805923402309418,
+      "step": 1745,
+      "train/kl_loss_qwen": 0.022857177117839456,
+      "train/kl_loss_r1": 0.014695218997076154,
+      "train/total_kl": 0.037552396580576894
+    },
+    {
+      "epoch": 2.1791601866251944,
+      "grad_norm": 1.078125,
+      "learning_rate": 1.3655043586550436e-05,
+      "loss": 0.2526,
+      "mean_token_accuracy": 0.9761855214834213,
+      "step": 1750,
+      "train/kl_loss_qwen": 0.021363981626927854,
+      "train/kl_loss_r1": 0.012706464645452798,
+      "train/total_kl": 0.034070446714758874
+    },
+    {
+      "epoch": 2.185381026438569,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.3608343711083437e-05,
+      "loss": 0.2522,
+      "mean_token_accuracy": 0.9779788672924041,
+      "step": 1755,
+      "train/kl_loss_qwen": 0.021653601573780178,
+      "train/kl_loss_r1": 0.013945041154511274,
+      "train/total_kl": 0.03559864275157452
+    },
+    {
+      "epoch": 2.191601866251944,
+      "grad_norm": 0.93359375,
+      "learning_rate": 1.3561643835616437e-05,
+      "loss": 0.3356,
+      "mean_token_accuracy": 0.9763142317533493,
+      "step": 1760,
+      "train/kl_loss_qwen": 0.034495850279927256,
+      "train/kl_loss_r1": 0.01701771658845246,
+      "train/total_kl": 0.0515135663561523
+    },
+    {
+      "epoch": 2.1978227060653186,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.3514943960149441e-05,
+      "loss": 0.2481,
+      "mean_token_accuracy": 0.9769844710826874,
+      "step": 1765,
+      "train/kl_loss_qwen": 0.022623211657628418,
+      "train/kl_loss_r1": 0.012992961728014053,
+      "train/total_kl": 0.035616173036396505
+    },
+    {
+      "epoch": 2.2040435458786938,
+      "grad_norm": 1.3203125,
+      "learning_rate": 1.3468244084682442e-05,
+      "loss": 0.2441,
+      "mean_token_accuracy": 0.9772080987691879,
+      "step": 1770,
+      "train/kl_loss_qwen": 0.02268908736295998,
+      "train/kl_loss_r1": 0.012291538529098034,
+      "train/total_kl": 0.034980626031756404
+    },
+    {
+      "epoch": 2.2102643856920685,
+      "grad_norm": 1.0859375,
+      "learning_rate": 1.3421544209215443e-05,
+      "loss": 0.2388,
+      "mean_token_accuracy": 0.9795448333024979,
+      "step": 1775,
+      "train/kl_loss_qwen": 0.019983268249779938,
+      "train/kl_loss_r1": 0.013864303706213832,
+      "train/total_kl": 0.033847571816295385
+    },
+    {
+      "epoch": 2.2164852255054432,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.3374844333748443e-05,
+      "loss": 0.2366,
+      "mean_token_accuracy": 0.9784791618585587,
+      "step": 1780,
+      "train/kl_loss_qwen": 0.02112103491090238,
+      "train/kl_loss_r1": 0.012184371682815253,
+      "train/total_kl": 0.03330540684983134
+    },
+    {
+      "epoch": 2.222706065318818,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.3328144458281444e-05,
+      "loss": 0.2326,
+      "mean_token_accuracy": 0.978312885761261,
+      "step": 1785,
+      "train/kl_loss_qwen": 0.019464704208076,
+      "train/kl_loss_r1": 0.011881216848269104,
+      "train/total_kl": 0.031345921102911234
+    },
+    {
+      "epoch": 2.2289269051321927,
+      "grad_norm": 0.96875,
+      "learning_rate": 1.3281444582814446e-05,
+      "loss": 0.2322,
+      "mean_token_accuracy": 0.9802702635526657,
+      "step": 1790,
+      "train/kl_loss_qwen": 0.019677631743252277,
+      "train/kl_loss_r1": 0.0136221659835428,
+      "train/total_kl": 0.03329979768022895
+    },
+    {
+      "epoch": 2.2351477449455674,
+      "grad_norm": 1.203125,
+      "learning_rate": 1.3234744707347447e-05,
+      "loss": 0.2517,
+      "mean_token_accuracy": 0.9778961509466171,
+      "step": 1795,
+      "train/kl_loss_qwen": 0.023012708965688945,
+      "train/kl_loss_r1": 0.013481559325009584,
+      "train/total_kl": 0.03649426787160337
+    },
+    {
+      "epoch": 2.2413685847589426,
+      "grad_norm": 0.8125,
+      "learning_rate": 1.3188044831880448e-05,
+      "loss": 0.2359,
+      "mean_token_accuracy": 0.9810913383960724,
+      "step": 1800,
+      "train/kl_loss_qwen": 0.021271700272336602,
+      "train/kl_loss_r1": 0.013056211965158581,
+      "train/total_kl": 0.034327912330627444
+    },
+    {
+      "epoch": 2.2475894245723174,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.3141344956413449e-05,
+      "loss": 0.2346,
+      "mean_token_accuracy": 0.9790261298418045,
+      "step": 1805,
+      "train/kl_loss_qwen": 0.020045140152797104,
+      "train/kl_loss_r1": 0.01360967019572854,
+      "train/total_kl": 0.033654810208827254
+    },
+    {
+      "epoch": 2.253810264385692,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.3094645080946453e-05,
+      "loss": 0.2477,
+      "mean_token_accuracy": 0.9797742009162903,
+      "step": 1810,
+      "train/kl_loss_qwen": 0.02143171979114413,
+      "train/kl_loss_r1": 0.013752156216651202,
+      "train/total_kl": 0.035183876287192106
+    },
+    {
+      "epoch": 2.260031104199067,
+      "grad_norm": 0.9921875,
+      "learning_rate": 1.3047945205479453e-05,
+      "loss": 0.2532,
+      "mean_token_accuracy": 0.9792828232049942,
+      "step": 1815,
+      "train/kl_loss_qwen": 0.021182486787438393,
+      "train/kl_loss_r1": 0.01435540292877704,
+      "train/total_kl": 0.035537889786064626
+    },
+    {
+      "epoch": 2.2662519440124416,
+      "grad_norm": 1.5,
+      "learning_rate": 1.3001245330012454e-05,
+      "loss": 0.4353,
+      "mean_token_accuracy": 0.9699081629514694,
+      "step": 1820,
+      "train/kl_loss_qwen": 0.04558118330314755,
+      "train/kl_loss_r1": 0.023637999431230128,
+      "train/total_kl": 0.06921918261796237
+    },
+    {
+      "epoch": 2.2724727838258163,
+      "grad_norm": 1.078125,
+      "learning_rate": 1.2954545454545455e-05,
+      "loss": 0.2321,
+      "mean_token_accuracy": 0.9784551143646241,
+      "step": 1825,
+      "train/kl_loss_qwen": 0.01986483633518219,
+      "train/kl_loss_r1": 0.012468844978138804,
+      "train/total_kl": 0.032333681266754864
+    },
+    {
+      "epoch": 2.2786936236391915,
+      "grad_norm": 1.1640625,
+      "learning_rate": 1.2907845579078455e-05,
+      "loss": 0.2454,
+      "mean_token_accuracy": 0.97675521671772,
+      "step": 1830,
+      "train/kl_loss_qwen": 0.02059942428022623,
+      "train/kl_loss_r1": 0.01287925757933408,
+      "train/total_kl": 0.03347868211567402
+    },
+    {
+      "epoch": 2.284914463452566,
+      "grad_norm": 0.97265625,
+      "learning_rate": 1.2861145703611458e-05,
+      "loss": 0.2431,
+      "mean_token_accuracy": 0.9791501730680465,
+      "step": 1835,
+      "train/kl_loss_qwen": 0.021640143543481826,
+      "train/kl_loss_r1": 0.013920898968353867,
+      "train/total_kl": 0.035561042837798595
+    },
+    {
+      "epoch": 2.291135303265941,
+      "grad_norm": 3.15625,
+      "learning_rate": 1.2814445828144458e-05,
+      "loss": 0.271,
+      "mean_token_accuracy": 0.979482638835907,
+      "step": 1840,
+      "train/kl_loss_qwen": 0.02593438569456339,
+      "train/kl_loss_r1": 0.015126829454675316,
+      "train/total_kl": 0.04106121482327581
+    },
+    {
+      "epoch": 2.2973561430793157,
+      "grad_norm": 1.1484375,
+      "learning_rate": 1.2767745952677459e-05,
+      "loss": 0.2398,
+      "mean_token_accuracy": 0.9809309899806976,
+      "step": 1845,
+      "train/kl_loss_qwen": 0.022775299800559878,
+      "train/kl_loss_r1": 0.01402155626565218,
+      "train/total_kl": 0.036796855833381416
+    },
+    {
+      "epoch": 2.3035769828926904,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.272104607721046e-05,
+      "loss": 0.2934,
+      "mean_token_accuracy": 0.9758806467056275,
+      "step": 1850,
+      "train/kl_loss_qwen": 0.02778648091480136,
+      "train/kl_loss_r1": 0.01586297785397619,
+      "train/total_kl": 0.043649458419531585
+    },
+    {
+      "epoch": 2.309797822706065,
+      "grad_norm": 0.828125,
+      "learning_rate": 1.2674346201743464e-05,
+      "loss": 0.2401,
+      "mean_token_accuracy": 0.9825777590274811,
+      "step": 1855,
+      "train/kl_loss_qwen": 0.021009960398077964,
+      "train/kl_loss_r1": 0.01420710023958236,
+      "train/total_kl": 0.03521706024184823
+    },
+    {
+      "epoch": 2.31601866251944,
+      "grad_norm": 1.2734375,
+      "learning_rate": 1.2627646326276464e-05,
+      "loss": 0.2827,
+      "mean_token_accuracy": 0.9746395200490952,
+      "step": 1860,
+      "train/kl_loss_qwen": 0.024723251862451435,
+      "train/kl_loss_r1": 0.015186823182739317,
+      "train/total_kl": 0.03991007544100285
+    },
+    {
+      "epoch": 2.322239502332815,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.2580946450809465e-05,
+      "loss": 0.2482,
+      "mean_token_accuracy": 0.9802426367998123,
+      "step": 1865,
+      "train/kl_loss_qwen": 0.0225432176142931,
+      "train/kl_loss_r1": 0.013892730651423334,
+      "train/total_kl": 0.036435948219150305
+    },
+    {
+      "epoch": 2.32846034214619,
+      "grad_norm": 1.2734375,
+      "learning_rate": 1.2534246575342466e-05,
+      "loss": 0.2551,
+      "mean_token_accuracy": 0.9752663284540176,
+      "step": 1870,
+      "train/kl_loss_qwen": 0.02081627896986902,
+      "train/kl_loss_r1": 0.013366595236584544,
+      "train/total_kl": 0.034182874485850334
+    },
+    {
+      "epoch": 2.3346811819595645,
+      "grad_norm": 1.109375,
+      "learning_rate": 1.2487546699875467e-05,
+      "loss": 0.2517,
+      "mean_token_accuracy": 0.9780875205993652,
+      "step": 1875,
+      "train/kl_loss_qwen": 0.02112504974938929,
+      "train/kl_loss_r1": 0.015057702036574482,
+      "train/total_kl": 0.036182751413434744
+    },
+    {
+      "epoch": 2.3409020217729393,
+      "grad_norm": 1.234375,
+      "learning_rate": 1.2440846824408469e-05,
+      "loss": 0.2783,
+      "mean_token_accuracy": 0.9777207732200622,
+      "step": 1880,
+      "train/kl_loss_qwen": 0.026220168406143784,
+      "train/kl_loss_r1": 0.0165558461798355,
+      "train/total_kl": 0.042776014655828476
+    },
+    {
+      "epoch": 2.347122861586314,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.239414694894147e-05,
+      "loss": 0.2401,
+      "mean_token_accuracy": 0.9795249253511429,
+      "step": 1885,
+      "train/kl_loss_qwen": 0.022062430484220387,
+      "train/kl_loss_r1": 0.01306857680901885,
+      "train/total_kl": 0.03513100752606988
+    },
+    {
+      "epoch": 2.353343701399689,
+      "grad_norm": 0.90625,
+      "learning_rate": 1.234744707347447e-05,
+      "loss": 0.2305,
+      "mean_token_accuracy": 0.9795470297336578,
+      "step": 1890,
+      "train/kl_loss_qwen": 0.019669259851798414,
+      "train/kl_loss_r1": 0.013222256046719848,
+      "train/total_kl": 0.032891515735536815
+    },
+    {
+      "epoch": 2.359564541213064,
+      "grad_norm": 0.8125,
+      "learning_rate": 1.2300747198007471e-05,
+      "loss": 0.2622,
+      "mean_token_accuracy": 0.9815815418958664,
+      "step": 1895,
+      "train/kl_loss_qwen": 0.02579428404569626,
+      "train/kl_loss_r1": 0.01565156860742718,
+      "train/total_kl": 0.04144585244357586
+    },
+    {
+      "epoch": 2.3657853810264386,
+      "grad_norm": 1.2890625,
+      "learning_rate": 1.2254047322540475e-05,
+      "loss": 0.2564,
+      "mean_token_accuracy": 0.9784956514835358,
+      "step": 1900,
+      "train/kl_loss_qwen": 0.022114967973902822,
+      "train/kl_loss_r1": 0.013662851555272937,
+      "train/total_kl": 0.03577781962230801
+    },
+    {
+      "epoch": 2.3720062208398134,
+      "grad_norm": 1.390625,
+      "learning_rate": 1.2207347447073476e-05,
+      "loss": 0.3352,
+      "mean_token_accuracy": 0.9750847607851029,
+      "step": 1905,
+      "train/kl_loss_qwen": 0.03689875598065555,
+      "train/kl_loss_r1": 0.016696492512710392,
+      "train/total_kl": 0.05359524823725224
+    },
+    {
+      "epoch": 2.378227060653188,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.2160647571606476e-05,
+      "loss": 0.2371,
+      "mean_token_accuracy": 0.9770203292369842,
+      "step": 1910,
+      "train/kl_loss_qwen": 0.019168762071058155,
+      "train/kl_loss_r1": 0.012693860824219883,
+      "train/total_kl": 0.03186262277886272
+    },
+    {
+      "epoch": 2.384447900466563,
+      "grad_norm": 0.94921875,
+      "learning_rate": 1.2113947696139477e-05,
+      "loss": 0.2373,
+      "mean_token_accuracy": 0.9789462387561798,
+      "step": 1915,
+      "train/kl_loss_qwen": 0.021564632654190063,
+      "train/kl_loss_r1": 0.012719874107278883,
+      "train/total_kl": 0.03428450655192137
+    },
+    {
+      "epoch": 2.3906687402799376,
+      "grad_norm": 1.4140625,
+      "learning_rate": 1.2067247820672478e-05,
+      "loss": 0.2346,
+      "mean_token_accuracy": 0.9807638555765152,
+      "step": 1920,
+      "train/kl_loss_qwen": 0.01933746659196913,
+      "train/kl_loss_r1": 0.013734246650710703,
+      "train/total_kl": 0.03307171277701855
+    },
+    {
+      "epoch": 2.3968895800933128,
+      "grad_norm": 1.3515625,
+      "learning_rate": 1.202054794520548e-05,
+      "loss": 0.2588,
+      "mean_token_accuracy": 0.9800190776586533,
+      "step": 1925,
+      "train/kl_loss_qwen": 0.02418329380452633,
+      "train/kl_loss_r1": 0.015474402927793562,
+      "train/total_kl": 0.03965769670903683
+    },
+    {
+      "epoch": 2.4031104199066875,
+      "grad_norm": 1.1171875,
+      "learning_rate": 1.197384806973848e-05,
+      "loss": 0.2492,
+      "mean_token_accuracy": 0.9790872097015381,
+      "step": 1930,
+      "train/kl_loss_qwen": 0.022408964531496166,
+      "train/kl_loss_r1": 0.013171161222271622,
+      "train/total_kl": 0.035580125823616984
+    },
+    {
+      "epoch": 2.4093312597200622,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.1927148194271482e-05,
+      "loss": 0.2521,
+      "mean_token_accuracy": 0.9767476886510849,
+      "step": 1935,
+      "train/kl_loss_qwen": 0.02167391194961965,
+      "train/kl_loss_r1": 0.014151084562763571,
+      "train/total_kl": 0.035824996419250965
+    },
+    {
+      "epoch": 2.415552099533437,
+      "grad_norm": 1.296875,
+      "learning_rate": 1.1880448318804482e-05,
+      "loss": 0.2498,
+      "mean_token_accuracy": 0.978100472688675,
+      "step": 1940,
+      "train/kl_loss_qwen": 0.02200420745648444,
+      "train/kl_loss_r1": 0.014483575685881078,
+      "train/total_kl": 0.03648778265342116
+    },
+    {
+      "epoch": 2.4217729393468117,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.1833748443337485e-05,
+      "loss": 0.2496,
+      "mean_token_accuracy": 0.9778286337852478,
+      "step": 1945,
+      "train/kl_loss_qwen": 0.022651279880665242,
+      "train/kl_loss_r1": 0.013287181942723691,
+      "train/total_kl": 0.03593846196308732
+    },
+    {
+      "epoch": 2.4279937791601864,
+      "grad_norm": 1.7109375,
+      "learning_rate": 1.1787048567870487e-05,
+      "loss": 0.2598,
+      "mean_token_accuracy": 0.977265328168869,
+      "step": 1950,
+      "train/kl_loss_qwen": 0.02262912839651108,
+      "train/kl_loss_r1": 0.014686035527847707,
+      "train/total_kl": 0.03731516385450959
+    },
+    {
+      "epoch": 2.4342146189735616,
+      "grad_norm": 2.234375,
+      "learning_rate": 1.1740348692403488e-05,
+      "loss": 0.3154,
+      "mean_token_accuracy": 0.974388661980629,
+      "step": 1955,
+      "train/kl_loss_qwen": 0.031231827940791844,
+      "train/kl_loss_r1": 0.01720571951009333,
+      "train/total_kl": 0.04843754768371582
+    },
+    {
+      "epoch": 2.4404354587869364,
+      "grad_norm": 0.8203125,
+      "learning_rate": 1.1693648816936488e-05,
+      "loss": 0.2214,
+      "mean_token_accuracy": 0.9811009198427201,
+      "step": 1960,
+      "train/kl_loss_qwen": 0.01896182936616242,
+      "train/kl_loss_r1": 0.011808227049186826,
+      "train/total_kl": 0.030770056508481504
+    },
+    {
+      "epoch": 2.446656298600311,
+      "grad_norm": 1.125,
+      "learning_rate": 1.1646948941469489e-05,
+      "loss": 0.2434,
+      "mean_token_accuracy": 0.9757021725177765,
+      "step": 1965,
+      "train/kl_loss_qwen": 0.01998060490004718,
+      "train/kl_loss_r1": 0.012626229273155331,
+      "train/total_kl": 0.03260683408007026
+    },
+    {
+      "epoch": 2.452877138413686,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.1600249066002491e-05,
+      "loss": 0.2558,
+      "mean_token_accuracy": 0.9779347211122513,
+      "step": 1970,
+      "train/kl_loss_qwen": 0.021612715255469084,
+      "train/kl_loss_r1": 0.013517844607122242,
+      "train/total_kl": 0.03513055983930826
+    },
+    {
+      "epoch": 2.4590979782270606,
+      "grad_norm": 1.21875,
+      "learning_rate": 1.1553549190535492e-05,
+      "loss": 0.2521,
+      "mean_token_accuracy": 0.9801371067762374,
+      "step": 1975,
+      "train/kl_loss_qwen": 0.02266769213601947,
+      "train/kl_loss_r1": 0.014395871269516646,
+      "train/total_kl": 0.037063563149422406
+    },
+    {
+      "epoch": 2.4653188180404353,
+      "grad_norm": 1.421875,
+      "learning_rate": 1.1506849315068493e-05,
+      "loss": 0.2738,
+      "mean_token_accuracy": 0.9786773502826691,
+      "step": 1980,
+      "train/kl_loss_qwen": 0.02531546037644148,
+      "train/kl_loss_r1": 0.015837101102806627,
+      "train/total_kl": 0.04115256136283278
+    },
+    {
+      "epoch": 2.4715396578538105,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.1460149439601493e-05,
+      "loss": 0.2666,
+      "mean_token_accuracy": 0.9820117831230164,
+      "step": 1985,
+      "train/kl_loss_qwen": 0.025500294240191578,
+      "train/kl_loss_r1": 0.017055929615162314,
+      "train/total_kl": 0.042556223925203085
+    },
+    {
+      "epoch": 2.477760497667185,
+      "grad_norm": 1.0703125,
+      "learning_rate": 1.1413449564134496e-05,
+      "loss": 0.2625,
+      "mean_token_accuracy": 0.9783152371644974,
+      "step": 1990,
+      "train/kl_loss_qwen": 0.022538991970941426,
+      "train/kl_loss_r1": 0.01549134326633066,
+      "train/total_kl": 0.03803033493459225
+    },
+    {
+      "epoch": 2.48398133748056,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.1366749688667498e-05,
+      "loss": 0.2557,
+      "mean_token_accuracy": 0.9759873390197754,
+      "step": 1995,
+      "train/kl_loss_qwen": 0.02168611092492938,
+      "train/kl_loss_r1": 0.01415776691865176,
+      "train/total_kl": 0.03584387795999646
+    },
+    {
+      "epoch": 2.4902021772939347,
+      "grad_norm": 1.140625,
+      "learning_rate": 1.1320049813200499e-05,
+      "loss": 0.2739,
+      "mean_token_accuracy": 0.9793085306882858,
+      "step": 2000,
+      "train/kl_loss_qwen": 0.02674374058842659,
+      "train/kl_loss_r1": 0.01565704217646271,
+      "train/total_kl": 0.04240078274160623
+    },
+    {
+      "epoch": 2.4964230171073094,
+      "grad_norm": 0.99609375,
+      "learning_rate": 1.12733499377335e-05,
+      "loss": 0.3333,
+      "mean_token_accuracy": 0.9758429080247879,
+      "step": 2005,
+      "train/kl_loss_qwen": 0.03534267195500433,
+      "train/kl_loss_r1": 0.015796015737578273,
+      "train/total_kl": 0.05113868797197938
+    },
+    {
+      "epoch": 2.502643856920684,
+      "grad_norm": 1.15625,
+      "learning_rate": 1.12266500622665e-05,
+      "loss": 0.2685,
+      "mean_token_accuracy": 0.9771882861852645,
+      "step": 2010,
+      "train/kl_loss_qwen": 0.02408771966584027,
+      "train/kl_loss_r1": 0.014890200644731521,
+      "train/total_kl": 0.038977920450270175
+    },
+    {
+      "epoch": 2.508864696734059,
+      "grad_norm": 1.3359375,
+      "learning_rate": 1.1179950186799503e-05,
+      "loss": 0.2471,
+      "mean_token_accuracy": 0.9786090284585953,
+      "step": 2015,
+      "train/kl_loss_qwen": 0.021793363522738217,
+      "train/kl_loss_r1": 0.012957709864713252,
+      "train/total_kl": 0.03475107317790389
+    },
+    {
+      "epoch": 2.515085536547434,
+      "grad_norm": 0.9765625,
+      "learning_rate": 1.1133250311332503e-05,
+      "loss": 0.2501,
+      "mean_token_accuracy": 0.981274637579918,
+      "step": 2020,
+      "train/kl_loss_qwen": 0.023206943599507212,
+      "train/kl_loss_r1": 0.013862681039609016,
+      "train/total_kl": 0.03706962484866381
+    },
+    {
+      "epoch": 2.521306376360809,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.1086550435865504e-05,
+      "loss": 0.2465,
+      "mean_token_accuracy": 0.9786695569753647,
+      "step": 2025,
+      "train/kl_loss_qwen": 0.020848573138937353,
+      "train/kl_loss_r1": 0.0137579798232764,
+      "train/total_kl": 0.03460655324161053
+    },
+    {
+      "epoch": 2.5275272161741835,
+      "grad_norm": 0.98828125,
+      "learning_rate": 1.1039850560398506e-05,
+      "loss": 0.2439,
+      "mean_token_accuracy": 0.9790724396705628,
+      "step": 2030,
+      "train/kl_loss_qwen": 0.021428097784519196,
+      "train/kl_loss_r1": 0.012936983909457921,
+      "train/total_kl": 0.03436508160084486
+    },
+    {
+      "epoch": 2.5337480559875583,
+      "grad_norm": 0.921875,
+      "learning_rate": 1.0993150684931507e-05,
+      "loss": 0.2571,
+      "mean_token_accuracy": 0.9793956607580185,
+      "step": 2035,
+      "train/kl_loss_qwen": 0.0228791618719697,
+      "train/kl_loss_r1": 0.015054402546957136,
+      "train/total_kl": 0.037933564838021995
+    },
+    {
+      "epoch": 2.539968895800933,
+      "grad_norm": 0.96484375,
+      "learning_rate": 1.094645080946451e-05,
+      "loss": 0.3809,
+      "mean_token_accuracy": 0.975305300951004,
+      "step": 2040,
+      "train/kl_loss_qwen": 0.0327933412976563,
+      "train/kl_loss_r1": 0.02881522406823933,
+      "train/total_kl": 0.061608564015477894
+    },
+    {
+      "epoch": 2.546189735614308,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.089975093399751e-05,
+      "loss": 0.2729,
+      "mean_token_accuracy": 0.9835954070091247,
+      "step": 2045,
+      "train/kl_loss_qwen": 0.02814694056287408,
+      "train/kl_loss_r1": 0.01594695230014622,
+      "train/total_kl": 0.04409389290958643
+    },
+    {
+      "epoch": 2.5524105754276825,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.085305105853051e-05,
+      "loss": 0.264,
+      "mean_token_accuracy": 0.9798797309398651,
+      "step": 2050,
+      "train/kl_loss_qwen": 0.024475245364010335,
+      "train/kl_loss_r1": 0.014863114035688341,
+      "train/total_kl": 0.03933835970237851
+    },
+    {
+      "epoch": 2.5586314152410576,
+      "grad_norm": 1.125,
+      "learning_rate": 1.0806351183063511e-05,
+      "loss": 0.2793,
+      "mean_token_accuracy": 0.9761617630720139,
+      "step": 2055,
+      "train/kl_loss_qwen": 0.023740977654233573,
+      "train/kl_loss_r1": 0.016528584342449904,
+      "train/total_kl": 0.04026956185698509
+    },
+    {
+      "epoch": 2.5648522550544324,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.0759651307596514e-05,
+      "loss": 0.3036,
+      "mean_token_accuracy": 0.9757337421178818,
+      "step": 2060,
+      "train/kl_loss_qwen": 0.031403438979759814,
+      "train/kl_loss_r1": 0.014756415458396076,
+      "train/total_kl": 0.04615985546261072
+    },
+    {
+      "epoch": 2.571073094867807,
+      "grad_norm": 0.953125,
+      "learning_rate": 1.0712951432129514e-05,
+      "loss": 0.2439,
+      "mean_token_accuracy": 0.9786640286445618,
+      "step": 2065,
+      "train/kl_loss_qwen": 0.022552527487277985,
+      "train/kl_loss_r1": 0.012621782696805894,
+      "train/total_kl": 0.035174309927970174
+    },
+    {
+      "epoch": 2.577293934681182,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.0666251556662515e-05,
+      "loss": 0.2411,
+      "mean_token_accuracy": 0.9811136335134506,
+      "step": 2070,
+      "train/kl_loss_qwen": 0.02110184049233794,
+      "train/kl_loss_r1": 0.014247958501800894,
+      "train/total_kl": 0.03534979866817593
+    },
+    {
+      "epoch": 2.5835147744945566,
+      "grad_norm": 1.53125,
+      "learning_rate": 1.0619551681195518e-05,
+      "loss": 0.2487,
+      "mean_token_accuracy": 0.978958186507225,
+      "step": 2075,
+      "train/kl_loss_qwen": 0.0206217001657933,
+      "train/kl_loss_r1": 0.013504683272913098,
+      "train/total_kl": 0.03412638353183865
+    },
+    {
+      "epoch": 2.5897356143079318,
+      "grad_norm": 0.88671875,
+      "learning_rate": 1.0572851805728518e-05,
+      "loss": 0.2444,
+      "mean_token_accuracy": 0.9794360756874084,
+      "step": 2080,
+      "train/kl_loss_qwen": 0.021966875065118074,
+      "train/kl_loss_r1": 0.013504690513946116,
+      "train/total_kl": 0.035471565742045644
+    },
+    {
+      "epoch": 2.5959564541213065,
+      "grad_norm": 1.453125,
+      "learning_rate": 1.052615193026152e-05,
+      "loss": 0.2507,
+      "mean_token_accuracy": 0.980023518204689,
+      "step": 2085,
+      "train/kl_loss_qwen": 0.022249753633514047,
+      "train/kl_loss_r1": 0.013989573833532632,
+      "train/total_kl": 0.03623932776972651
+    },
+    {
+      "epoch": 2.6021772939346812,
+      "grad_norm": 1.1171875,
+      "learning_rate": 1.0479452054794521e-05,
+      "loss": 0.2527,
+      "mean_token_accuracy": 0.9773518353700638,
+      "step": 2090,
+      "train/kl_loss_qwen": 0.021709746960550547,
+      "train/kl_loss_r1": 0.013672297820448875,
+      "train/total_kl": 0.03538204478099942
+    },
+    {
+      "epoch": 2.608398133748056,
+      "grad_norm": 1.0390625,
+      "learning_rate": 1.0432752179327522e-05,
+      "loss": 0.255,
+      "mean_token_accuracy": 0.9789919286966324,
+      "step": 2095,
+      "train/kl_loss_qwen": 0.02221295302733779,
+      "train/kl_loss_r1": 0.01521359293255955,
+      "train/total_kl": 0.03742654556408524
+    },
+    {
+      "epoch": 2.6146189735614307,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.0386052303860523e-05,
+      "loss": 0.2456,
+      "mean_token_accuracy": 0.9789235025644303,
+      "step": 2100,
+      "train/kl_loss_qwen": 0.02003535218536854,
+      "train/kl_loss_r1": 0.013994471030309796,
+      "train/total_kl": 0.03402982326224446
+    },
+    {
+      "epoch": 2.620839813374806,
+      "grad_norm": 1.953125,
+      "learning_rate": 1.0339352428393525e-05,
+      "loss": 0.2683,
+      "mean_token_accuracy": 0.9778778046369553,
+      "step": 2105,
+      "train/kl_loss_qwen": 0.024578660633414982,
+      "train/kl_loss_r1": 0.015411347197368742,
+      "train/total_kl": 0.03999000806361437
+    },
+    {
+      "epoch": 2.62706065318818,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.0292652552926526e-05,
+      "loss": 0.2502,
+      "mean_token_accuracy": 0.9789797216653824,
+      "step": 2110,
+      "train/kl_loss_qwen": 0.02267702422104776,
+      "train/kl_loss_r1": 0.013770601083524524,
+      "train/total_kl": 0.036447625048458575
+    },
+    {
+      "epoch": 2.6332814930015553,
+      "grad_norm": 1.046875,
+      "learning_rate": 1.0245952677459526e-05,
+      "loss": 0.2559,
+      "mean_token_accuracy": 0.9788319110870362,
+      "step": 2115,
+      "train/kl_loss_qwen": 0.02250348115339875,
+      "train/kl_loss_r1": 0.015042783576063812,
+      "train/total_kl": 0.037546264659613374
+    },
+    {
+      "epoch": 2.63950233281493,
+      "grad_norm": 1.171875,
+      "learning_rate": 1.0199252801992529e-05,
+      "loss": 0.2475,
+      "mean_token_accuracy": 0.9789767384529113,
+      "step": 2120,
+      "train/kl_loss_qwen": 0.0217635712120682,
+      "train/kl_loss_r1": 0.013855892582796514,
+      "train/total_kl": 0.03561946395784617
+    },
+    {
+      "epoch": 2.645723172628305,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.015255292652553e-05,
+      "loss": 0.3489,
+      "mean_token_accuracy": 0.9748771101236343,
+      "step": 2125,
+      "train/kl_loss_qwen": 0.03360332241281867,
+      "train/kl_loss_r1": 0.018315932666882872,
+      "train/total_kl": 0.051919255033135416
+    },
+    {
+      "epoch": 2.6519440124416795,
+      "grad_norm": 1.1875,
+      "learning_rate": 1.0105853051058532e-05,
+      "loss": 0.2347,
+      "mean_token_accuracy": 0.9795038014650345,
+      "step": 2130,
+      "train/kl_loss_qwen": 0.02050428814254701,
+      "train/kl_loss_r1": 0.011696344916708767,
+      "train/total_kl": 0.0322006331756711
+    },
+    {
+      "epoch": 2.6581648522550543,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.0059153175591532e-05,
+      "loss": 0.2775,
+      "mean_token_accuracy": 0.979162546992302,
+      "step": 2135,
+      "train/kl_loss_qwen": 0.026292033307254315,
+      "train/kl_loss_r1": 0.016034953179769218,
+      "train/total_kl": 0.04232698623090982
+    },
+    {
+      "epoch": 2.6643856920684295,
+      "grad_norm": 1.203125,
+      "learning_rate": 1.0012453300124533e-05,
+      "loss": 0.2311,
+      "mean_token_accuracy": 0.9770093053579331,
+      "step": 2140,
+      "train/kl_loss_qwen": 0.019046885380521416,
+      "train/kl_loss_r1": 0.012393153086304665,
+      "train/total_kl": 0.031440038606524466
+    },
+    {
+      "epoch": 2.670606531881804,
+      "grad_norm": 1.0546875,
+      "learning_rate": 9.965753424657534e-06,
+      "loss": 0.2415,
+      "mean_token_accuracy": 0.9810254126787186,
+      "step": 2145,
+      "train/kl_loss_qwen": 0.022190214600414038,
+      "train/kl_loss_r1": 0.013294363114982843,
+      "train/total_kl": 0.035484577808529136
+    },
+    {
+      "epoch": 2.676827371695179,
+      "grad_norm": 1.109375,
+      "learning_rate": 9.919053549190536e-06,
+      "loss": 0.235,
+      "mean_token_accuracy": 0.9792218834161759,
+      "step": 2150,
+      "train/kl_loss_qwen": 0.021480456925928593,
+      "train/kl_loss_r1": 0.012514953222125768,
+      "train/total_kl": 0.033995410334318875
+    },
+    {
+      "epoch": 2.6830482115085537,
+      "grad_norm": 0.98046875,
+      "learning_rate": 9.872353673723537e-06,
+      "loss": 0.2255,
+      "mean_token_accuracy": 0.9813844114542007,
+      "step": 2155,
+      "train/kl_loss_qwen": 0.019305172516033052,
+      "train/kl_loss_r1": 0.012685006693936884,
+      "train/total_kl": 0.03199017941951752
+    },
+    {
+      "epoch": 2.6892690513219284,
+      "grad_norm": 0.94921875,
+      "learning_rate": 9.825653798256538e-06,
+      "loss": 0.2614,
+      "mean_token_accuracy": 0.9802846044301987,
+      "step": 2160,
+      "train/kl_loss_qwen": 0.023292775312438608,
+      "train/kl_loss_r1": 0.014105364657007157,
+      "train/total_kl": 0.03739814003929496
+    },
+    {
+      "epoch": 2.695489891135303,
+      "grad_norm": 1.0546875,
+      "learning_rate": 9.77895392278954e-06,
+      "loss": 0.2779,
+      "mean_token_accuracy": 0.9766510784626007,
+      "step": 2165,
+      "train/kl_loss_qwen": 0.02564525925554335,
+      "train/kl_loss_r1": 0.015102893207222223,
+      "train/total_kl": 0.040748152509331705
+    },
+    {
+      "epoch": 2.701710730948678,
+      "grad_norm": 1.046875,
+      "learning_rate": 9.73225404732254e-06,
+      "loss": 0.2647,
+      "mean_token_accuracy": 0.9788645833730698,
+      "step": 2170,
+      "train/kl_loss_qwen": 0.023770625423640012,
+      "train/kl_loss_r1": 0.01477400113362819,
+      "train/total_kl": 0.03854462616145611
+    },
+    {
+      "epoch": 2.707931570762053,
+      "grad_norm": 0.98828125,
+      "learning_rate": 9.685554171855543e-06,
+      "loss": 0.2438,
+      "mean_token_accuracy": 0.9803738176822663,
+      "step": 2175,
+      "train/kl_loss_qwen": 0.020186571637168527,
+      "train/kl_loss_r1": 0.013914901157841086,
+      "train/total_kl": 0.0341014726087451
+    },
+    {
+      "epoch": 2.7141524105754278,
+      "grad_norm": 1.4609375,
+      "learning_rate": 9.638854296388544e-06,
+      "loss": 0.2489,
+      "mean_token_accuracy": 0.9783189207315445,
+      "step": 2180,
+      "train/kl_loss_qwen": 0.020895938901230694,
+      "train/kl_loss_r1": 0.013940211269073188,
+      "train/total_kl": 0.034836150519549844
+    },
+    {
+      "epoch": 2.7203732503888025,
+      "grad_norm": 1.0390625,
+      "learning_rate": 9.592154420921544e-06,
+      "loss": 0.2291,
+      "mean_token_accuracy": 0.9809900194406509,
+      "step": 2185,
+      "train/kl_loss_qwen": 0.019283190369606018,
+      "train/kl_loss_r1": 0.011813020100817084,
+      "train/total_kl": 0.031096210610121487
+    },
+    {
+      "epoch": 2.7265940902021772,
+      "grad_norm": 1.0859375,
+      "learning_rate": 9.545454545454545e-06,
+      "loss": 0.2716,
+      "mean_token_accuracy": 0.979528221487999,
+      "step": 2190,
+      "train/kl_loss_qwen": 0.024975268309935926,
+      "train/kl_loss_r1": 0.014736651070415973,
+      "train/total_kl": 0.03971191914752126
+    },
+    {
+      "epoch": 2.732814930015552,
+      "grad_norm": 0.9375,
+      "learning_rate": 9.498754669987546e-06,
+      "loss": 0.2547,
+      "mean_token_accuracy": 0.9805773764848709,
+      "step": 2195,
+      "train/kl_loss_qwen": 0.023621318116784095,
+      "train/kl_loss_r1": 0.014904734306037426,
+      "train/total_kl": 0.038526052702218296
+    },
+    {
+      "epoch": 2.739035769828927,
+      "grad_norm": 1.2421875,
+      "learning_rate": 9.452054794520548e-06,
+      "loss": 0.2519,
+      "mean_token_accuracy": 0.9800098180770874,
+      "step": 2200,
+      "train/kl_loss_qwen": 0.022577690612524747,
+      "train/kl_loss_r1": 0.014399249549023807,
+      "train/total_kl": 0.036976939905434845
+    },
+    {
+      "epoch": 2.7452566096423014,
+      "grad_norm": 0.98828125,
+      "learning_rate": 9.405354919053549e-06,
+      "loss": 0.2523,
+      "mean_token_accuracy": 0.9802424728870391,
+      "step": 2205,
+      "train/kl_loss_qwen": 0.023771630600094795,
+      "train/kl_loss_r1": 0.014084443286992609,
+      "train/total_kl": 0.03785607386380434
+    },
+    {
+      "epoch": 2.7514774494556766,
+      "grad_norm": 1.0625,
+      "learning_rate": 9.358655043586551e-06,
+      "loss": 0.2483,
+      "mean_token_accuracy": 0.9757383853197098,
+      "step": 2210,
+      "train/kl_loss_qwen": 0.020856014778837563,
+      "train/kl_loss_r1": 0.012496178969740868,
+      "train/total_kl": 0.03335219379514456
+    },
+    {
+      "epoch": 2.7576982892690514,
+      "grad_norm": 1.2265625,
+      "learning_rate": 9.311955168119552e-06,
+      "loss": 0.2609,
+      "mean_token_accuracy": 0.9750352591276169,
+      "step": 2215,
+      "train/kl_loss_qwen": 0.021947943326085805,
+      "train/kl_loss_r1": 0.013721926184371114,
+      "train/total_kl": 0.035669869370758535
+    },
+    {
+      "epoch": 2.763919129082426,
+      "grad_norm": 1.125,
+      "learning_rate": 9.265255292652554e-06,
+      "loss": 0.2498,
+      "mean_token_accuracy": 0.9788256555795669,
+      "step": 2220,
+      "train/kl_loss_qwen": 0.022918132552877068,
+      "train/kl_loss_r1": 0.013518631807528436,
+      "train/total_kl": 0.036436764243990186
+    },
+    {
+      "epoch": 2.770139968895801,
+      "grad_norm": 1.1796875,
+      "learning_rate": 9.218555417185555e-06,
+      "loss": 0.2567,
+      "mean_token_accuracy": 0.98291976749897,
+      "step": 2225,
+      "train/kl_loss_qwen": 0.02417180915363133,
+      "train/kl_loss_r1": 0.016105480049736796,
+      "train/total_kl": 0.04027728922665119
+    },
+    {
+      "epoch": 2.7763608087091756,
+      "grad_norm": 0.90234375,
+      "learning_rate": 9.171855541718556e-06,
+      "loss": 0.2501,
+      "mean_token_accuracy": 0.9805917859077453,
+      "step": 2230,
+      "train/kl_loss_qwen": 0.022929491940885782,
+      "train/kl_loss_r1": 0.014057897846214473,
+      "train/total_kl": 0.03698738953098655
+    },
+    {
+      "epoch": 2.7825816485225507,
+      "grad_norm": 1.1015625,
+      "learning_rate": 9.125155666251556e-06,
+      "loss": 0.3112,
+      "mean_token_accuracy": 0.9757759094238281,
+      "step": 2235,
+      "train/kl_loss_qwen": 0.030990847293287514,
+      "train/kl_loss_r1": 0.017853675317019223,
+      "train/total_kl": 0.048844522703438996
+    },
+    {
+      "epoch": 2.7888024883359255,
+      "grad_norm": 1.2109375,
+      "learning_rate": 9.078455790784557e-06,
+      "loss": 0.2578,
+      "mean_token_accuracy": 0.9765969723463058,
+      "step": 2240,
+      "train/kl_loss_qwen": 0.022567249741405247,
+      "train/kl_loss_r1": 0.014588132430799306,
+      "train/total_kl": 0.03715538214892149
+    },
+    {
+      "epoch": 2.7950233281493,
+      "grad_norm": 0.93359375,
+      "learning_rate": 9.03175591531756e-06,
+      "loss": 0.2468,
+      "mean_token_accuracy": 0.9806161612272263,
+      "step": 2245,
+      "train/kl_loss_qwen": 0.022640920570120217,
+      "train/kl_loss_r1": 0.013426762819290162,
+      "train/total_kl": 0.03606768334284425
+    },
+    {
+      "epoch": 2.801244167962675,
+      "grad_norm": 1.28125,
+      "learning_rate": 8.98505603985056e-06,
+      "loss": 0.2405,
+      "mean_token_accuracy": 0.9788148522377014,
+      "step": 2250,
+      "train/kl_loss_qwen": 0.020399608230218292,
+      "train/kl_loss_r1": 0.013216133532114326,
+      "train/total_kl": 0.03361574159935117
+    },
+    {
+      "epoch": 2.8074650077760497,
+      "grad_norm": 0.93359375,
+      "learning_rate": 8.938356164383562e-06,
+      "loss": 0.2289,
+      "mean_token_accuracy": 0.9812757492065429,
+      "step": 2255,
+      "train/kl_loss_qwen": 0.019857667246833445,
+      "train/kl_loss_r1": 0.013221802725456654,
+      "train/total_kl": 0.03307946994900703
+    },
+    {
+      "epoch": 2.8136858475894244,
+      "grad_norm": 1.296875,
+      "learning_rate": 8.891656288916563e-06,
+      "loss": 0.2606,
+      "mean_token_accuracy": 0.9791488260030746,
+      "step": 2260,
+      "train/kl_loss_qwen": 0.02399991424754262,
+      "train/kl_loss_r1": 0.015469257766380907,
+      "train/total_kl": 0.03946917224675417
+    },
+    {
+      "epoch": 2.819906687402799,
+      "grad_norm": 1.6875,
+      "learning_rate": 8.844956413449565e-06,
+      "loss": 0.2846,
+      "mean_token_accuracy": 0.979676753282547,
+      "step": 2265,
+      "train/kl_loss_qwen": 0.02852729302830994,
+      "train/kl_loss_r1": 0.01608511172235012,
+      "train/total_kl": 0.04461240470409393
+    },
+    {
+      "epoch": 2.8261275272161743,
+      "grad_norm": 1.0,
+      "learning_rate": 8.798256537982566e-06,
+      "loss": 0.2421,
+      "mean_token_accuracy": 0.979563570022583,
+      "step": 2270,
+      "train/kl_loss_qwen": 0.021803312422707676,
+      "train/kl_loss_r1": 0.012825471581891179,
+      "train/total_kl": 0.03462878372520208
+    },
+    {
+      "epoch": 2.832348367029549,
+      "grad_norm": 0.953125,
+      "learning_rate": 8.751556662515567e-06,
+      "loss": 0.255,
+      "mean_token_accuracy": 0.9786518633365631,
+      "step": 2275,
+      "train/kl_loss_qwen": 0.02268552640452981,
+      "train/kl_loss_r1": 0.014874024945311249,
+      "train/total_kl": 0.037559551559388636
+    },
+    {
+      "epoch": 2.838569206842924,
+      "grad_norm": 0.87109375,
+      "learning_rate": 8.704856787048568e-06,
+      "loss": 0.2709,
+      "mean_token_accuracy": 0.9777145385742188,
+      "step": 2280,
+      "train/kl_loss_qwen": 0.02695997697301209,
+      "train/kl_loss_r1": 0.015009569097310304,
+      "train/total_kl": 0.04196954630315304
+    },
+    {
+      "epoch": 2.8447900466562985,
+      "grad_norm": 1.015625,
+      "learning_rate": 8.658156911581568e-06,
+      "loss": 0.4012,
+      "mean_token_accuracy": 0.9728323191404342,
+      "step": 2285,
+      "train/kl_loss_qwen": 0.04328306377865374,
+      "train/kl_loss_r1": 0.019736759038642047,
+      "train/total_kl": 0.06301982244476675
+    },
+    {
+      "epoch": 2.8510108864696733,
+      "grad_norm": 1.1171875,
+      "learning_rate": 8.61145703611457e-06,
+      "loss": 0.2336,
+      "mean_token_accuracy": 0.9794570118188858,
+      "step": 2290,
+      "train/kl_loss_qwen": 0.01980093107558787,
+      "train/kl_loss_r1": 0.01263779290020466,
+      "train/total_kl": 0.0324387239292264
+    },
+    {
+      "epoch": 2.8572317262830484,
+      "grad_norm": 0.96484375,
+      "learning_rate": 8.564757160647571e-06,
+      "loss": 0.2557,
+      "mean_token_accuracy": 0.9764771848917008,
+      "step": 2295,
+      "train/kl_loss_qwen": 0.023008445603773,
+      "train/kl_loss_r1": 0.013009098009206355,
+      "train/total_kl": 0.036017543729394676
+    },
+    {
+      "epoch": 2.863452566096423,
+      "grad_norm": 1.21875,
+      "learning_rate": 8.518057285180574e-06,
+      "loss": 0.2353,
+      "mean_token_accuracy": 0.9793502748012543,
+      "step": 2300,
+      "train/kl_loss_qwen": 0.02138944426551461,
+      "train/kl_loss_r1": 0.012340294336900115,
+      "train/total_kl": 0.033729738742113116
+    },
+    {
+      "epoch": 2.869673405909798,
+      "grad_norm": 1.1328125,
+      "learning_rate": 8.471357409713574e-06,
+      "loss": 0.2507,
+      "mean_token_accuracy": 0.9794678062200546,
+      "step": 2305,
+      "train/kl_loss_qwen": 0.0230550997890532,
+      "train/kl_loss_r1": 0.01367465585935861,
+      "train/total_kl": 0.03672975599765778
+    },
+    {
+      "epoch": 2.8758942457231726,
+      "grad_norm": 1.3671875,
+      "learning_rate": 8.424657534246577e-06,
+      "loss": 0.2519,
+      "mean_token_accuracy": 0.9780717104673385,
+      "step": 2310,
+      "train/kl_loss_qwen": 0.022003966965712607,
+      "train/kl_loss_r1": 0.013341469364240766,
+      "train/total_kl": 0.03534543639980257
+    },
+    {
+      "epoch": 2.8821150855365474,
+      "grad_norm": 1.1328125,
+      "learning_rate": 8.377957658779577e-06,
+      "loss": 0.2565,
+      "mean_token_accuracy": 0.9794615656137466,
+      "step": 2315,
+      "train/kl_loss_qwen": 0.022503159195184707,
+      "train/kl_loss_r1": 0.014681264385581016,
+      "train/total_kl": 0.03718442320823669
+    },
+    {
+      "epoch": 2.888335925349922,
+      "grad_norm": 0.875,
+      "learning_rate": 8.331257783312578e-06,
+      "loss": 0.3476,
+      "mean_token_accuracy": 0.9739361822605133,
+      "step": 2320,
+      "train/kl_loss_qwen": 0.033717150893062355,
+      "train/kl_loss_r1": 0.017250060685910285,
+      "train/total_kl": 0.05096721239387989
+    },
+    {
+      "epoch": 2.894556765163297,
+      "grad_norm": 0.8515625,
+      "learning_rate": 8.284557907845579e-06,
+      "loss": 0.2501,
+      "mean_token_accuracy": 0.9814224302768707,
+      "step": 2325,
+      "train/kl_loss_qwen": 0.023675559274852275,
+      "train/kl_loss_r1": 0.014361454313620925,
+      "train/total_kl": 0.03803701344877482
+    },
+    {
+      "epoch": 2.900777604976672,
+      "grad_norm": 1.171875,
+      "learning_rate": 8.23785803237858e-06,
+      "loss": 0.3194,
+      "mean_token_accuracy": 0.9765466809272766,
+      "step": 2330,
+      "train/kl_loss_qwen": 0.03182602096349001,
+      "train/kl_loss_r1": 0.018149445950984954,
+      "train/total_kl": 0.04997546775266528
+    },
+    {
+      "epoch": 2.9069984447900468,
+      "grad_norm": 1.0703125,
+      "learning_rate": 8.191158156911582e-06,
+      "loss": 0.243,
+      "mean_token_accuracy": 0.9765629231929779,
+      "step": 2335,
+      "train/kl_loss_qwen": 0.019837050791829826,
+      "train/kl_loss_r1": 0.013477956131100654,
+      "train/total_kl": 0.03331500729545951
+    },
+    {
+      "epoch": 2.9132192846034215,
+      "grad_norm": 0.9296875,
+      "learning_rate": 8.144458281444582e-06,
+      "loss": 0.249,
+      "mean_token_accuracy": 0.9794359922409057,
+      "step": 2340,
+      "train/kl_loss_qwen": 0.021764508541673423,
+      "train/kl_loss_r1": 0.014173974050208926,
+      "train/total_kl": 0.03593848291784525
+    },
+    {
+      "epoch": 2.9194401244167962,
+      "grad_norm": 0.98828125,
+      "learning_rate": 8.097758405977585e-06,
+      "loss": 0.3243,
+      "mean_token_accuracy": 0.9754409641027451,
+      "step": 2345,
+      "train/kl_loss_qwen": 0.032446541963145135,
+      "train/kl_loss_r1": 0.016262303036637605,
+      "train/total_kl": 0.04870884427800774
+    },
+    {
+      "epoch": 2.925660964230171,
+      "grad_norm": 0.99609375,
+      "learning_rate": 8.051058530510586e-06,
+      "loss": 0.2413,
+      "mean_token_accuracy": 0.9799944281578064,
+      "step": 2350,
+      "train/kl_loss_qwen": 0.02185902800410986,
+      "train/kl_loss_r1": 0.012194494740106166,
+      "train/total_kl": 0.03405352234840393
+    },
+    {
+      "epoch": 2.931881804043546,
+      "grad_norm": 0.9453125,
+      "learning_rate": 8.004358655043588e-06,
+      "loss": 0.2488,
+      "mean_token_accuracy": 0.9812228798866272,
+      "step": 2355,
+      "train/kl_loss_qwen": 0.023556680977344514,
+      "train/kl_loss_r1": 0.01458953726105392,
+      "train/total_kl": 0.038146217819303274
+    },
+    {
+      "epoch": 2.9381026438569204,
+      "grad_norm": 1.296875,
+      "learning_rate": 7.957658779576589e-06,
+      "loss": 0.2386,
+      "mean_token_accuracy": 0.9794855356216431,
+      "step": 2360,
+      "train/kl_loss_qwen": 0.019754012860357762,
+      "train/kl_loss_r1": 0.0134123150492087,
+      "train/total_kl": 0.033166327979415655
+    },
+    {
+      "epoch": 2.9443234836702956,
+      "grad_norm": 0.90625,
+      "learning_rate": 7.91095890410959e-06,
+      "loss": 0.2524,
+      "mean_token_accuracy": 0.9802679121494293,
+      "step": 2365,
+      "train/kl_loss_qwen": 0.021867240034043788,
+      "train/kl_loss_r1": 0.015403355937451124,
+      "train/total_kl": 0.03727059587836266
+    },
+    {
+      "epoch": 2.9505443234836704,
+      "grad_norm": 0.92578125,
+      "learning_rate": 7.86425902864259e-06,
+      "loss": 0.2384,
+      "mean_token_accuracy": 0.9784934759140015,
+      "step": 2370,
+      "train/kl_loss_qwen": 0.019248896767385303,
+      "train/kl_loss_r1": 0.013327605556696653,
+      "train/total_kl": 0.032576502207666634
+    },
+    {
+      "epoch": 2.956765163297045,
+      "grad_norm": 1.1171875,
+      "learning_rate": 7.81755915317559e-06,
+      "loss": 0.2528,
+      "mean_token_accuracy": 0.9785091251134872,
+      "step": 2375,
+      "train/kl_loss_qwen": 0.02136381221935153,
+      "train/kl_loss_r1": 0.01490559543017298,
+      "train/total_kl": 0.03626940799877047
+    },
+    {
+      "epoch": 2.96298600311042,
+      "grad_norm": 1.0234375,
+      "learning_rate": 7.770859277708593e-06,
+      "loss": 0.254,
+      "mean_token_accuracy": 0.9753740966320038,
+      "step": 2380,
+      "train/kl_loss_qwen": 0.020779022481292486,
+      "train/kl_loss_r1": 0.012893599388189615,
+      "train/total_kl": 0.033672621753066775
+    },
+    {
+      "epoch": 2.9692068429237946,
+      "grad_norm": 1.296875,
+      "learning_rate": 7.724159402241594e-06,
+      "loss": 0.2723,
+      "mean_token_accuracy": 0.9807368129491806,
+      "step": 2385,
+      "train/kl_loss_qwen": 0.02706731208600104,
+      "train/kl_loss_r1": 0.015577892120927573,
+      "train/total_kl": 0.04264520406723023
+    },
+    {
+      "epoch": 2.9754276827371697,
+      "grad_norm": 1.1796875,
+      "learning_rate": 7.677459526774596e-06,
+      "loss": 0.2339,
+      "mean_token_accuracy": 0.9778724700212479,
+      "step": 2390,
+      "train/kl_loss_qwen": 0.019125875597819687,
+      "train/kl_loss_r1": 0.012182414485141634,
+      "train/total_kl": 0.03130828971043229
+    },
+    {
+      "epoch": 2.9816485225505445,
+      "grad_norm": 1.2578125,
+      "learning_rate": 7.630759651307597e-06,
+      "loss": 0.243,
+      "mean_token_accuracy": 0.9793420016765595,
+      "step": 2395,
+      "train/kl_loss_qwen": 0.021404419979080557,
+      "train/kl_loss_r1": 0.012169917370192707,
+      "train/total_kl": 0.033574337419122455
+    },
+    {
+      "epoch": 2.987869362363919,
+      "grad_norm": 0.984375,
+      "learning_rate": 7.584059775840598e-06,
+      "loss": 0.2408,
+      "mean_token_accuracy": 0.9772315412759781,
+      "step": 2400,
+      "train/kl_loss_qwen": 0.02061320682987571,
+      "train/kl_loss_r1": 0.013303012656979262,
+      "train/total_kl": 0.03391621951013803
+    },
+    {
+      "epoch": 2.994090202177294,
+      "grad_norm": 0.94921875,
+      "learning_rate": 7.537359900373599e-06,
+      "loss": 0.2771,
+      "mean_token_accuracy": 0.9785916924476623,
+      "step": 2405,
+      "train/kl_loss_qwen": 0.0252801182679832,
+      "train/kl_loss_r1": 0.01622594220098108,
+      "train/total_kl": 0.0415060605853796
+    },
+    {
+      "epoch": 3.001244167962675,
+      "grad_norm": 1.578125,
+      "learning_rate": 7.4906600249066005e-06,
+      "loss": 0.2953,
+      "mean_token_accuracy": 0.9816724224524065,
+      "step": 2410,
+      "train/kl_loss_qwen": 0.025597207248210907,
+      "train/kl_loss_r1": 0.01563291643238203,
+      "train/total_kl": 0.041230123659426514
+    },
+    {
+      "epoch": 3.00746500777605,
+      "grad_norm": 0.85546875,
+      "learning_rate": 7.443960149439601e-06,
+      "loss": 0.3073,
+      "mean_token_accuracy": 0.9799118191003799,
+      "step": 2415,
+      "train/kl_loss_qwen": 0.030909589771181346,
+      "train/kl_loss_r1": 0.01667693620547652,
+      "train/total_kl": 0.047586525231599806
+    },
+    {
+      "epoch": 3.0136858475894246,
+      "grad_norm": 0.984375,
+      "learning_rate": 7.397260273972603e-06,
+      "loss": 0.2404,
+      "mean_token_accuracy": 0.9838899701833725,
+      "step": 2420,
+      "train/kl_loss_qwen": 0.021309804217889906,
+      "train/kl_loss_r1": 0.013884992711246014,
+      "train/total_kl": 0.03519479688256979
+    },
+    {
+      "epoch": 3.0199066874027993,
+      "grad_norm": 0.875,
+      "learning_rate": 7.350560398505604e-06,
+      "loss": 0.2295,
+      "mean_token_accuracy": 0.9842253357172013,
+      "step": 2425,
+      "train/kl_loss_qwen": 0.021593318693339823,
+      "train/kl_loss_r1": 0.012967960792593658,
+      "train/total_kl": 0.034561279509216544
+    },
+    {
+      "epoch": 3.026127527216174,
+      "grad_norm": 0.8125,
+      "learning_rate": 7.303860523038606e-06,
+      "loss": 0.226,
+      "mean_token_accuracy": 0.9844829171895981,
+      "step": 2430,
+      "train/kl_loss_qwen": 0.019423839217051864,
+      "train/kl_loss_r1": 0.012976301368325949,
+      "train/total_kl": 0.032400140445679425
+    },
+    {
+      "epoch": 3.032348367029549,
+      "grad_norm": 1.578125,
+      "learning_rate": 7.2571606475716064e-06,
+      "loss": 0.2594,
+      "mean_token_accuracy": 0.9829055160284043,
+      "step": 2435,
+      "train/kl_loss_qwen": 0.02393628782592714,
+      "train/kl_loss_r1": 0.015033241617493332,
+      "train/total_kl": 0.03896952914074063
+    },
+    {
+      "epoch": 3.038569206842924,
+      "grad_norm": 0.89453125,
+      "learning_rate": 7.210460772104608e-06,
+      "loss": 0.2435,
+      "mean_token_accuracy": 0.9866632699966431,
+      "step": 2440,
+      "train/kl_loss_qwen": 0.023871434153988957,
+      "train/kl_loss_r1": 0.01572964512743056,
+      "train/total_kl": 0.03960107890889049
+    },
+    {
+      "epoch": 3.0447900466562987,
+      "grad_norm": 1.0234375,
+      "learning_rate": 7.1637608966376095e-06,
+      "loss": 0.2378,
+      "mean_token_accuracy": 0.9842400878667832,
+      "step": 2445,
+      "train/kl_loss_qwen": 0.022447610320523382,
+      "train/kl_loss_r1": 0.013542116852477193,
+      "train/total_kl": 0.03598972680047154
+    },
+    {
+      "epoch": 3.0510108864696734,
+      "grad_norm": 0.90625,
+      "learning_rate": 7.11706102117061e-06,
+      "loss": 0.2568,
+      "mean_token_accuracy": 0.9835693746805191,
+      "step": 2450,
+      "train/kl_loss_qwen": 0.0250473961699754,
+      "train/kl_loss_r1": 0.015254210913553834,
+      "train/total_kl": 0.040301607269793746
+    },
+    {
+      "epoch": 3.057231726283048,
+      "grad_norm": 0.8359375,
+      "learning_rate": 7.070361145703612e-06,
+      "loss": 0.2386,
+      "mean_token_accuracy": 0.9851376801729202,
+      "step": 2455,
+      "train/kl_loss_qwen": 0.023010611534118652,
+      "train/kl_loss_r1": 0.013890899089165031,
+      "train/total_kl": 0.036901510879397394
+    },
+    {
+      "epoch": 3.063452566096423,
+      "grad_norm": 1.1171875,
+      "learning_rate": 7.023661270236612e-06,
+      "loss": 0.2295,
+      "mean_token_accuracy": 0.9852157533168793,
+      "step": 2460,
+      "train/kl_loss_qwen": 0.020923609985038637,
+      "train/kl_loss_r1": 0.013110134052112699,
+      "train/total_kl": 0.034033743944019076
+    },
+    {
+      "epoch": 3.0696734059097976,
+      "grad_norm": 0.84375,
+      "learning_rate": 6.976961394769615e-06,
+      "loss": 0.2367,
+      "mean_token_accuracy": 0.984752967953682,
+      "step": 2465,
+      "train/kl_loss_qwen": 0.021800487814471126,
+      "train/kl_loss_r1": 0.014962652372196317,
+      "train/total_kl": 0.03676314065232873
+    },
+    {
+      "epoch": 3.075894245723173,
+      "grad_norm": 0.953125,
+      "learning_rate": 6.9302615193026155e-06,
+      "loss": 0.2176,
+      "mean_token_accuracy": 0.9840360462665558,
+      "step": 2470,
+      "train/kl_loss_qwen": 0.019439062848687173,
+      "train/kl_loss_r1": 0.012102234200574458,
+      "train/total_kl": 0.03154129749163985
+    },
+    {
+      "epoch": 3.0821150855365476,
+      "grad_norm": 0.91015625,
+      "learning_rate": 6.883561643835617e-06,
+      "loss": 0.24,
+      "mean_token_accuracy": 0.9856468439102173,
+      "step": 2475,
+      "train/kl_loss_qwen": 0.02432550024241209,
+      "train/kl_loss_r1": 0.014322337484918535,
+      "train/total_kl": 0.03864783812314272
+    },
+    {
+      "epoch": 3.0883359253499223,
+      "grad_norm": 0.8203125,
+      "learning_rate": 6.836861768368618e-06,
+      "loss": 0.2422,
+      "mean_token_accuracy": 0.9857906192541123,
+      "step": 2480,
+      "train/kl_loss_qwen": 0.024130608467385174,
+      "train/kl_loss_r1": 0.014549318444915115,
+      "train/total_kl": 0.03867992656305432
+    },
+    {
+      "epoch": 3.094556765163297,
+      "grad_norm": 0.7734375,
+      "learning_rate": 6.790161892901618e-06,
+      "loss": 0.2854,
+      "mean_token_accuracy": 0.9790276646614074,
+      "step": 2485,
+      "train/kl_loss_qwen": 0.02656708569265902,
+      "train/kl_loss_r1": 0.014770055492408573,
+      "train/total_kl": 0.04133714027702808
+    },
+    {
+      "epoch": 3.1007776049766718,
+      "grad_norm": 0.98828125,
+      "learning_rate": 6.743462017434621e-06,
+      "loss": 0.2284,
+      "mean_token_accuracy": 0.9816806197166443,
+      "step": 2490,
+      "train/kl_loss_qwen": 0.020325628202408554,
+      "train/kl_loss_r1": 0.012261225422844291,
+      "train/total_kl": 0.032586853578686716
+    },
+    {
+      "epoch": 3.1069984447900465,
+      "grad_norm": 0.80078125,
+      "learning_rate": 6.696762141967621e-06,
+      "loss": 0.2392,
+      "mean_token_accuracy": 0.9851239174604416,
+      "step": 2495,
+      "train/kl_loss_qwen": 0.023483004746958615,
+      "train/kl_loss_r1": 0.013791011273860931,
+      "train/total_kl": 0.037274016067385674
+    },
+    {
+      "epoch": 3.1132192846034217,
+      "grad_norm": 0.87890625,
+      "learning_rate": 6.650062266500623e-06,
+      "loss": 0.2369,
+      "mean_token_accuracy": 0.9840823352336884,
+      "step": 2500,
+      "train/kl_loss_qwen": 0.022309778584167363,
+      "train/kl_loss_r1": 0.012782163382507861,
+      "train/total_kl": 0.0350919421762228
+    },
+    {
+      "epoch": 3.1194401244167964,
+      "grad_norm": 0.8515625,
+      "learning_rate": 6.603362391033624e-06,
+      "loss": 0.229,
+      "mean_token_accuracy": 0.9852802842855454,
+      "step": 2505,
+      "train/kl_loss_qwen": 0.022154048085212708,
+      "train/kl_loss_r1": 0.01309482108335942,
+      "train/total_kl": 0.035248869378119704
+    },
+    {
+      "epoch": 3.125660964230171,
+      "grad_norm": 0.8359375,
+      "learning_rate": 6.556662515566626e-06,
+      "loss": 0.2327,
+      "mean_token_accuracy": 0.9822622090578079,
+      "step": 2510,
+      "train/kl_loss_qwen": 0.020434158109128474,
+      "train/kl_loss_r1": 0.012905464507639408,
+      "train/total_kl": 0.03333962252363563
+    },
+    {
+      "epoch": 3.131881804043546,
+      "grad_norm": 0.98046875,
+      "learning_rate": 6.509962640099627e-06,
+      "loss": 0.2825,
+      "mean_token_accuracy": 0.9828780442476273,
+      "step": 2515,
+      "train/kl_loss_qwen": 0.028179554687812924,
+      "train/kl_loss_r1": 0.016975441295653582,
+      "train/total_kl": 0.04515499575063586
+    },
+    {
+      "epoch": 3.1381026438569206,
+      "grad_norm": 1.2578125,
+      "learning_rate": 6.463262764632628e-06,
+      "loss": 0.2292,
+      "mean_token_accuracy": 0.9861107170581818,
+      "step": 2520,
+      "train/kl_loss_qwen": 0.020859976392239334,
+      "train/kl_loss_r1": 0.012708152551203966,
+      "train/total_kl": 0.03356812903657556
+    },
+    {
+      "epoch": 3.1443234836702953,
+      "grad_norm": 0.890625,
+      "learning_rate": 6.416562889165629e-06,
+      "loss": 0.2433,
+      "mean_token_accuracy": 0.9862571328878402,
+      "step": 2525,
+      "train/kl_loss_qwen": 0.022917523747310042,
+      "train/kl_loss_r1": 0.014852290134876966,
+      "train/total_kl": 0.037769814115017654
+    },
+    {
+      "epoch": 3.15054432348367,
+      "grad_norm": 0.83984375,
+      "learning_rate": 6.3698630136986296e-06,
+      "loss": 0.2509,
+      "mean_token_accuracy": 0.9852677673101425,
+      "step": 2530,
+      "train/kl_loss_qwen": 0.025660164793953298,
+      "train/kl_loss_r1": 0.014589101611636578,
+      "train/total_kl": 0.04024926638230682
+    },
+    {
+      "epoch": 3.1567651632970453,
+      "grad_norm": 1.9140625,
+      "learning_rate": 6.323163138231632e-06,
+      "loss": 0.2669,
+      "mean_token_accuracy": 0.9850893825292587,
+      "step": 2535,
+      "train/kl_loss_qwen": 0.027122320886701346,
+      "train/kl_loss_r1": 0.014969939785078167,
+      "train/total_kl": 0.04209226043894887
+    },
+    {
+      "epoch": 3.16298600311042,
+      "grad_norm": 1.1875,
+      "learning_rate": 6.276463262764633e-06,
+      "loss": 0.2448,
+      "mean_token_accuracy": 0.9816835910081864,
+      "step": 2540,
+      "train/kl_loss_qwen": 0.02137654577381909,
+      "train/kl_loss_r1": 0.013053974509239197,
+      "train/total_kl": 0.03443052032962442
+    },
+    {
+      "epoch": 3.1692068429237947,
+      "grad_norm": 0.890625,
+      "learning_rate": 6.229763387297634e-06,
+      "loss": 0.2434,
+      "mean_token_accuracy": 0.9859046876430512,
+      "step": 2545,
+      "train/kl_loss_qwen": 0.02334826118312776,
+      "train/kl_loss_r1": 0.014492624066770076,
+      "train/total_kl": 0.037840885668993
+    },
+    {
+      "epoch": 3.1754276827371695,
+      "grad_norm": 0.91015625,
+      "learning_rate": 6.183063511830635e-06,
+      "loss": 0.2375,
+      "mean_token_accuracy": 0.9857018113136291,
+      "step": 2550,
+      "train/kl_loss_qwen": 0.02287781867198646,
+      "train/kl_loss_r1": 0.014699449087493122,
+      "train/total_kl": 0.03757726764306426
+    },
+    {
+      "epoch": 3.181648522550544,
+      "grad_norm": 0.8671875,
+      "learning_rate": 6.136363636363637e-06,
+      "loss": 0.2169,
+      "mean_token_accuracy": 0.9859404623508453,
+      "step": 2555,
+      "train/kl_loss_qwen": 0.01947087454609573,
+      "train/kl_loss_r1": 0.01351618163753301,
+      "train/total_kl": 0.03298705592751503
+    },
+    {
+      "epoch": 3.187869362363919,
+      "grad_norm": 1.15625,
+      "learning_rate": 6.089663760896638e-06,
+      "loss": 0.2239,
+      "mean_token_accuracy": 0.9842052280902862,
+      "step": 2560,
+      "train/kl_loss_qwen": 0.020881024189293385,
+      "train/kl_loss_r1": 0.01321516155730933,
+      "train/total_kl": 0.034096185490489005
+    },
+    {
+      "epoch": 3.194090202177294,
+      "grad_norm": 0.984375,
+      "learning_rate": 6.042963885429639e-06,
+      "loss": 0.2523,
+      "mean_token_accuracy": 0.9886922299861908,
+      "step": 2565,
+      "train/kl_loss_qwen": 0.025517759006470443,
+      "train/kl_loss_r1": 0.01657579594757408,
+      "train/total_kl": 0.042093554977327585
+    },
+    {
+      "epoch": 3.200311041990669,
+      "grad_norm": 0.94140625,
+      "learning_rate": 5.99626400996264e-06,
+      "loss": 0.2401,
+      "mean_token_accuracy": 0.984867662191391,
+      "step": 2570,
+      "train/kl_loss_qwen": 0.023075867211446166,
+      "train/kl_loss_r1": 0.013981684250757099,
+      "train/total_kl": 0.03705755146220326
+    },
+    {
+      "epoch": 3.2065318818040436,
+      "grad_norm": 0.94921875,
+      "learning_rate": 5.949564134495641e-06,
+      "loss": 0.2792,
+      "mean_token_accuracy": 0.9805339246988296,
+      "step": 2575,
+      "train/kl_loss_qwen": 0.025687551125884056,
+      "train/kl_loss_r1": 0.01688022376038134,
+      "train/total_kl": 0.042567774560302495
+    },
+    {
+      "epoch": 3.2127527216174183,
+      "grad_norm": 0.78515625,
+      "learning_rate": 5.902864259028643e-06,
+      "loss": 0.2359,
+      "mean_token_accuracy": 0.9867773085832596,
+      "step": 2580,
+      "train/kl_loss_qwen": 0.022642063442617655,
+      "train/kl_loss_r1": 0.01564873745664954,
+      "train/total_kl": 0.038290800713002685
+    },
+    {
+      "epoch": 3.218973561430793,
+      "grad_norm": 0.77734375,
+      "learning_rate": 5.856164383561644e-06,
+      "loss": 0.2185,
+      "mean_token_accuracy": 0.9818668901920319,
+      "step": 2585,
+      "train/kl_loss_qwen": 0.01890602051280439,
+      "train/kl_loss_r1": 0.011296134302392602,
+      "train/total_kl": 0.030202154675498603
+    },
+    {
+      "epoch": 3.225194401244168,
+      "grad_norm": 0.82421875,
+      "learning_rate": 5.809464508094645e-06,
+      "loss": 0.246,
+      "mean_token_accuracy": 0.9839199364185334,
+      "step": 2590,
+      "train/kl_loss_qwen": 0.025229747220873833,
+      "train/kl_loss_r1": 0.013694578688591719,
+      "train/total_kl": 0.038924325909465554
+    },
+    {
+      "epoch": 3.231415241057543,
+      "grad_norm": 0.97265625,
+      "learning_rate": 5.762764632627646e-06,
+      "loss": 0.2688,
+      "mean_token_accuracy": 0.9858499974012375,
+      "step": 2595,
+      "train/kl_loss_qwen": 0.026664407551288606,
+      "train/kl_loss_r1": 0.01758768109139055,
+      "train/total_kl": 0.04425208875909448
+    },
+    {
+      "epoch": 3.2376360808709177,
+      "grad_norm": 0.76953125,
+      "learning_rate": 5.7160647571606484e-06,
+      "loss": 0.2406,
+      "mean_token_accuracy": 0.9867913007736206,
+      "step": 2600,
+      "train/kl_loss_qwen": 0.02284672809764743,
+      "train/kl_loss_r1": 0.014909073058515788,
+      "train/total_kl": 0.03775580152869225
+    },
+    {
+      "epoch": 3.2438569206842924,
+      "grad_norm": 0.98828125,
+      "learning_rate": 5.669364881693649e-06,
+      "loss": 0.2292,
+      "mean_token_accuracy": 0.9871266216039658,
+      "step": 2605,
+      "train/kl_loss_qwen": 0.021353343920782208,
+      "train/kl_loss_r1": 0.013833488407544791,
+      "train/total_kl": 0.035186832677572964
+    },
+    {
+      "epoch": 3.250077760497667,
+      "grad_norm": 0.8359375,
+      "learning_rate": 5.62266500622665e-06,
+      "loss": 0.243,
+      "mean_token_accuracy": 0.9875899940729141,
+      "step": 2610,
+      "train/kl_loss_qwen": 0.02369601596146822,
+      "train/kl_loss_r1": 0.015644747391343118,
+      "train/total_kl": 0.03934076325967908
+    },
+    {
+      "epoch": 3.256298600311042,
+      "grad_norm": 0.92578125,
+      "learning_rate": 5.575965130759651e-06,
+      "loss": 0.2631,
+      "mean_token_accuracy": 0.9842116951942443,
+      "step": 2615,
+      "train/kl_loss_qwen": 0.02683533593080938,
+      "train/kl_loss_r1": 0.015043137269094587,
+      "train/total_kl": 0.04187847292050719
+    },
+    {
+      "epoch": 3.2625194401244166,
+      "grad_norm": 0.94140625,
+      "learning_rate": 5.529265255292652e-06,
+      "loss": 0.2267,
+      "mean_token_accuracy": 0.9818090766668319,
+      "step": 2620,
+      "train/kl_loss_qwen": 0.02027418022044003,
+      "train/kl_loss_r1": 0.011717557231895626,
+      "train/total_kl": 0.03199173752218485
+    },
+    {
+      "epoch": 3.2687402799377914,
+      "grad_norm": 0.828125,
+      "learning_rate": 5.482565379825654e-06,
+      "loss": 0.2873,
+      "mean_token_accuracy": 0.9823738813400269,
+      "step": 2625,
+      "train/kl_loss_qwen": 0.029073307709768414,
+      "train/kl_loss_r1": 0.016109167714603245,
+      "train/total_kl": 0.045182475447654726
+    },
+    {
+      "epoch": 3.2749611197511665,
+      "grad_norm": 0.9375,
+      "learning_rate": 5.435865504358655e-06,
+      "loss": 0.2492,
+      "mean_token_accuracy": 0.9855016142129898,
+      "step": 2630,
+      "train/kl_loss_qwen": 0.023096950352191926,
+      "train/kl_loss_r1": 0.015297620929777623,
+      "train/total_kl": 0.038394571375101806
+    },
+    {
+      "epoch": 3.2811819595645413,
+      "grad_norm": 0.921875,
+      "learning_rate": 5.389165628891657e-06,
+      "loss": 0.239,
+      "mean_token_accuracy": 0.9847406953573227,
+      "step": 2635,
+      "train/kl_loss_qwen": 0.021637895377352834,
+      "train/kl_loss_r1": 0.01458637174218893,
+      "train/total_kl": 0.03622426679357886
+    },
+    {
+      "epoch": 3.287402799377916,
+      "grad_norm": 0.87890625,
+      "learning_rate": 5.342465753424657e-06,
+      "loss": 0.2682,
+      "mean_token_accuracy": 0.9817533552646637,
+      "step": 2640,
+      "train/kl_loss_qwen": 0.026832366455346347,
+      "train/kl_loss_r1": 0.015036878059618175,
+      "train/total_kl": 0.041869244445115326
+    },
+    {
+      "epoch": 3.2936236391912908,
+      "grad_norm": 1.046875,
+      "learning_rate": 5.29576587795766e-06,
+      "loss": 0.2263,
+      "mean_token_accuracy": 0.9815866380929947,
+      "step": 2645,
+      "train/kl_loss_qwen": 0.020331070409156382,
+      "train/kl_loss_r1": 0.012977494508959353,
+      "train/total_kl": 0.03330856496468186
+    },
+    {
+      "epoch": 3.2998444790046655,
+      "grad_norm": 0.875,
+      "learning_rate": 5.24906600249066e-06,
+      "loss": 0.2629,
+      "mean_token_accuracy": 0.9831759363412857,
+      "step": 2650,
+      "train/kl_loss_qwen": 0.02541449787095189,
+      "train/kl_loss_r1": 0.015678114630281924,
+      "train/total_kl": 0.04109261250123382
+    },
+    {
+      "epoch": 3.3060653188180407,
+      "grad_norm": 1.3515625,
+      "learning_rate": 5.202366127023661e-06,
+      "loss": 0.2377,
+      "mean_token_accuracy": 0.9843966364860535,
+      "step": 2655,
+      "train/kl_loss_qwen": 0.022267935564741492,
+      "train/kl_loss_r1": 0.014039589231833816,
+      "train/total_kl": 0.03630752470344305
+    },
+    {
+      "epoch": 3.3122861586314154,
+      "grad_norm": 1.03125,
+      "learning_rate": 5.1556662515566625e-06,
+      "loss": 0.2288,
+      "mean_token_accuracy": 0.9853380292654037,
+      "step": 2660,
+      "train/kl_loss_qwen": 0.020760743832215666,
+      "train/kl_loss_r1": 0.013922068243846297,
+      "train/total_kl": 0.03468281263485551
+    },
+    {
+      "epoch": 3.31850699844479,
+      "grad_norm": 1.3046875,
+      "learning_rate": 5.108966376089663e-06,
+      "loss": 0.2746,
+      "mean_token_accuracy": 0.9855523884296418,
+      "step": 2665,
+      "train/kl_loss_qwen": 0.028636119095608592,
+      "train/kl_loss_r1": 0.016617331188172102,
+      "train/total_kl": 0.04525345005095005
+    },
+    {
+      "epoch": 3.324727838258165,
+      "grad_norm": 0.87890625,
+      "learning_rate": 5.062266500622666e-06,
+      "loss": 0.2796,
+      "mean_token_accuracy": 0.9843825936317444,
+      "step": 2670,
+      "train/kl_loss_qwen": 0.02719255774281919,
+      "train/kl_loss_r1": 0.01748746286612004,
+      "train/total_kl": 0.04468002058565616
+    },
+    {
+      "epoch": 3.3309486780715396,
+      "grad_norm": 0.890625,
+      "learning_rate": 5.015566625155666e-06,
+      "loss": 0.2395,
+      "mean_token_accuracy": 0.985189613699913,
+      "step": 2675,
+      "train/kl_loss_qwen": 0.022867527883499862,
+      "train/kl_loss_r1": 0.01357824713923037,
+      "train/total_kl": 0.03644577506929636
+    },
+    {
+      "epoch": 3.3371695178849143,
+      "grad_norm": 0.984375,
+      "learning_rate": 4.968866749688668e-06,
+      "loss": 0.2537,
+      "mean_token_accuracy": 0.9825487434864044,
+      "step": 2680,
+      "train/kl_loss_qwen": 0.023506175680086016,
+      "train/kl_loss_r1": 0.014184463652782142,
+      "train/total_kl": 0.03769063977524638
+    },
+    {
+      "epoch": 3.343390357698289,
+      "grad_norm": 0.859375,
+      "learning_rate": 4.9221668742216685e-06,
+      "loss": 0.2369,
+      "mean_token_accuracy": 0.9860921442508698,
+      "step": 2685,
+      "train/kl_loss_qwen": 0.023705671727657317,
+      "train/kl_loss_r1": 0.014161279378458857,
+      "train/total_kl": 0.03786695105955005
+    },
+    {
+      "epoch": 3.3496111975116643,
+      "grad_norm": 1.015625,
+      "learning_rate": 4.87546699875467e-06,
+      "loss": 0.2299,
+      "mean_token_accuracy": 0.9845267534255981,
+      "step": 2690,
+      "train/kl_loss_qwen": 0.021310369204729795,
+      "train/kl_loss_r1": 0.013280869252048433,
+      "train/total_kl": 0.03459123857319355
+    },
+    {
+      "epoch": 3.355832037325039,
+      "grad_norm": 0.73046875,
+      "learning_rate": 4.8287671232876716e-06,
+      "loss": 0.2275,
+      "mean_token_accuracy": 0.9860312432050705,
+      "step": 2695,
+      "train/kl_loss_qwen": 0.02123181507922709,
+      "train/kl_loss_r1": 0.013362622214481234,
+      "train/total_kl": 0.03459443720057607
+    },
+    {
+      "epoch": 3.3620528771384137,
+      "grad_norm": 1.0625,
+      "learning_rate": 4.782067247820672e-06,
+      "loss": 0.2565,
+      "mean_token_accuracy": 0.9833629161119462,
+      "step": 2700,
+      "train/kl_loss_qwen": 0.024914097087457777,
+      "train/kl_loss_r1": 0.013864034041762351,
+      "train/total_kl": 0.038778131082654
+    },
+    {
+      "epoch": 3.3682737169517885,
+      "grad_norm": 1.1015625,
+      "learning_rate": 4.735367372353674e-06,
+      "loss": 0.2274,
+      "mean_token_accuracy": 0.985034841299057,
+      "step": 2705,
+      "train/kl_loss_qwen": 0.021230686455965042,
+      "train/kl_loss_r1": 0.01340056979097426,
+      "train/total_kl": 0.03463125610724092
+    },
+    {
+      "epoch": 3.374494556765163,
+      "grad_norm": 1.0625,
+      "learning_rate": 4.6886674968866744e-06,
+      "loss": 0.2197,
+      "mean_token_accuracy": 0.9832078158855438,
+      "step": 2710,
+      "train/kl_loss_qwen": 0.019347388157621027,
+      "train/kl_loss_r1": 0.011695200949907303,
+      "train/total_kl": 0.031042589247226714
+    },
+    {
+      "epoch": 3.380715396578538,
+      "grad_norm": 0.875,
+      "learning_rate": 4.641967621419677e-06,
+      "loss": 0.2359,
+      "mean_token_accuracy": 0.9849886059761047,
+      "step": 2715,
+      "train/kl_loss_qwen": 0.022014891589060427,
+      "train/kl_loss_r1": 0.014034424582496286,
+      "train/total_kl": 0.03604931645095348
+    },
+    {
+      "epoch": 3.386936236391913,
+      "grad_norm": 0.87109375,
+      "learning_rate": 4.5952677459526775e-06,
+      "loss": 0.2505,
+      "mean_token_accuracy": 0.984162563085556,
+      "step": 2720,
+      "train/kl_loss_qwen": 0.02434834372252226,
+      "train/kl_loss_r1": 0.014979369170032442,
+      "train/total_kl": 0.03932771300897002
+    },
+    {
+      "epoch": 3.393157076205288,
+      "grad_norm": 0.8203125,
+      "learning_rate": 4.548567870485679e-06,
+      "loss": 0.2663,
+      "mean_token_accuracy": 0.984138822555542,
+      "step": 2725,
+      "train/kl_loss_qwen": 0.026038512215018274,
+      "train/kl_loss_r1": 0.015358842397108673,
+      "train/total_kl": 0.04139735447242856
+    },
+    {
+      "epoch": 3.3993779160186626,
+      "grad_norm": 1.046875,
+      "learning_rate": 4.50186799501868e-06,
+      "loss": 0.2214,
+      "mean_token_accuracy": 0.9856981605291366,
+      "step": 2730,
+      "train/kl_loss_qwen": 0.019446768146008254,
+      "train/kl_loss_r1": 0.012801338406279682,
+      "train/total_kl": 0.0322481069713831
+    },
+    {
+      "epoch": 3.4055987558320373,
+      "grad_norm": 1.1796875,
+      "learning_rate": 4.455168119551681e-06,
+      "loss": 0.2338,
+      "mean_token_accuracy": 0.9839027732610702,
+      "step": 2735,
+      "train/kl_loss_qwen": 0.02064052429050207,
+      "train/kl_loss_r1": 0.012976438086479903,
+      "train/total_kl": 0.033616962376981974
+    },
+    {
+      "epoch": 3.411819595645412,
+      "grad_norm": 1.0078125,
+      "learning_rate": 4.408468244084683e-06,
+      "loss": 0.2274,
+      "mean_token_accuracy": 0.9858880639076233,
+      "step": 2740,
+      "train/kl_loss_qwen": 0.020935347443446516,
+      "train/kl_loss_r1": 0.013251440855674446,
+      "train/total_kl": 0.03418678799644113
+    },
+    {
+      "epoch": 3.4180404354587868,
+      "grad_norm": 0.83984375,
+      "learning_rate": 4.3617683686176835e-06,
+      "loss": 0.2285,
+      "mean_token_accuracy": 0.9858361423015595,
+      "step": 2745,
+      "train/kl_loss_qwen": 0.021212745131924747,
+      "train/kl_loss_r1": 0.01365222295280546,
+      "train/total_kl": 0.03486496862024069
+    },
+    {
+      "epoch": 3.424261275272162,
+      "grad_norm": 1.0078125,
+      "learning_rate": 4.315068493150685e-06,
+      "loss": 0.2295,
+      "mean_token_accuracy": 0.9845366030931473,
+      "step": 2750,
+      "train/kl_loss_qwen": 0.020564079657196997,
+      "train/kl_loss_r1": 0.013706768536940217,
+      "train/total_kl": 0.034270848426967856
+    },
+    {
+      "epoch": 3.4304821150855367,
+      "grad_norm": 1.0078125,
+      "learning_rate": 4.2683686176836865e-06,
+      "loss": 0.225,
+      "mean_token_accuracy": 0.9845517426729202,
+      "step": 2755,
+      "train/kl_loss_qwen": 0.020470702182501556,
+      "train/kl_loss_r1": 0.013152860198169946,
+      "train/total_kl": 0.03362356275320053
+    },
+    {
+      "epoch": 3.4367029548989114,
+      "grad_norm": 0.8046875,
+      "learning_rate": 4.221668742216688e-06,
+      "loss": 0.2341,
+      "mean_token_accuracy": 0.984512385725975,
+      "step": 2760,
+      "train/kl_loss_qwen": 0.021487886970862746,
+      "train/kl_loss_r1": 0.013046193053014576,
+      "train/total_kl": 0.03453407995402813
+    },
+    {
+      "epoch": 3.442923794712286,
+      "grad_norm": 1.0625,
+      "learning_rate": 4.174968866749689e-06,
+      "loss": 0.2231,
+      "mean_token_accuracy": 0.9859201848506928,
+      "step": 2765,
+      "train/kl_loss_qwen": 0.020332144340500236,
+      "train/kl_loss_r1": 0.01277754541952163,
+      "train/total_kl": 0.033109689690172674
+    },
+    {
+      "epoch": 3.449144634525661,
+      "grad_norm": 1.0234375,
+      "learning_rate": 4.12826899128269e-06,
+      "loss": 0.2343,
+      "mean_token_accuracy": 0.9809245645999909,
+      "step": 2770,
+      "train/kl_loss_qwen": 0.020196593133732677,
+      "train/kl_loss_r1": 0.012610416440293194,
+      "train/total_kl": 0.03280700957402587
+    },
+    {
+      "epoch": 3.4553654743390356,
+      "grad_norm": 1.015625,
+      "learning_rate": 4.081569115815691e-06,
+      "loss": 0.2344,
+      "mean_token_accuracy": 0.9842493683099747,
+      "step": 2775,
+      "train/kl_loss_qwen": 0.021661369316279887,
+      "train/kl_loss_r1": 0.013402773765847087,
+      "train/total_kl": 0.03506414322182536
+    },
+    {
+      "epoch": 3.4615863141524104,
+      "grad_norm": 0.875,
+      "learning_rate": 4.0348692403486925e-06,
+      "loss": 0.2494,
+      "mean_token_accuracy": 0.985339805483818,
+      "step": 2780,
+      "train/kl_loss_qwen": 0.023978083208203316,
+      "train/kl_loss_r1": 0.014692715555429458,
+      "train/total_kl": 0.038670798763632774
+    },
+    {
+      "epoch": 3.4678071539657855,
+      "grad_norm": 0.8671875,
+      "learning_rate": 3.988169364881694e-06,
+      "loss": 0.2273,
+      "mean_token_accuracy": 0.9854962199926376,
+      "step": 2785,
+      "train/kl_loss_qwen": 0.020899151754565536,
+      "train/kl_loss_r1": 0.013141945214010775,
+      "train/total_kl": 0.03404109720140695
+    },
+    {
+      "epoch": 3.4740279937791603,
+      "grad_norm": 1.4609375,
+      "learning_rate": 3.941469489414695e-06,
+      "loss": 0.247,
+      "mean_token_accuracy": 0.9863706976175308,
+      "step": 2790,
+      "train/kl_loss_qwen": 0.02398818014189601,
+      "train/kl_loss_r1": 0.014264530991204083,
+      "train/total_kl": 0.038252711109817025
+    },
+    {
+      "epoch": 3.480248833592535,
+      "grad_norm": 0.79296875,
+      "learning_rate": 3.894769613947696e-06,
+      "loss": 0.2306,
+      "mean_token_accuracy": 0.9847693234682083,
+      "step": 2795,
+      "train/kl_loss_qwen": 0.020988399861380458,
+      "train/kl_loss_r1": 0.013396636093966663,
+      "train/total_kl": 0.03438503593206406
+    },
+    {
+      "epoch": 3.4864696734059097,
+      "grad_norm": 0.83984375,
+      "learning_rate": 3.848069738480698e-06,
+      "loss": 0.2387,
+      "mean_token_accuracy": 0.9844767868518829,
+      "step": 2800,
+      "train/kl_loss_qwen": 0.021816241182386877,
+      "train/kl_loss_r1": 0.014730063872411847,
+      "train/total_kl": 0.03654630491510034
+    },
+    {
+      "epoch": 3.4926905132192845,
+      "grad_norm": 1.078125,
+      "learning_rate": 3.801369863013699e-06,
+      "loss": 0.238,
+      "mean_token_accuracy": 0.9856153309345246,
+      "step": 2805,
+      "train/kl_loss_qwen": 0.022997609293088316,
+      "train/kl_loss_r1": 0.014218732388690114,
+      "train/total_kl": 0.037216341588646173
+    },
+    {
+      "epoch": 3.4989113530326597,
+      "grad_norm": 0.81640625,
+      "learning_rate": 3.7546699875467e-06,
+      "loss": 0.2414,
+      "mean_token_accuracy": 0.9863660573959351,
+      "step": 2810,
+      "train/kl_loss_qwen": 0.023191145109012722,
+      "train/kl_loss_r1": 0.014856462017633021,
+      "train/total_kl": 0.03804760677739978
+    },
+    {
+      "epoch": 3.505132192846034,
+      "grad_norm": 1.3046875,
+      "learning_rate": 3.707970112079701e-06,
+      "loss": 0.224,
+      "mean_token_accuracy": 0.9856793075799942,
+      "step": 2815,
+      "train/kl_loss_qwen": 0.020576135814189912,
+      "train/kl_loss_r1": 0.01223513432778418,
+      "train/total_kl": 0.032811269722878934
+    },
+    {
+      "epoch": 3.511353032659409,
+      "grad_norm": 1.03125,
+      "learning_rate": 3.6612702366127026e-06,
+      "loss": 0.2458,
+      "mean_token_accuracy": 0.9840625703334809,
+      "step": 2820,
+      "train/kl_loss_qwen": 0.021890020091086627,
+      "train/kl_loss_r1": 0.015305952937342226,
+      "train/total_kl": 0.03719597272574902
+    },
+    {
+      "epoch": 3.517573872472784,
+      "grad_norm": 0.75390625,
+      "learning_rate": 3.6145703611457037e-06,
+      "loss": 0.2366,
+      "mean_token_accuracy": 0.9833911150693894,
+      "step": 2825,
+      "train/kl_loss_qwen": 0.021799473231658338,
+      "train/kl_loss_r1": 0.013875545375049115,
+      "train/total_kl": 0.035675018839538095
+    },
+    {
+      "epoch": 3.5237947122861586,
+      "grad_norm": 1.15625,
+      "learning_rate": 3.567870485678705e-06,
+      "loss": 0.232,
+      "mean_token_accuracy": 0.9845116376876831,
+      "step": 2830,
+      "train/kl_loss_qwen": 0.021351216733455657,
+      "train/kl_loss_r1": 0.013609965075738728,
+      "train/total_kl": 0.034961181692779064
+    },
+    {
+      "epoch": 3.5300155520995333,
+      "grad_norm": 0.8515625,
+      "learning_rate": 3.5211706102117063e-06,
+      "loss": 0.2336,
+      "mean_token_accuracy": 0.9829171359539032,
+      "step": 2835,
+      "train/kl_loss_qwen": 0.02112548053264618,
+      "train/kl_loss_r1": 0.01297055478207767,
+      "train/total_kl": 0.034096034802496435
+    },
+    {
+      "epoch": 3.536236391912908,
+      "grad_norm": 0.97265625,
+      "learning_rate": 3.4744707347447074e-06,
+      "loss": 0.2601,
+      "mean_token_accuracy": 0.9832925647497177,
+      "step": 2840,
+      "train/kl_loss_qwen": 0.024747973820194603,
+      "train/kl_loss_r1": 0.014901431510224938,
+      "train/total_kl": 0.03964940514415503
+    },
+    {
+      "epoch": 3.5424572317262832,
+      "grad_norm": 0.86328125,
+      "learning_rate": 3.4277708592777085e-06,
+      "loss": 0.2468,
+      "mean_token_accuracy": 0.9836038619279861,
+      "step": 2845,
+      "train/kl_loss_qwen": 0.02415000000037253,
+      "train/kl_loss_r1": 0.014615483209490776,
+      "train/total_kl": 0.03876548307016492
+    },
+    {
+      "epoch": 3.548678071539658,
+      "grad_norm": 0.82421875,
+      "learning_rate": 3.38107098381071e-06,
+      "loss": 0.2488,
+      "mean_token_accuracy": 0.9870301902294158,
+      "step": 2850,
+      "train/kl_loss_qwen": 0.02544220401905477,
+      "train/kl_loss_r1": 0.015255382750183344,
+      "train/total_kl": 0.040697587002068755
+    },
+    {
+      "epoch": 3.5548989113530327,
+      "grad_norm": 0.87109375,
+      "learning_rate": 3.334371108343711e-06,
+      "loss": 0.2663,
+      "mean_token_accuracy": 0.9848790228366852,
+      "step": 2855,
+      "train/kl_loss_qwen": 0.02661481872200966,
+      "train/kl_loss_r1": 0.016790007473900914,
+      "train/total_kl": 0.04340482642874122
+    },
+    {
+      "epoch": 3.5611197511664074,
+      "grad_norm": 1.0859375,
+      "learning_rate": 3.2876712328767123e-06,
+      "loss": 0.3302,
+      "mean_token_accuracy": 0.9799584716558456,
+      "step": 2860,
+      "train/kl_loss_qwen": 0.03250132179819047,
+      "train/kl_loss_r1": 0.01764731693547219,
+      "train/total_kl": 0.05014863768592477
+    },
+    {
+      "epoch": 3.567340590979782,
+      "grad_norm": 0.8359375,
+      "learning_rate": 3.240971357409714e-06,
+      "loss": 0.2314,
+      "mean_token_accuracy": 0.9842890292406082,
+      "step": 2865,
+      "train/kl_loss_qwen": 0.020937564410269262,
+      "train/kl_loss_r1": 0.012288552708923817,
+      "train/total_kl": 0.033226117584854366
+    },
+    {
+      "epoch": 3.573561430793157,
+      "grad_norm": 5.96875,
+      "learning_rate": 3.194271481942715e-06,
+      "loss": 0.4016,
+      "mean_token_accuracy": 0.9750429719686509,
+      "step": 2870,
+      "train/kl_loss_qwen": 0.04201047793030739,
+      "train/kl_loss_r1": 0.022988221771083773,
+      "train/total_kl": 0.06499869888648391
+    },
+    {
+      "epoch": 3.5797822706065316,
+      "grad_norm": 2.0625,
+      "learning_rate": 3.1475716064757164e-06,
+      "loss": 0.414,
+      "mean_token_accuracy": 0.9800378859043122,
+      "step": 2875,
+      "train/kl_loss_qwen": 0.037109590321779254,
+      "train/kl_loss_r1": 0.03484028337989002,
+      "train/total_kl": 0.07194987395778299
+    },
+    {
+      "epoch": 3.586003110419907,
+      "grad_norm": 0.77734375,
+      "learning_rate": 3.100871731008717e-06,
+      "loss": 0.2773,
+      "mean_token_accuracy": 0.9851849496364593,
+      "step": 2880,
+      "train/kl_loss_qwen": 0.0281123923137784,
+      "train/kl_loss_r1": 0.01716474846471101,
+      "train/total_kl": 0.04527714066207409
+    },
+    {
+      "epoch": 3.5922239502332816,
+      "grad_norm": 1.0703125,
+      "learning_rate": 3.0541718555417186e-06,
+      "loss": 0.2348,
+      "mean_token_accuracy": 0.9836596220731735,
+      "step": 2885,
+      "train/kl_loss_qwen": 0.02031282209791243,
+      "train/kl_loss_r1": 0.01318891711998731,
+      "train/total_kl": 0.033501739241182805
+    },
+    {
+      "epoch": 3.5984447900466563,
+      "grad_norm": 0.828125,
+      "learning_rate": 3.0074719800747197e-06,
+      "loss": 0.2329,
+      "mean_token_accuracy": 0.9853809595108032,
+      "step": 2890,
+      "train/kl_loss_qwen": 0.021543790493160485,
+      "train/kl_loss_r1": 0.013360623200424016,
+      "train/total_kl": 0.03490441376343369
+    },
+    {
+      "epoch": 3.604665629860031,
+      "grad_norm": 0.93359375,
+      "learning_rate": 2.9607721046077213e-06,
+      "loss": 0.232,
+      "mean_token_accuracy": 0.9842139780521393,
+      "step": 2895,
+      "train/kl_loss_qwen": 0.019965836266055702,
+      "train/kl_loss_r1": 0.013940862705931067,
+      "train/total_kl": 0.03390669859945774
+    },
+    {
+      "epoch": 3.6108864696734058,
+      "grad_norm": 0.8671875,
+      "learning_rate": 2.9140722291407224e-06,
+      "loss": 0.2179,
+      "mean_token_accuracy": 0.9845876067876815,
+      "step": 2900,
+      "train/kl_loss_qwen": 0.01916212271898985,
+      "train/kl_loss_r1": 0.011550022312439978,
+      "train/total_kl": 0.030712145008146764
+    },
+    {
+      "epoch": 3.617107309486781,
+      "grad_norm": 0.8125,
+      "learning_rate": 2.8673723536737235e-06,
+      "loss": 0.2293,
+      "mean_token_accuracy": 0.9862509101629258,
+      "step": 2905,
+      "train/kl_loss_qwen": 0.02187169035896659,
+      "train/kl_loss_r1": 0.013140536611899734,
+      "train/total_kl": 0.03501222673803568
+    },
+    {
+      "epoch": 3.6233281493001557,
+      "grad_norm": 0.8359375,
+      "learning_rate": 2.820672478206725e-06,
+      "loss": 0.2403,
+      "mean_token_accuracy": 0.9864229768514633,
+      "step": 2910,
+      "train/kl_loss_qwen": 0.022343243472278118,
+      "train/kl_loss_r1": 0.014794543385505676,
+      "train/total_kl": 0.03713778667151928
+    },
+    {
+      "epoch": 3.6295489891135304,
+      "grad_norm": 0.9296875,
+      "learning_rate": 2.773972602739726e-06,
+      "loss": 0.2342,
+      "mean_token_accuracy": 0.9859908074140549,
+      "step": 2915,
+      "train/kl_loss_qwen": 0.020766420010477304,
+      "train/kl_loss_r1": 0.013887669052928686,
+      "train/total_kl": 0.03465408906340599
+    },
+    {
+      "epoch": 3.635769828926905,
+      "grad_norm": 0.93359375,
+      "learning_rate": 2.7272727272727272e-06,
+      "loss": 0.258,
+      "mean_token_accuracy": 0.9863515943288803,
+      "step": 2920,
+      "train/kl_loss_qwen": 0.026655311090871693,
+      "train/kl_loss_r1": 0.015320436703041196,
+      "train/total_kl": 0.04197574770078063
+    },
+    {
+      "epoch": 3.64199066874028,
+      "grad_norm": 0.87890625,
+      "learning_rate": 2.6805728518057283e-06,
+      "loss": 0.2441,
+      "mean_token_accuracy": 0.986115324497223,
+      "step": 2925,
+      "train/kl_loss_qwen": 0.023562894901260734,
+      "train/kl_loss_r1": 0.014923631912097335,
+      "train/total_kl": 0.038486526999622586
+    },
+    {
+      "epoch": 3.6482115085536546,
+      "grad_norm": 1.4453125,
+      "learning_rate": 2.63387297633873e-06,
+      "loss": 0.2312,
+      "mean_token_accuracy": 0.983121731877327,
+      "step": 2930,
+      "train/kl_loss_qwen": 0.021840346092358232,
+      "train/kl_loss_r1": 0.012263339408673347,
+      "train/total_kl": 0.034103685058653355
+    },
+    {
+      "epoch": 3.6544323483670293,
+      "grad_norm": 0.87890625,
+      "learning_rate": 2.587173100871731e-06,
+      "loss": 0.2305,
+      "mean_token_accuracy": 0.9866573423147201,
+      "step": 2935,
+      "train/kl_loss_qwen": 0.021724556758999824,
+      "train/kl_loss_r1": 0.013378932792693376,
+      "train/total_kl": 0.03510348927229643
+    },
+    {
+      "epoch": 3.6606531881804045,
+      "grad_norm": 0.890625,
+      "learning_rate": 2.5404732254047325e-06,
+      "loss": 0.263,
+      "mean_token_accuracy": 0.9836262166500092,
+      "step": 2940,
+      "train/kl_loss_qwen": 0.025465904222801327,
+      "train/kl_loss_r1": 0.015349965426139534,
+      "train/total_kl": 0.04081586915999651
+    },
+    {
+      "epoch": 3.6668740279937793,
+      "grad_norm": 0.890625,
+      "learning_rate": 2.4937733499377336e-06,
+      "loss": 0.2345,
+      "mean_token_accuracy": 0.9856616973876953,
+      "step": 2945,
+      "train/kl_loss_qwen": 0.022425138484686614,
+      "train/kl_loss_r1": 0.01358982149977237,
+      "train/total_kl": 0.036014959961175916
+    },
+    {
+      "epoch": 3.673094867807154,
+      "grad_norm": 0.94921875,
+      "learning_rate": 2.447073474470735e-06,
+      "loss": 0.275,
+      "mean_token_accuracy": 0.9830880254507065,
+      "step": 2950,
+      "train/kl_loss_qwen": 0.02608721745200455,
+      "train/kl_loss_r1": 0.015567693463526665,
+      "train/total_kl": 0.04165491117164492
+    },
+    {
+      "epoch": 3.6793157076205287,
+      "grad_norm": 1.03125,
+      "learning_rate": 2.4003735990037362e-06,
+      "loss": 0.2202,
+      "mean_token_accuracy": 0.9819297671318055,
+      "step": 2955,
+      "train/kl_loss_qwen": 0.018545484030619265,
+      "train/kl_loss_r1": 0.011856895894743502,
+      "train/total_kl": 0.030402379855513573
+    },
+    {
+      "epoch": 3.6855365474339035,
+      "grad_norm": 0.86328125,
+      "learning_rate": 2.353673723536737e-06,
+      "loss": 0.2471,
+      "mean_token_accuracy": 0.9828756898641586,
+      "step": 2960,
+      "train/kl_loss_qwen": 0.022580026322975753,
+      "train/kl_loss_r1": 0.013090070732869209,
+      "train/total_kl": 0.03567009726539254
+    },
+    {
+      "epoch": 3.6917573872472786,
+      "grad_norm": 0.90625,
+      "learning_rate": 2.3069738480697384e-06,
+      "loss": 0.2389,
+      "mean_token_accuracy": 0.9842888534069061,
+      "step": 2965,
+      "train/kl_loss_qwen": 0.021884618513286113,
+      "train/kl_loss_r1": 0.014056896232068539,
+      "train/total_kl": 0.035941514931619165
+    },
+    {
+      "epoch": 3.697978227060653,
+      "grad_norm": 1.40625,
+      "learning_rate": 2.2602739726027396e-06,
+      "loss": 0.25,
+      "mean_token_accuracy": 0.9838718444108963,
+      "step": 2970,
+      "train/kl_loss_qwen": 0.023744882782921195,
+      "train/kl_loss_r1": 0.014738423796370626,
+      "train/total_kl": 0.038483306765556335
+    },
+    {
+      "epoch": 3.704199066874028,
+      "grad_norm": 1.0390625,
+      "learning_rate": 2.213574097135741e-06,
+      "loss": 0.2358,
+      "mean_token_accuracy": 0.9839303970336915,
+      "step": 2975,
+      "train/kl_loss_qwen": 0.02131864381954074,
+      "train/kl_loss_r1": 0.014065050520002842,
+      "train/total_kl": 0.03538369433954358
+    },
+    {
+      "epoch": 3.710419906687403,
+      "grad_norm": 0.80078125,
+      "learning_rate": 2.166874221668742e-06,
+      "loss": 0.2444,
+      "mean_token_accuracy": 0.9859850376844406,
+      "step": 2980,
+      "train/kl_loss_qwen": 0.023973260773345828,
+      "train/kl_loss_r1": 0.014935575192794204,
+      "train/total_kl": 0.03890883605927229
+    },
+    {
+      "epoch": 3.7166407465007776,
+      "grad_norm": 1.03125,
+      "learning_rate": 2.1201743462017437e-06,
+      "loss": 0.2339,
+      "mean_token_accuracy": 0.9822094053030014,
+      "step": 2985,
+      "train/kl_loss_qwen": 0.020026676869019867,
+      "train/kl_loss_r1": 0.012334840325638651,
+      "train/total_kl": 0.032361517008394
+    },
+    {
+      "epoch": 3.7228615863141523,
+      "grad_norm": 0.921875,
+      "learning_rate": 2.073474470734745e-06,
+      "loss": 0.22,
+      "mean_token_accuracy": 0.9863646864891052,
+      "step": 2990,
+      "train/kl_loss_qwen": 0.0210922843310982,
+      "train/kl_loss_r1": 0.012671014782972634,
+      "train/total_kl": 0.033763298857957126
+    },
+    {
+      "epoch": 3.729082426127527,
+      "grad_norm": 0.8828125,
+      "learning_rate": 2.0267745952677463e-06,
+      "loss": 0.2267,
+      "mean_token_accuracy": 0.9849774003028869,
+      "step": 2995,
+      "train/kl_loss_qwen": 0.021177000598981977,
+      "train/kl_loss_r1": 0.01177487033419311,
+      "train/total_kl": 0.03295187065377832
+    },
+    {
+      "epoch": 3.7353032659409022,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.9800747198007475e-06,
+      "loss": 0.2236,
+      "mean_token_accuracy": 0.9847380638122558,
+      "step": 3000,
+      "train/kl_loss_qwen": 0.019074604008346797,
+      "train/kl_loss_r1": 0.013245354662649334,
+      "train/total_kl": 0.03231995878741145
+    },
+    {
+      "epoch": 3.741524105754277,
+      "grad_norm": 0.8671875,
+      "learning_rate": 1.933374844333748e-06,
+      "loss": 0.3323,
+      "mean_token_accuracy": 0.9824711471796036,
+      "step": 3005,
+      "train/kl_loss_qwen": 0.03514354228973389,
+      "train/kl_loss_r1": 0.02021597446873784,
+      "train/total_kl": 0.05535951666533947
+    },
+    {
+      "epoch": 3.7477449455676517,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.8866749688667499e-06,
+      "loss": 0.2289,
+      "mean_token_accuracy": 0.9856206774711609,
+      "step": 3010,
+      "train/kl_loss_qwen": 0.021088129049167036,
+      "train/kl_loss_r1": 0.013338046660646797,
+      "train/total_kl": 0.03442617561668158
+    },
+    {
+      "epoch": 3.7539657853810264,
+      "grad_norm": 0.87890625,
+      "learning_rate": 1.839975093399751e-06,
+      "loss": 0.4054,
+      "mean_token_accuracy": 0.9761894792318344,
+      "step": 3015,
+      "train/kl_loss_qwen": 0.043745915545150636,
+      "train/kl_loss_r1": 0.021301075490191577,
+      "train/total_kl": 0.06504699122160673
+    },
+    {
+      "epoch": 3.760186625194401,
+      "grad_norm": 0.87109375,
+      "learning_rate": 1.7932752179327523e-06,
+      "loss": 0.2172,
+      "mean_token_accuracy": 0.9812994867563247,
+      "step": 3020,
+      "train/kl_loss_qwen": 0.018437253870069982,
+      "train/kl_loss_r1": 0.010920788859948516,
+      "train/total_kl": 0.02935804296284914
+    },
+    {
+      "epoch": 3.766407465007776,
+      "grad_norm": 1.0546875,
+      "learning_rate": 1.7465753424657534e-06,
+      "loss": 0.2311,
+      "mean_token_accuracy": 0.9857212871313095,
+      "step": 3025,
+      "train/kl_loss_qwen": 0.0210646481718868,
+      "train/kl_loss_r1": 0.014203881798312067,
+      "train/total_kl": 0.035268529783934355
+    },
+    {
+      "epoch": 3.7726283048211506,
+      "grad_norm": 0.84765625,
+      "learning_rate": 1.6998754669987547e-06,
+      "loss": 0.2289,
+      "mean_token_accuracy": 0.9842836320400238,
+      "step": 3030,
+      "train/kl_loss_qwen": 0.02159463716670871,
+      "train/kl_loss_r1": 0.012302979337982833,
+      "train/total_kl": 0.03389761624857783
+    },
+    {
+      "epoch": 3.778849144634526,
+      "grad_norm": 0.79296875,
+      "learning_rate": 1.6531755915317558e-06,
+      "loss": 0.2267,
+      "mean_token_accuracy": 0.9854977697134018,
+      "step": 3035,
+      "train/kl_loss_qwen": 0.020470092073082924,
+      "train/kl_loss_r1": 0.013319645705632866,
+      "train/total_kl": 0.03378973761573434
+    },
+    {
+      "epoch": 3.7850699844479005,
+      "grad_norm": 0.953125,
+      "learning_rate": 1.6064757160647571e-06,
+      "loss": 0.2509,
+      "mean_token_accuracy": 0.9862113922834397,
+      "step": 3040,
+      "train/kl_loss_qwen": 0.0245562803465873,
+      "train/kl_loss_r1": 0.015157862054184078,
+      "train/total_kl": 0.03971414268016815
+    },
+    {
+      "epoch": 3.7912908242612753,
+      "grad_norm": 0.82421875,
+      "learning_rate": 1.5597758405977585e-06,
+      "loss": 0.2228,
+      "mean_token_accuracy": 0.9858496934175491,
+      "step": 3045,
+      "train/kl_loss_qwen": 0.020629500132054092,
+      "train/kl_loss_r1": 0.013019497063942254,
+      "train/total_kl": 0.03364899717271328
+    },
+    {
+      "epoch": 3.79751166407465,
+      "grad_norm": 1.0625,
+      "learning_rate": 1.5130759651307598e-06,
+      "loss": 0.2332,
+      "mean_token_accuracy": 0.9846426278352738,
+      "step": 3050,
+      "train/kl_loss_qwen": 0.021104606799781322,
+      "train/kl_loss_r1": 0.013108310452662409,
+      "train/total_kl": 0.0342129172757268
+    },
+    {
+      "epoch": 3.8037325038880248,
+      "grad_norm": 0.875,
+      "learning_rate": 1.4663760896637609e-06,
+      "loss": 0.2584,
+      "mean_token_accuracy": 0.9858539253473282,
+      "step": 3055,
+      "train/kl_loss_qwen": 0.025427423184737563,
+      "train/kl_loss_r1": 0.016527001629583538,
+      "train/total_kl": 0.04195442469790578
+    },
+    {
+      "epoch": 3.8099533437014,
+      "grad_norm": 0.95703125,
+      "learning_rate": 1.4196762141967622e-06,
+      "loss": 0.2181,
+      "mean_token_accuracy": 0.9840226501226426,
+      "step": 3060,
+      "train/kl_loss_qwen": 0.01866429247893393,
+      "train/kl_loss_r1": 0.012438310799188912,
+      "train/total_kl": 0.03110260358080268
+    },
+    {
+      "epoch": 3.816174183514774,
+      "grad_norm": 1.0234375,
+      "learning_rate": 1.3729763387297635e-06,
+      "loss": 0.2452,
+      "mean_token_accuracy": 0.9825713008642196,
+      "step": 3065,
+      "train/kl_loss_qwen": 0.02215881203301251,
+      "train/kl_loss_r1": 0.013574238121509551,
+      "train/total_kl": 0.035733049549162385
+    },
+    {
+      "epoch": 3.8223950233281494,
+      "grad_norm": 0.92578125,
+      "learning_rate": 1.3262764632627648e-06,
+      "loss": 0.2237,
+      "mean_token_accuracy": 0.9860330015420914,
+      "step": 3070,
+      "train/kl_loss_qwen": 0.020841342536732553,
+      "train/kl_loss_r1": 0.012603346444666385,
+      "train/total_kl": 0.03344468884170056
+    },
+    {
+      "epoch": 3.828615863141524,
+      "grad_norm": 0.8828125,
+      "learning_rate": 1.2795765877957657e-06,
+      "loss": 0.2413,
+      "mean_token_accuracy": 0.9861990302801132,
+      "step": 3075,
+      "train/kl_loss_qwen": 0.023467224929481743,
+      "train/kl_loss_r1": 0.014058938180096448,
+      "train/total_kl": 0.03752616299316287
+    },
+    {
+      "epoch": 3.834836702954899,
+      "grad_norm": 0.85546875,
+      "learning_rate": 1.232876712328767e-06,
+      "loss": 0.2327,
+      "mean_token_accuracy": 0.9857694715261459,
+      "step": 3080,
+      "train/kl_loss_qwen": 0.021740892436355352,
+      "train/kl_loss_r1": 0.013577925111167132,
+      "train/total_kl": 0.03531881738454103
+    },
+    {
+      "epoch": 3.8410575427682736,
+      "grad_norm": 1.03125,
+      "learning_rate": 1.1861768368617684e-06,
+      "loss": 0.2356,
+      "mean_token_accuracy": 0.9867552161216736,
+      "step": 3085,
+      "train/kl_loss_qwen": 0.021518895542249082,
+      "train/kl_loss_r1": 0.014279638021253049,
+      "train/total_kl": 0.03579853363335132
+    },
+    {
+      "epoch": 3.8472783825816483,
+      "grad_norm": 1.140625,
+      "learning_rate": 1.1394769613947697e-06,
+      "loss": 0.2505,
+      "mean_token_accuracy": 0.9849425464868545,
+      "step": 3090,
+      "train/kl_loss_qwen": 0.023463033000007273,
+      "train/kl_loss_r1": 0.015782216051593422,
+      "train/total_kl": 0.03924524867907166
+    },
+    {
+      "epoch": 3.8534992223950235,
+      "grad_norm": 0.82421875,
+      "learning_rate": 1.0927770859277708e-06,
+      "loss": 0.4111,
+      "mean_token_accuracy": 0.9756690293550492,
+      "step": 3095,
+      "train/kl_loss_qwen": 0.04434741260483861,
+      "train/kl_loss_r1": 0.02056443770416081,
+      "train/total_kl": 0.0649118491448462
+    },
+    {
+      "epoch": 3.8597200622083983,
+      "grad_norm": 0.9140625,
+      "learning_rate": 1.046077210460772e-06,
+      "loss": 0.2567,
+      "mean_token_accuracy": 0.9853157699108124,
+      "step": 3100,
+      "train/kl_loss_qwen": 0.026550929993391037,
+      "train/kl_loss_r1": 0.015549480146728457,
+      "train/total_kl": 0.04210041025653481
+    },
+    {
+      "epoch": 3.865940902021773,
+      "grad_norm": 0.98828125,
+      "learning_rate": 9.993773349937734e-07,
+      "loss": 0.25,
+      "mean_token_accuracy": 0.9842557400465012,
+      "step": 3105,
+      "train/kl_loss_qwen": 0.022726377053186298,
+      "train/kl_loss_r1": 0.01626164559274912,
+      "train/total_kl": 0.038988023158162834
+    },
+    {
+      "epoch": 3.8721617418351477,
+      "grad_norm": 0.7734375,
+      "learning_rate": 9.526774595267746e-07,
+      "loss": 0.2306,
+      "mean_token_accuracy": 0.9841700613498687,
+      "step": 3110,
+      "train/kl_loss_qwen": 0.02183036059141159,
+      "train/kl_loss_r1": 0.012751943548209965,
+      "train/total_kl": 0.034582304395735265
+    },
+    {
+      "epoch": 3.8783825816485225,
+      "grad_norm": 0.9453125,
+      "learning_rate": 9.059775840597758e-07,
+      "loss": 0.2701,
+      "mean_token_accuracy": 0.984642618894577,
+      "step": 3115,
+      "train/kl_loss_qwen": 0.02554837050847709,
+      "train/kl_loss_r1": 0.016179563431069254,
+      "train/total_kl": 0.041727933939546345
+    },
+    {
+      "epoch": 3.8846034214618976,
+      "grad_norm": 0.98828125,
+      "learning_rate": 8.592777085927771e-07,
+      "loss": 0.219,
+      "mean_token_accuracy": 0.9848039954900741,
+      "step": 3120,
+      "train/kl_loss_qwen": 0.018509938986971973,
+      "train/kl_loss_r1": 0.013476863643154502,
+      "train/total_kl": 0.03198680253699422
+    },
+    {
+      "epoch": 3.890824261275272,
+      "grad_norm": 0.828125,
+      "learning_rate": 8.125778331257784e-07,
+      "loss": 0.3768,
+      "mean_token_accuracy": 0.9785630255937576,
+      "step": 3125,
+      "train/kl_loss_qwen": 0.03686713627539575,
+      "train/kl_loss_r1": 0.024776964401826262,
+      "train/total_kl": 0.06164409937337041
+    },
+    {
+      "epoch": 3.897045101088647,
+      "grad_norm": 0.99609375,
+      "learning_rate": 7.658779576587797e-07,
+      "loss": 0.2351,
+      "mean_token_accuracy": 0.9840314447879791,
+      "step": 3130,
+      "train/kl_loss_qwen": 0.022078597731888295,
+      "train/kl_loss_r1": 0.013947232184000314,
+      "train/total_kl": 0.03602582989260554
+    },
+    {
+      "epoch": 3.903265940902022,
+      "grad_norm": 0.9296875,
+      "learning_rate": 7.191780821917808e-07,
+      "loss": 0.232,
+      "mean_token_accuracy": 0.9858786344528199,
+      "step": 3135,
+      "train/kl_loss_qwen": 0.02128533157519996,
+      "train/kl_loss_r1": 0.01425166679546237,
+      "train/total_kl": 0.0355369983240962
+    },
+    {
+      "epoch": 3.9094867807153966,
+      "grad_norm": 0.92578125,
+      "learning_rate": 6.724782067247821e-07,
+      "loss": 0.2524,
+      "mean_token_accuracy": 0.9848507881164551,
+      "step": 3140,
+      "train/kl_loss_qwen": 0.02394880629144609,
+      "train/kl_loss_r1": 0.015210675774142146,
+      "train/total_kl": 0.03915948234498501
+    },
+    {
+      "epoch": 3.9157076205287713,
+      "grad_norm": 0.78125,
+      "learning_rate": 6.257783312577833e-07,
+      "loss": 0.2599,
+      "mean_token_accuracy": 0.9854637980461121,
+      "step": 3145,
+      "train/kl_loss_qwen": 0.02829183344729245,
+      "train/kl_loss_r1": 0.013507619872689247,
+      "train/total_kl": 0.04179945318028331
+    },
+    {
+      "epoch": 3.921928460342146,
+      "grad_norm": 0.75,
+      "learning_rate": 5.790784557907846e-07,
+      "loss": 0.2671,
+      "mean_token_accuracy": 0.9863925516605377,
+      "step": 3150,
+      "train/kl_loss_qwen": 0.027312430460005997,
+      "train/kl_loss_r1": 0.01701681511476636,
+      "train/total_kl": 0.044329245761036876
+    },
+    {
+      "epoch": 3.928149300155521,
+      "grad_norm": 0.8203125,
+      "learning_rate": 5.323785803237859e-07,
+      "loss": 0.3209,
+      "mean_token_accuracy": 0.97878338098526,
+      "step": 3155,
+      "train/kl_loss_qwen": 0.03283619345165789,
+      "train/kl_loss_r1": 0.017961669899523258,
+      "train/total_kl": 0.05079786339774728
+    },
+    {
+      "epoch": 3.934370139968896,
+      "grad_norm": 1.15625,
+      "learning_rate": 4.856787048567871e-07,
+      "loss": 0.2415,
+      "mean_token_accuracy": 0.9832513570785523,
+      "step": 3160,
+      "train/kl_loss_qwen": 0.02150732106529176,
+      "train/kl_loss_r1": 0.014138684188947082,
+      "train/total_kl": 0.035646005533635616
+    },
+    {
+      "epoch": 3.9405909797822707,
+      "grad_norm": 0.78515625,
+      "learning_rate": 4.389788293897883e-07,
+      "loss": 0.3302,
+      "mean_token_accuracy": 0.9821478694677352,
+      "step": 3165,
+      "train/kl_loss_qwen": 0.035413914173841474,
+      "train/kl_loss_r1": 0.019884266471490265,
+      "train/total_kl": 0.05529817976057529
+    },
+    {
+      "epoch": 3.9468118195956454,
+      "grad_norm": 0.8046875,
+      "learning_rate": 3.9227895392278954e-07,
+      "loss": 0.2315,
+      "mean_token_accuracy": 0.9847661107778549,
+      "step": 3170,
+      "train/kl_loss_qwen": 0.021044514747336505,
+      "train/kl_loss_r1": 0.01309770722873509,
+      "train/total_kl": 0.03414222216233611
+    },
+    {
+      "epoch": 3.95303265940902,
+      "grad_norm": 1.0859375,
+      "learning_rate": 3.4557907845579075e-07,
+      "loss": 0.3341,
+      "mean_token_accuracy": 0.9789846986532211,
+      "step": 3175,
+      "train/kl_loss_qwen": 0.03371955840848386,
+      "train/kl_loss_r1": 0.017813174077309667,
+      "train/total_kl": 0.05153273269534111
+    },
+    {
+      "epoch": 3.959253499222395,
+      "grad_norm": 0.9296875,
+      "learning_rate": 2.98879202988792e-07,
+      "loss": 0.2994,
+      "mean_token_accuracy": 0.9830943882465363,
+      "step": 3180,
+      "train/kl_loss_qwen": 0.033274567080661656,
+      "train/kl_loss_r1": 0.015842707734555005,
+      "train/total_kl": 0.04911727402359247
+    },
+    {
+      "epoch": 3.9654743390357696,
+      "grad_norm": 1.265625,
+      "learning_rate": 2.521793275217933e-07,
+      "loss": 0.2351,
+      "mean_token_accuracy": 0.984268057346344,
+      "step": 3185,
+      "train/kl_loss_qwen": 0.021436449000611903,
+      "train/kl_loss_r1": 0.014701427333056927,
+      "train/total_kl": 0.03613787675276399
+    },
+    {
+      "epoch": 3.971695178849145,
+      "grad_norm": 0.85546875,
+      "learning_rate": 2.0547945205479452e-07,
+      "loss": 0.2427,
+      "mean_token_accuracy": 0.9866203486919403,
+      "step": 3190,
+      "train/kl_loss_qwen": 0.02276371791958809,
+      "train/kl_loss_r1": 0.015056712529622018,
+      "train/total_kl": 0.03782043047249317
+    },
+    {
+      "epoch": 3.9779160186625195,
+      "grad_norm": 0.75390625,
+      "learning_rate": 1.5877957658779578e-07,
+      "loss": 0.2355,
+      "mean_token_accuracy": 0.9860924631357193,
+      "step": 3195,
+      "train/kl_loss_qwen": 0.022292080451734364,
+      "train/kl_loss_r1": 0.01344187636859715,
+      "train/total_kl": 0.035733957309275866
+    },
+    {
+      "epoch": 3.9841368584758943,
+      "grad_norm": 0.8515625,
+      "learning_rate": 1.1207970112079702e-07,
+      "loss": 0.2508,
+      "mean_token_accuracy": 0.985159307718277,
+      "step": 3200,
+      "train/kl_loss_qwen": 0.02407874008640647,
+      "train/kl_loss_r1": 0.015484006190672518,
+      "train/total_kl": 0.039562746044248344
+    },
+    {
+      "epoch": 3.990357698289269,
+      "grad_norm": 0.80078125,
+      "learning_rate": 6.537982565379826e-08,
+      "loss": 0.2122,
+      "mean_token_accuracy": 0.9835577666759491,
+      "step": 3205,
+      "train/kl_loss_qwen": 0.018145430739969014,
+      "train/kl_loss_r1": 0.011138362321071326,
+      "train/total_kl": 0.029283793363720178
+    },
+    {
+      "epoch": 3.9965785381026437,
+      "grad_norm": 0.90234375,
+      "learning_rate": 1.86799501867995e-08,
+      "loss": 0.2528,
+      "mean_token_accuracy": 0.9819873452186585,
+      "step": 3210,
+      "train/kl_loss_qwen": 0.023181884456425904,
+      "train/kl_loss_r1": 0.01377418297342956,
+      "train/total_kl": 0.03695606719702482
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 3212,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.0794727176859484e+18,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}