| { |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 1.9954853273137698, |
| "eval_steps": 500, |
| "global_step": 442, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.05, |
| "grad_norm": 3.34375, |
| "learning_rate": 4.9936877763590664e-06, |
| "log_odds_chosen": 0.2890412509441376, |
| "log_odds_ratio": -0.6332851648330688, |
| "logits/chosen": -3.1468911170959473, |
| "logits/rejected": -3.138887643814087, |
| "logps/chosen": -0.8719667196273804, |
| "logps/rejected": -1.0382100343704224, |
| "loss": 0.6482, |
| "nll_loss": 0.5332359075546265, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.17439337074756622, |
| "rewards/margins": 0.033248625695705414, |
| "rewards/rejected": -0.20764198899269104, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.09, |
| "grad_norm": 3.046875, |
| "learning_rate": 4.9747829807701e-06, |
| "log_odds_chosen": 0.3164082169532776, |
| "log_odds_ratio": -0.6052478551864624, |
| "logits/chosen": -3.0846054553985596, |
| "logits/rejected": -3.0846097469329834, |
| "logps/chosen": -0.8607593774795532, |
| "logps/rejected": -1.0491193532943726, |
| "loss": 0.6131, |
| "nll_loss": 0.4854271411895752, |
| "rewards/accuracies": 0.637499988079071, |
| "rewards/chosen": -0.17215189337730408, |
| "rewards/margins": 0.03767203167080879, |
| "rewards/rejected": -0.20982392132282257, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.14, |
| "grad_norm": 3.671875, |
| "learning_rate": 4.943381078271214e-06, |
| "log_odds_chosen": 0.3783247768878937, |
| "log_odds_ratio": -0.6129769086837769, |
| "logits/chosen": -3.084336996078491, |
| "logits/rejected": -3.0786914825439453, |
| "logps/chosen": -0.8424398303031921, |
| "logps/rejected": -1.0533756017684937, |
| "loss": 0.602, |
| "nll_loss": 0.4758261740207672, |
| "rewards/accuracies": 0.675000011920929, |
| "rewards/chosen": -0.16848795115947723, |
| "rewards/margins": 0.0421871617436409, |
| "rewards/rejected": -0.21067512035369873, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.18, |
| "grad_norm": 3.0, |
| "learning_rate": 4.89964064152747e-06, |
| "log_odds_chosen": 0.3220486640930176, |
| "log_odds_ratio": -0.6127817630767822, |
| "logits/chosen": -3.073997974395752, |
| "logits/rejected": -3.069819688796997, |
| "logps/chosen": -0.8541064262390137, |
| "logps/rejected": -1.0315220355987549, |
| "loss": 0.5993, |
| "nll_loss": 0.47563648223876953, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.17082129418849945, |
| "rewards/margins": 0.03548307344317436, |
| "rewards/rejected": -0.2063043862581253, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.23, |
| "grad_norm": 2.90625, |
| "learning_rate": 4.84378255007397e-06, |
| "log_odds_chosen": 0.3029581606388092, |
| "log_odds_ratio": -0.6211820840835571, |
| "logits/chosen": -3.0848793983459473, |
| "logits/rejected": -3.0841445922851562, |
| "logps/chosen": -0.8334375619888306, |
| "logps/rejected": -1.0042452812194824, |
| "loss": 0.5913, |
| "nll_loss": 0.4705049991607666, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.1666875034570694, |
| "rewards/margins": 0.03416154906153679, |
| "rewards/rejected": -0.20084905624389648, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.27, |
| "grad_norm": 3.015625, |
| "learning_rate": 4.7760888749230414e-06, |
| "log_odds_chosen": 0.4831446707248688, |
| "log_odds_ratio": -0.5493755340576172, |
| "logits/chosen": -3.095670461654663, |
| "logits/rejected": -3.0957283973693848, |
| "logps/chosen": -0.8023236989974976, |
| "logps/rejected": -1.0722805261611938, |
| "loss": 0.5964, |
| "nll_loss": 0.4757974147796631, |
| "rewards/accuracies": 0.706250011920929, |
| "rewards/chosen": -0.16046474874019623, |
| "rewards/margins": 0.05399137735366821, |
| "rewards/rejected": -0.21445612609386444, |
| "step": 60 |
| }, |
| { |
| "epoch": 0.32, |
| "grad_norm": 3.09375, |
| "learning_rate": 4.696901454167989e-06, |
| "log_odds_chosen": 0.3349444270133972, |
| "log_odds_ratio": -0.6006864309310913, |
| "logits/chosen": -3.0792784690856934, |
| "logits/rejected": -3.073970079421997, |
| "logps/chosen": -0.8161932229995728, |
| "logps/rejected": -0.9950379133224487, |
| "loss": 0.6022, |
| "nll_loss": 0.48054131865501404, |
| "rewards/accuracies": 0.6812499761581421, |
| "rewards/chosen": -0.16323864459991455, |
| "rewards/margins": 0.035768941044807434, |
| "rewards/rejected": -0.19900760054588318, |
| "step": 70 |
| }, |
| { |
| "epoch": 0.36, |
| "grad_norm": 3.171875, |
| "learning_rate": 4.6066201667762944e-06, |
| "log_odds_chosen": 0.46497654914855957, |
| "log_odds_ratio": -0.5399398803710938, |
| "logits/chosen": -3.1036829948425293, |
| "logits/rejected": -3.099396228790283, |
| "logps/chosen": -0.8102518916130066, |
| "logps/rejected": -1.0698442459106445, |
| "loss": 0.5611, |
| "nll_loss": 0.47170430421829224, |
| "rewards/accuracies": 0.7437499761581421, |
| "rewards/chosen": -0.16205039620399475, |
| "rewards/margins": 0.05191843584179878, |
| "rewards/rejected": -0.21396884322166443, |
| "step": 80 |
| }, |
| { |
| "epoch": 0.41, |
| "grad_norm": 3.234375, |
| "learning_rate": 4.505700913289246e-06, |
| "log_odds_chosen": 0.385485976934433, |
| "log_odds_ratio": -0.5975229144096375, |
| "logits/chosen": -3.096364974975586, |
| "logits/rejected": -3.085583209991455, |
| "logps/chosen": -0.8479864001274109, |
| "logps/rejected": -1.076240062713623, |
| "loss": 0.6009, |
| "nll_loss": 0.47817081212997437, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.1695972979068756, |
| "rewards/margins": 0.04565075412392616, |
| "rewards/rejected": -0.21524803340435028, |
| "step": 90 |
| }, |
| { |
| "epoch": 0.45, |
| "grad_norm": 2.890625, |
| "learning_rate": 4.3946533136249926e-06, |
| "log_odds_chosen": 0.43778282403945923, |
| "log_odds_ratio": -0.5804494619369507, |
| "logits/chosen": -3.0998694896698, |
| "logits/rejected": -3.0987915992736816, |
| "logps/chosen": -0.8241409063339233, |
| "logps/rejected": -1.0574108362197876, |
| "loss": 0.5988, |
| "nll_loss": 0.4965757429599762, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.16482816636562347, |
| "rewards/margins": 0.046653974801301956, |
| "rewards/rejected": -0.21148213744163513, |
| "step": 100 |
| }, |
| { |
| "epoch": 0.5, |
| "grad_norm": 3.203125, |
| "learning_rate": 4.274038133610629e-06, |
| "log_odds_chosen": 0.46187520027160645, |
| "log_odds_ratio": -0.5494967103004456, |
| "logits/chosen": -3.1161484718322754, |
| "logits/rejected": -3.114893913269043, |
| "logps/chosen": -0.7468121647834778, |
| "logps/rejected": -0.9928558468818665, |
| "loss": 0.5811, |
| "nll_loss": 0.4424869120121002, |
| "rewards/accuracies": 0.7562500238418579, |
| "rewards/chosen": -0.14936241507530212, |
| "rewards/margins": 0.04920873045921326, |
| "rewards/rejected": -0.19857117533683777, |
| "step": 110 |
| }, |
| { |
| "epoch": 0.54, |
| "grad_norm": 3.375, |
| "learning_rate": 4.1444644532387485e-06, |
| "log_odds_chosen": 0.4036913812160492, |
| "log_odds_ratio": -0.6148982048034668, |
| "logits/chosen": -3.0943081378936768, |
| "logits/rejected": -3.089369297027588, |
| "logps/chosen": -0.8185051083564758, |
| "logps/rejected": -1.0283105373382568, |
| "loss": 0.5743, |
| "nll_loss": 0.4471573233604431, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.16370102763175964, |
| "rewards/margins": 0.04196108505129814, |
| "rewards/rejected": -0.2056621015071869, |
| "step": 120 |
| }, |
| { |
| "epoch": 0.59, |
| "grad_norm": 3.109375, |
| "learning_rate": 4.006586590948141e-06, |
| "log_odds_chosen": 0.38943806290626526, |
| "log_odds_ratio": -0.5949686169624329, |
| "logits/chosen": -3.105708360671997, |
| "logits/rejected": -3.1006293296813965, |
| "logps/chosen": -0.8150092363357544, |
| "logps/rejected": -1.0240681171417236, |
| "loss": 0.5787, |
| "nll_loss": 0.46157145500183105, |
| "rewards/accuracies": 0.6875, |
| "rewards/chosen": -0.16300183534622192, |
| "rewards/margins": 0.041811779141426086, |
| "rewards/rejected": -0.20481359958648682, |
| "step": 130 |
| }, |
| { |
| "epoch": 0.63, |
| "grad_norm": 3.109375, |
| "learning_rate": 3.861100799460336e-06, |
| "log_odds_chosen": 0.5897554159164429, |
| "log_odds_ratio": -0.542380690574646, |
| "logits/chosen": -3.100862503051758, |
| "logits/rejected": -3.0946009159088135, |
| "logps/chosen": -0.7791546583175659, |
| "logps/rejected": -1.0703848600387573, |
| "loss": 0.5747, |
| "nll_loss": 0.5026167631149292, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.15583094954490662, |
| "rewards/margins": 0.05824603885412216, |
| "rewards/rejected": -0.214076966047287, |
| "step": 140 |
| }, |
| { |
| "epoch": 0.68, |
| "grad_norm": 3.03125, |
| "learning_rate": 3.7087417498572946e-06, |
| "log_odds_chosen": 0.4274469316005707, |
| "log_odds_ratio": -0.5712376236915588, |
| "logits/chosen": -3.102602005004883, |
| "logits/rejected": -3.1062116622924805, |
| "logps/chosen": -0.8305424451828003, |
| "logps/rejected": -1.077083945274353, |
| "loss": 0.5692, |
| "nll_loss": 0.4742380678653717, |
| "rewards/accuracies": 0.7250000238418579, |
| "rewards/chosen": -0.16610850393772125, |
| "rewards/margins": 0.04930829256772995, |
| "rewards/rejected": -0.2154167890548706, |
| "step": 150 |
| }, |
| { |
| "epoch": 0.72, |
| "grad_norm": 3.28125, |
| "learning_rate": 3.550278821654866e-06, |
| "log_odds_chosen": 0.34345707297325134, |
| "log_odds_ratio": -0.6129624247550964, |
| "logits/chosen": -3.095494508743286, |
| "logits/rejected": -3.0863471031188965, |
| "logps/chosen": -0.8034600019454956, |
| "logps/rejected": -0.9921743273735046, |
| "loss": 0.587, |
| "nll_loss": 0.4692765176296234, |
| "rewards/accuracies": 0.699999988079071, |
| "rewards/chosen": -0.1606920212507248, |
| "rewards/margins": 0.037742845714092255, |
| "rewards/rejected": -0.19843485951423645, |
| "step": 160 |
| }, |
| { |
| "epoch": 0.77, |
| "grad_norm": 4.28125, |
| "learning_rate": 3.386512217606339e-06, |
| "log_odds_chosen": 0.3175373375415802, |
| "log_odds_ratio": -0.6278958320617676, |
| "logits/chosen": -3.1244797706604004, |
| "logits/rejected": -3.1204514503479004, |
| "logps/chosen": -0.7932506799697876, |
| "logps/rejected": -0.9527732133865356, |
| "loss": 0.6102, |
| "nll_loss": 0.472064733505249, |
| "rewards/accuracies": 0.6499999761581421, |
| "rewards/chosen": -0.15865013003349304, |
| "rewards/margins": 0.031904518604278564, |
| "rewards/rejected": -0.1905546486377716, |
| "step": 170 |
| }, |
| { |
| "epoch": 0.81, |
| "grad_norm": 3.765625, |
| "learning_rate": 3.218268922855452e-06, |
| "log_odds_chosen": 0.42049679160118103, |
| "log_odds_ratio": -0.5808514356613159, |
| "logits/chosen": -3.1028733253479004, |
| "logits/rejected": -3.0980591773986816, |
| "logps/chosen": -0.7747179269790649, |
| "logps/rejected": -0.9938864707946777, |
| "loss": 0.5825, |
| "nll_loss": 0.44331270456314087, |
| "rewards/accuracies": 0.6937500238418579, |
| "rewards/chosen": -0.154943585395813, |
| "rewards/margins": 0.04383372142910957, |
| "rewards/rejected": -0.19877730309963226, |
| "step": 180 |
| }, |
| { |
| "epoch": 0.86, |
| "grad_norm": 3.328125, |
| "learning_rate": 3.046398528844248e-06, |
| "log_odds_chosen": 0.5076274275779724, |
| "log_odds_ratio": -0.5742210745811462, |
| "logits/chosen": -3.104665517807007, |
| "logits/rejected": -3.095426082611084, |
| "logps/chosen": -0.776441216468811, |
| "logps/rejected": -1.0134727954864502, |
| "loss": 0.5712, |
| "nll_loss": 0.4288255274295807, |
| "rewards/accuracies": 0.7124999761581421, |
| "rewards/chosen": -0.15528824925422668, |
| "rewards/margins": 0.04740630090236664, |
| "rewards/rejected": -0.20269456505775452, |
| "step": 190 |
| }, |
| { |
| "epoch": 0.9, |
| "grad_norm": 2.953125, |
| "learning_rate": 2.871768943064129e-06, |
| "log_odds_chosen": 0.3015575408935547, |
| "log_odds_ratio": -0.6296088695526123, |
| "logits/chosen": -3.0799665451049805, |
| "logits/rejected": -3.07787823677063, |
| "logps/chosen": -0.8669988512992859, |
| "logps/rejected": -1.0326287746429443, |
| "loss": 0.5744, |
| "nll_loss": 0.47870683670043945, |
| "rewards/accuracies": 0.643750011920929, |
| "rewards/chosen": -0.17339977622032166, |
| "rewards/margins": 0.033125996589660645, |
| "rewards/rejected": -0.2065257579088211, |
| "step": 200 |
| }, |
| { |
| "epoch": 0.95, |
| "grad_norm": 2.859375, |
| "learning_rate": 2.695262006314912e-06, |
| "log_odds_chosen": 0.3043849766254425, |
| "log_odds_ratio": -0.6296250224113464, |
| "logits/chosen": -3.0792198181152344, |
| "logits/rejected": -3.0685715675354004, |
| "logps/chosen": -0.8420785069465637, |
| "logps/rejected": -1.0109227895736694, |
| "loss": 0.5851, |
| "nll_loss": 0.49167829751968384, |
| "rewards/accuracies": 0.65625, |
| "rewards/chosen": -0.16841569542884827, |
| "rewards/margins": 0.03376886993646622, |
| "rewards/rejected": -0.20218458771705627, |
| "step": 210 |
| }, |
| { |
| "epoch": 0.99, |
| "grad_norm": 3.390625, |
| "learning_rate": 2.517769039603744e-06, |
| "log_odds_chosen": 0.47437596321105957, |
| "log_odds_ratio": -0.5693033933639526, |
| "logits/chosen": -3.0923125743865967, |
| "logits/rejected": -3.08634614944458, |
| "logps/chosen": -0.7851861119270325, |
| "logps/rejected": -1.0160605907440186, |
| "loss": 0.6031, |
| "nll_loss": 0.4828670620918274, |
| "rewards/accuracies": 0.71875, |
| "rewards/chosen": -0.15703722834587097, |
| "rewards/margins": 0.04617489501833916, |
| "rewards/rejected": -0.20321211218833923, |
| "step": 220 |
| }, |
| { |
| "epoch": 1.04, |
| "grad_norm": 5.125, |
| "learning_rate": 2.3401863431710864e-06, |
| "log_odds_chosen": 0.8126243352890015, |
| "log_odds_ratio": -0.4562348425388336, |
| "logits/chosen": -3.0906317234039307, |
| "logits/rejected": -3.08894681930542, |
| "logps/chosen": -0.6749258041381836, |
| "logps/rejected": -1.0676677227020264, |
| "loss": 0.5063, |
| "nll_loss": 0.4003027081489563, |
| "rewards/accuracies": 0.78125, |
| "rewards/chosen": -0.13498517870903015, |
| "rewards/margins": 0.0785483792424202, |
| "rewards/rejected": -0.21353355050086975, |
| "step": 230 |
| }, |
| { |
| "epoch": 1.08, |
| "grad_norm": 3.34375, |
| "learning_rate": 2.163410670372652e-06, |
| "log_odds_chosen": 0.8564330339431763, |
| "log_odds_ratio": -0.4268684387207031, |
| "logits/chosen": -3.0698952674865723, |
| "logits/rejected": -3.070732831954956, |
| "logps/chosen": -0.7170256972312927, |
| "logps/rejected": -1.1589481830596924, |
| "loss": 0.4975, |
| "nll_loss": 0.45029813051223755, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.14340513944625854, |
| "rewards/margins": 0.08838452398777008, |
| "rewards/rejected": -0.23178966343402863, |
| "step": 240 |
| }, |
| { |
| "epoch": 1.13, |
| "grad_norm": 3.34375, |
| "learning_rate": 1.9883346992732256e-06, |
| "log_odds_chosen": 1.0222368240356445, |
| "log_odds_ratio": -0.36799290776252747, |
| "logits/chosen": -3.055107593536377, |
| "logits/rejected": -3.045011281967163, |
| "logps/chosen": -0.6467097997665405, |
| "logps/rejected": -1.2033334970474243, |
| "loss": 0.4936, |
| "nll_loss": 0.3834058940410614, |
| "rewards/accuracies": 0.9125000238418579, |
| "rewards/chosen": -0.1293419599533081, |
| "rewards/margins": 0.1113247275352478, |
| "rewards/rejected": -0.24066667258739471, |
| "step": 250 |
| }, |
| { |
| "epoch": 1.17, |
| "grad_norm": 2.921875, |
| "learning_rate": 1.8158425248197931e-06, |
| "log_odds_chosen": 0.9676607251167297, |
| "log_odds_ratio": -0.3842445909976959, |
| "logits/chosen": -3.078850269317627, |
| "logits/rejected": -3.0772275924682617, |
| "logps/chosen": -0.6771007180213928, |
| "logps/rejected": -1.177585482597351, |
| "loss": 0.5028, |
| "nll_loss": 0.44024211168289185, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.13542014360427856, |
| "rewards/margins": 0.10009696334600449, |
| "rewards/rejected": -0.23551711440086365, |
| "step": 260 |
| }, |
| { |
| "epoch": 1.22, |
| "grad_norm": 3.1875, |
| "learning_rate": 1.6468051943575242e-06, |
| "log_odds_chosen": 0.9456725120544434, |
| "log_odds_ratio": -0.39273351430892944, |
| "logits/chosen": -3.085444450378418, |
| "logits/rejected": -3.08333420753479, |
| "logps/chosen": -0.6596750020980835, |
| "logps/rejected": -1.1495213508605957, |
| "loss": 0.4758, |
| "nll_loss": 0.3547888398170471, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.1319349855184555, |
| "rewards/margins": 0.09796925634145737, |
| "rewards/rejected": -0.22990426421165466, |
| "step": 270 |
| }, |
| { |
| "epoch": 1.26, |
| "grad_norm": 3.328125, |
| "learning_rate": 1.482076309033254e-06, |
| "log_odds_chosen": 0.8887540102005005, |
| "log_odds_ratio": -0.43248239159584045, |
| "logits/chosen": -3.0775530338287354, |
| "logits/rejected": -3.0756869316101074, |
| "logps/chosen": -0.6567140817642212, |
| "logps/rejected": -1.1050218343734741, |
| "loss": 0.4872, |
| "nll_loss": 0.3975124657154083, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.1313427984714508, |
| "rewards/margins": 0.08966155350208282, |
| "rewards/rejected": -0.22100436687469482, |
| "step": 280 |
| }, |
| { |
| "epoch": 1.31, |
| "grad_norm": 2.96875, |
| "learning_rate": 1.3224877132984131e-06, |
| "log_odds_chosen": 1.054574728012085, |
| "log_odds_ratio": -0.3588215708732605, |
| "logits/chosen": -3.103698253631592, |
| "logits/rejected": -3.107422351837158, |
| "logps/chosen": -0.6697841882705688, |
| "logps/rejected": -1.2304431200027466, |
| "loss": 0.4864, |
| "nll_loss": 0.4196816384792328, |
| "rewards/accuracies": 0.918749988079071, |
| "rewards/chosen": -0.13395683467388153, |
| "rewards/margins": 0.11213177442550659, |
| "rewards/rejected": -0.24608862400054932, |
| "step": 290 |
| }, |
| { |
| "epoch": 1.35, |
| "grad_norm": 3.109375, |
| "learning_rate": 1.1688452942784592e-06, |
| "log_odds_chosen": 1.018751621246338, |
| "log_odds_ratio": -0.3791121542453766, |
| "logits/chosen": -3.0898470878601074, |
| "logits/rejected": -3.0825257301330566, |
| "logps/chosen": -0.6466778516769409, |
| "logps/rejected": -1.1818552017211914, |
| "loss": 0.4914, |
| "nll_loss": 0.3884134292602539, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.12933556735515594, |
| "rewards/margins": 0.10703550279140472, |
| "rewards/rejected": -0.23637107014656067, |
| "step": 300 |
| }, |
| { |
| "epoch": 1.4, |
| "grad_norm": 2.984375, |
| "learning_rate": 1.021924912221062e-06, |
| "log_odds_chosen": 0.8282191157341003, |
| "log_odds_ratio": -0.42912721633911133, |
| "logits/chosen": -3.1064085960388184, |
| "logits/rejected": -3.105886697769165, |
| "logps/chosen": -0.669070839881897, |
| "logps/rejected": -1.1031153202056885, |
| "loss": 0.4969, |
| "nll_loss": 0.4509221017360687, |
| "rewards/accuracies": 0.8687499761581421, |
| "rewards/chosen": -0.13381414115428925, |
| "rewards/margins": 0.08680891990661621, |
| "rewards/rejected": -0.22062306106090546, |
| "step": 310 |
| }, |
| { |
| "epoch": 1.44, |
| "grad_norm": 3.515625, |
| "learning_rate": 8.824684825733865e-07, |
| "log_odds_chosen": 1.0580097436904907, |
| "log_odds_ratio": -0.38433724641799927, |
| "logits/chosen": -3.081042528152466, |
| "logits/rejected": -3.075192451477051, |
| "logps/chosen": -0.6307107210159302, |
| "logps/rejected": -1.1325790882110596, |
| "loss": 0.493, |
| "nll_loss": 0.4108586311340332, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.12614212930202484, |
| "rewards/margins": 0.10037368535995483, |
| "rewards/rejected": -0.22651581466197968, |
| "step": 320 |
| }, |
| { |
| "epoch": 1.49, |
| "grad_norm": 3.046875, |
| "learning_rate": 7.51180229473116e-07, |
| "log_odds_chosen": 0.9154760241508484, |
| "log_odds_ratio": -0.415397584438324, |
| "logits/chosen": -3.0937957763671875, |
| "logits/rejected": -3.0928263664245605, |
| "logps/chosen": -0.6775897741317749, |
| "logps/rejected": -1.1316546201705933, |
| "loss": 0.5044, |
| "nll_loss": 0.4353519380092621, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.13551795482635498, |
| "rewards/margins": 0.09081296622753143, |
| "rewards/rejected": -0.22633090615272522, |
| "step": 330 |
| }, |
| { |
| "epoch": 1.53, |
| "grad_norm": 3.15625, |
| "learning_rate": 6.28723129572247e-07, |
| "log_odds_chosen": 1.139094352722168, |
| "log_odds_ratio": -0.351654052734375, |
| "logits/chosen": -3.077972888946533, |
| "logits/rejected": -3.0774223804473877, |
| "logps/chosen": -0.6138228178024292, |
| "logps/rejected": -1.158979058265686, |
| "loss": 0.4728, |
| "nll_loss": 0.3857228457927704, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.12276456505060196, |
| "rewards/margins": 0.10903123766183853, |
| "rewards/rejected": -0.2317957878112793, |
| "step": 340 |
| }, |
| { |
| "epoch": 1.58, |
| "grad_norm": 3.234375, |
| "learning_rate": 5.157155641515766e-07, |
| "log_odds_chosen": 1.0156279802322388, |
| "log_odds_ratio": -0.3680209219455719, |
| "logits/chosen": -3.093818426132202, |
| "logits/rejected": -3.095491409301758, |
| "logps/chosen": -0.6614006161689758, |
| "logps/rejected": -1.1964319944381714, |
| "loss": 0.5024, |
| "nll_loss": 0.4554772973060608, |
| "rewards/accuracies": 0.9375, |
| "rewards/chosen": -0.13228009641170502, |
| "rewards/margins": 0.10700628906488419, |
| "rewards/rejected": -0.2392864227294922, |
| "step": 350 |
| }, |
| { |
| "epoch": 1.63, |
| "grad_norm": 3.109375, |
| "learning_rate": 4.127281964319446e-07, |
| "log_odds_chosen": 1.1072419881820679, |
| "log_odds_ratio": -0.365578830242157, |
| "logits/chosen": -3.0632922649383545, |
| "logits/rejected": -3.053469657897949, |
| "logps/chosen": -0.6406615376472473, |
| "logps/rejected": -1.1723295450210571, |
| "loss": 0.4717, |
| "nll_loss": 0.39158958196640015, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.12813231348991394, |
| "rewards/margins": 0.10633359849452972, |
| "rewards/rejected": -0.23446591198444366, |
| "step": 360 |
| }, |
| { |
| "epoch": 1.67, |
| "grad_norm": 3.171875, |
| "learning_rate": 3.202810898511424e-07, |
| "log_odds_chosen": 1.0118036270141602, |
| "log_odds_ratio": -0.3721558451652527, |
| "logits/chosen": -3.0760838985443115, |
| "logits/rejected": -3.0786654949188232, |
| "logps/chosen": -0.6295793652534485, |
| "logps/rejected": -1.1554322242736816, |
| "loss": 0.4954, |
| "nll_loss": 0.3821503221988678, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.12591585516929626, |
| "rewards/margins": 0.1051705852150917, |
| "rewards/rejected": -0.23108646273612976, |
| "step": 370 |
| }, |
| { |
| "epoch": 1.72, |
| "grad_norm": 3.015625, |
| "learning_rate": 2.388410818585263e-07, |
| "log_odds_chosen": 1.0309429168701172, |
| "log_odds_ratio": -0.3757808804512024, |
| "logits/chosen": -3.0944666862487793, |
| "logits/rejected": -3.102954149246216, |
| "logps/chosen": -0.6560705304145813, |
| "logps/rejected": -1.1918388605117798, |
| "loss": 0.4848, |
| "nll_loss": 0.4180088937282562, |
| "rewards/accuracies": 0.893750011920929, |
| "rewards/chosen": -0.13121411204338074, |
| "rewards/margins": 0.10715366899967194, |
| "rewards/rejected": -0.23836776614189148, |
| "step": 380 |
| }, |
| { |
| "epoch": 1.76, |
| "grad_norm": 3.0625, |
| "learning_rate": 1.6881942648911077e-07, |
| "log_odds_chosen": 1.045115351676941, |
| "log_odds_ratio": -0.3773229420185089, |
| "logits/chosen": -3.099276304244995, |
| "logits/rejected": -3.0971086025238037, |
| "logps/chosen": -0.6689329743385315, |
| "logps/rejected": -1.209533452987671, |
| "loss": 0.4865, |
| "nll_loss": 0.4016016125679016, |
| "rewards/accuracies": 0.887499988079071, |
| "rewards/chosen": -0.13378658890724182, |
| "rewards/margins": 0.10812009871006012, |
| "rewards/rejected": -0.24190667271614075, |
| "step": 390 |
| }, |
| { |
| "epoch": 1.81, |
| "grad_norm": 3.109375, |
| "learning_rate": 1.1056971762161584e-07, |
| "log_odds_chosen": 0.8860037922859192, |
| "log_odds_ratio": -0.4059659540653229, |
| "logits/chosen": -3.0970611572265625, |
| "logits/rejected": -3.0827975273132324, |
| "logps/chosen": -0.6797454357147217, |
| "logps/rejected": -1.1361665725708008, |
| "loss": 0.4883, |
| "nll_loss": 0.4087887406349182, |
| "rewards/accuracies": 0.90625, |
| "rewards/chosen": -0.13594909012317657, |
| "rewards/margins": 0.09128421545028687, |
| "rewards/rejected": -0.22723329067230225, |
| "step": 400 |
| }, |
| { |
| "epoch": 1.85, |
| "grad_norm": 3.015625, |
| "learning_rate": 6.438610340755464e-08, |
| "log_odds_chosen": 0.9491767883300781, |
| "log_odds_ratio": -0.3955945372581482, |
| "logits/chosen": -3.095716953277588, |
| "logits/rejected": -3.0901029109954834, |
| "logps/chosen": -0.6622263193130493, |
| "logps/rejected": -1.1627901792526245, |
| "loss": 0.4715, |
| "nll_loss": 0.4081706404685974, |
| "rewards/accuracies": 0.8999999761581421, |
| "rewards/chosen": -0.13244526088237762, |
| "rewards/margins": 0.10011278092861176, |
| "rewards/rejected": -0.23255804181098938, |
| "step": 410 |
| }, |
| { |
| "epoch": 1.9, |
| "grad_norm": 3.328125, |
| "learning_rate": 3.050180088809973e-08, |
| "log_odds_chosen": 0.89338219165802, |
| "log_odds_ratio": -0.41199594736099243, |
| "logits/chosen": -3.078094482421875, |
| "logits/rejected": -3.074589252471924, |
| "logps/chosen": -0.6442387700080872, |
| "logps/rejected": -1.103659987449646, |
| "loss": 0.4976, |
| "nll_loss": 0.40062204003334045, |
| "rewards/accuracies": 0.862500011920929, |
| "rewards/chosen": -0.12884774804115295, |
| "rewards/margins": 0.09188426285982132, |
| "rewards/rejected": -0.22073200345039368, |
| "step": 420 |
| }, |
| { |
| "epoch": 1.94, |
| "grad_norm": 3.15625, |
| "learning_rate": 9.087918299586772e-09, |
| "log_odds_chosen": 0.9813147783279419, |
| "log_odds_ratio": -0.4028463363647461, |
| "logits/chosen": -3.079704523086548, |
| "logits/rejected": -3.0714404582977295, |
| "logps/chosen": -0.6575345993041992, |
| "logps/rejected": -1.1326944828033447, |
| "loss": 0.4832, |
| "nll_loss": 0.39662784337997437, |
| "rewards/accuracies": 0.856249988079071, |
| "rewards/chosen": -0.13150693476200104, |
| "rewards/margins": 0.0950319841504097, |
| "rewards/rejected": -0.22653889656066895, |
| "step": 430 |
| }, |
| { |
| "epoch": 1.99, |
| "grad_norm": 4.03125, |
| "learning_rate": 2.525910147516131e-10, |
| "log_odds_chosen": 0.966279149055481, |
| "log_odds_ratio": -0.4019397795200348, |
| "logits/chosen": -3.0766844749450684, |
| "logits/rejected": -3.0788207054138184, |
| "logps/chosen": -0.6589742302894592, |
| "logps/rejected": -1.1595604419708252, |
| "loss": 0.48, |
| "nll_loss": 0.38374659419059753, |
| "rewards/accuracies": 0.8500000238418579, |
| "rewards/chosen": -0.13179484009742737, |
| "rewards/margins": 0.10011725127696991, |
| "rewards/rejected": -0.23191209137439728, |
| "step": 440 |
| }, |
| { |
| "epoch": 2.0, |
| "step": 442, |
| "total_flos": 0.0, |
| "train_loss": 0.5399103337283587, |
| "train_runtime": 8872.0808, |
| "train_samples_per_second": 3.191, |
| "train_steps_per_second": 0.05 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 442, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 2, |
| "save_steps": 500, |
| "total_flos": 0.0, |
| "train_batch_size": 8, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
