| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 478, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "learning_rate": 1.0416666666666666e-08, | |
| "logits/chosen": -2.771872043609619, | |
| "logits/rejected": -2.836427688598633, | |
| "logps/chosen": -186.35714721679688, | |
| "logps/pi_response": -56.843544006347656, | |
| "logps/ref_response": -56.843544006347656, | |
| "logps/rejected": -159.2776336669922, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "learning_rate": 1.0416666666666667e-07, | |
| "logits/chosen": -2.812751054763794, | |
| "logits/rejected": -2.805162191390991, | |
| "logps/chosen": -250.5966796875, | |
| "logps/pi_response": -84.03253173828125, | |
| "logps/ref_response": -84.02335357666016, | |
| "logps/rejected": -215.7916717529297, | |
| "loss": 0.6932, | |
| "rewards/accuracies": 0.4444444477558136, | |
| "rewards/chosen": 0.0001410771074006334, | |
| "rewards/margins": 0.00040674934280104935, | |
| "rewards/rejected": -0.00026567222084850073, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "learning_rate": 2.0833333333333333e-07, | |
| "logits/chosen": -2.818342924118042, | |
| "logits/rejected": -2.815077304840088, | |
| "logps/chosen": -247.14089965820312, | |
| "logps/pi_response": -69.06600189208984, | |
| "logps/ref_response": -68.91864013671875, | |
| "logps/rejected": -225.91415405273438, | |
| "loss": 0.6927, | |
| "rewards/accuracies": 0.45625001192092896, | |
| "rewards/chosen": 0.001155191333964467, | |
| "rewards/margins": -0.00012961139145772904, | |
| "rewards/rejected": 0.0012848025653511286, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": -2.7623016834259033, | |
| "logits/rejected": -2.729628324508667, | |
| "logps/chosen": -243.77188110351562, | |
| "logps/pi_response": -83.35380554199219, | |
| "logps/ref_response": -82.44132995605469, | |
| "logps/rejected": -221.338134765625, | |
| "loss": 0.6902, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.006917249411344528, | |
| "rewards/margins": 0.005432372447103262, | |
| "rewards/rejected": 0.00148487719707191, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "learning_rate": 4.1666666666666667e-07, | |
| "logits/chosen": -2.799405097961426, | |
| "logits/rejected": -2.7605228424072266, | |
| "logps/chosen": -235.19711303710938, | |
| "logps/pi_response": -72.81827545166016, | |
| "logps/ref_response": -70.3433837890625, | |
| "logps/rejected": -198.21481323242188, | |
| "loss": 0.6836, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": 0.027682146057486534, | |
| "rewards/margins": 0.023167867213487625, | |
| "rewards/rejected": 0.004514278843998909, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "learning_rate": 4.999733114418725e-07, | |
| "logits/chosen": -2.7923216819763184, | |
| "logits/rejected": -2.780092716217041, | |
| "logps/chosen": -244.62802124023438, | |
| "logps/pi_response": -87.8843002319336, | |
| "logps/ref_response": -81.94856262207031, | |
| "logps/rejected": -257.96270751953125, | |
| "loss": 0.6748, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": 0.04818359389901161, | |
| "rewards/margins": 0.05035162717103958, | |
| "rewards/rejected": -0.0021680318750441074, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "learning_rate": 4.990398100856366e-07, | |
| "logits/chosen": -2.744097948074341, | |
| "logits/rejected": -2.7316837310791016, | |
| "logps/chosen": -204.2379608154297, | |
| "logps/pi_response": -71.70608520507812, | |
| "logps/ref_response": -57.30153274536133, | |
| "logps/rejected": -191.46578979492188, | |
| "loss": 0.6608, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": 0.0015774101484566927, | |
| "rewards/margins": 0.05679439380764961, | |
| "rewards/rejected": -0.055216990411281586, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "learning_rate": 4.967775735898179e-07, | |
| "logits/chosen": -2.6771161556243896, | |
| "logits/rejected": -2.670588970184326, | |
| "logps/chosen": -212.1810302734375, | |
| "logps/pi_response": -96.94953155517578, | |
| "logps/ref_response": -65.7808609008789, | |
| "logps/rejected": -224.86795043945312, | |
| "loss": 0.6418, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.05775865167379379, | |
| "rewards/margins": 0.1283641755580902, | |
| "rewards/rejected": -0.1861228346824646, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "learning_rate": 4.931986719649298e-07, | |
| "logits/chosen": -2.6362526416778564, | |
| "logits/rejected": -2.591085910797119, | |
| "logps/chosen": -237.90524291992188, | |
| "logps/pi_response": -120.31071472167969, | |
| "logps/ref_response": -76.52632141113281, | |
| "logps/rejected": -243.9879608154297, | |
| "loss": 0.6356, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.10214054584503174, | |
| "rewards/margins": 0.2090938836336136, | |
| "rewards/rejected": -0.31123441457748413, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "learning_rate": 4.883222001996351e-07, | |
| "logits/chosen": -2.662489414215088, | |
| "logits/rejected": -2.6212925910949707, | |
| "logps/chosen": -238.72500610351562, | |
| "logps/pi_response": -122.1798095703125, | |
| "logps/ref_response": -70.05473327636719, | |
| "logps/rejected": -236.9344024658203, | |
| "loss": 0.6331, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.16823504865169525, | |
| "rewards/margins": 0.17120732367038727, | |
| "rewards/rejected": -0.3394424021244049, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "learning_rate": 4.821741763807186e-07, | |
| "logits/chosen": -2.6857962608337402, | |
| "logits/rejected": -2.6634621620178223, | |
| "logps/chosen": -234.5189208984375, | |
| "logps/pi_response": -132.98020935058594, | |
| "logps/ref_response": -72.16676330566406, | |
| "logps/rejected": -242.3022918701172, | |
| "loss": 0.6265, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.20184996724128723, | |
| "rewards/margins": 0.15526965260505676, | |
| "rewards/rejected": -0.357119619846344, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "learning_rate": 4.747874028753375e-07, | |
| "logits/chosen": -2.6297690868377686, | |
| "logits/rejected": -2.594348430633545, | |
| "logps/chosen": -280.73272705078125, | |
| "logps/pi_response": -147.92088317871094, | |
| "logps/ref_response": -78.85017395019531, | |
| "logps/rejected": -264.87420654296875, | |
| "loss": 0.6288, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -0.22727267444133759, | |
| "rewards/margins": 0.32433170080184937, | |
| "rewards/rejected": -0.5516043901443481, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "learning_rate": 4.662012913161997e-07, | |
| "logits/chosen": -2.6496646404266357, | |
| "logits/rejected": -2.619614839553833, | |
| "logps/chosen": -314.30682373046875, | |
| "logps/pi_response": -197.1916046142578, | |
| "logps/ref_response": -79.45616149902344, | |
| "logps/rejected": -317.90936279296875, | |
| "loss": 0.5952, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.5182690024375916, | |
| "rewards/margins": 0.3716340661048889, | |
| "rewards/rejected": -0.8899030685424805, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "learning_rate": 4.5646165232345103e-07, | |
| "logits/chosen": -2.669473648071289, | |
| "logits/rejected": -2.6336374282836914, | |
| "logps/chosen": -293.299072265625, | |
| "logps/pi_response": -204.230224609375, | |
| "logps/ref_response": -77.86495208740234, | |
| "logps/rejected": -316.03448486328125, | |
| "loss": 0.6044, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.616094708442688, | |
| "rewards/margins": 0.3894798159599304, | |
| "rewards/rejected": -1.0055744647979736, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "learning_rate": 4.456204510851956e-07, | |
| "logits/chosen": -2.600029706954956, | |
| "logits/rejected": -2.552990198135376, | |
| "logps/chosen": -274.93023681640625, | |
| "logps/pi_response": -175.21990966796875, | |
| "logps/ref_response": -76.53099060058594, | |
| "logps/rejected": -297.7197570800781, | |
| "loss": 0.6096, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.3966221511363983, | |
| "rewards/margins": 0.4644131660461426, | |
| "rewards/rejected": -0.8610353469848633, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "learning_rate": 4.337355301007335e-07, | |
| "logits/chosen": -2.451113224029541, | |
| "logits/rejected": -2.440601348876953, | |
| "logps/chosen": -244.29443359375, | |
| "logps/pi_response": -189.71490478515625, | |
| "logps/ref_response": -59.47999954223633, | |
| "logps/rejected": -276.4666442871094, | |
| "loss": 0.5883, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -0.521985650062561, | |
| "rewards/margins": 0.40682539343833923, | |
| "rewards/rejected": -0.9288110733032227, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "learning_rate": 4.2087030056579986e-07, | |
| "logits/chosen": -2.392782688140869, | |
| "logits/rejected": -2.3590378761291504, | |
| "logps/chosen": -293.15496826171875, | |
| "logps/pi_response": -246.6798095703125, | |
| "logps/ref_response": -73.12691497802734, | |
| "logps/rejected": -316.5404968261719, | |
| "loss": 0.6101, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.7774425745010376, | |
| "rewards/margins": 0.32445091009140015, | |
| "rewards/rejected": -1.101893424987793, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "learning_rate": 4.070934040463998e-07, | |
| "logits/chosen": -2.3637728691101074, | |
| "logits/rejected": -2.3082034587860107, | |
| "logps/chosen": -330.51300048828125, | |
| "logps/pi_response": -247.8793182373047, | |
| "logps/ref_response": -69.19013214111328, | |
| "logps/rejected": -340.6532287597656, | |
| "loss": 0.5765, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.8771257400512695, | |
| "rewards/margins": 0.37836703658103943, | |
| "rewards/rejected": -1.255492925643921, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "learning_rate": 3.9247834624635404e-07, | |
| "logits/chosen": -2.3222761154174805, | |
| "logits/rejected": -2.290357828140259, | |
| "logps/chosen": -317.992919921875, | |
| "logps/pi_response": -246.22482299804688, | |
| "logps/ref_response": -64.53086853027344, | |
| "logps/rejected": -345.11138916015625, | |
| "loss": 0.5736, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -0.8496101498603821, | |
| "rewards/margins": 0.627398669719696, | |
| "rewards/rejected": -1.4770088195800781, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "learning_rate": 3.7710310482256523e-07, | |
| "logits/chosen": -2.2988460063934326, | |
| "logits/rejected": -2.2790465354919434, | |
| "logps/chosen": -323.6756286621094, | |
| "logps/pi_response": -238.6248016357422, | |
| "logps/ref_response": -72.57078552246094, | |
| "logps/rejected": -355.2505798339844, | |
| "loss": 0.5828, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -1.0282012224197388, | |
| "rewards/margins": 0.3955053389072418, | |
| "rewards/rejected": -1.4237064123153687, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "learning_rate": 3.610497133404795e-07, | |
| "logits/chosen": -2.335862636566162, | |
| "logits/rejected": -2.277442693710327, | |
| "logps/chosen": -346.7106628417969, | |
| "logps/pi_response": -245.89291381835938, | |
| "logps/ref_response": -73.11415100097656, | |
| "logps/rejected": -370.8809814453125, | |
| "loss": 0.5449, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.9884869456291199, | |
| "rewards/margins": 0.5412154793739319, | |
| "rewards/rejected": -1.5297023057937622, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "learning_rate": 3.4440382358952115e-07, | |
| "logits/chosen": -2.2967655658721924, | |
| "logits/rejected": -2.263054609298706, | |
| "logps/chosen": -343.8060302734375, | |
| "logps/pi_response": -267.05694580078125, | |
| "logps/ref_response": -70.57121276855469, | |
| "logps/rejected": -349.5938415527344, | |
| "loss": 0.5706, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.133570671081543, | |
| "rewards/margins": 0.5091238617897034, | |
| "rewards/rejected": -1.6426944732666016, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "learning_rate": 3.272542485937368e-07, | |
| "logits/chosen": -2.383333921432495, | |
| "logits/rejected": -2.3383095264434814, | |
| "logps/chosen": -331.64178466796875, | |
| "logps/pi_response": -235.6920623779297, | |
| "logps/ref_response": -68.40555572509766, | |
| "logps/rejected": -365.8157043457031, | |
| "loss": 0.5675, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.9668046236038208, | |
| "rewards/margins": 0.5283567905426025, | |
| "rewards/rejected": -1.495161533355713, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "learning_rate": 3.096924887558854e-07, | |
| "logits/chosen": -2.343796491622925, | |
| "logits/rejected": -2.309424877166748, | |
| "logps/chosen": -322.5975036621094, | |
| "logps/pi_response": -233.7900390625, | |
| "logps/ref_response": -67.48506164550781, | |
| "logps/rejected": -376.9325256347656, | |
| "loss": 0.5833, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.9696311950683594, | |
| "rewards/margins": 0.49308356642723083, | |
| "rewards/rejected": -1.462714672088623, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "learning_rate": 2.9181224366319943e-07, | |
| "logits/chosen": -2.346538782119751, | |
| "logits/rejected": -2.2966227531433105, | |
| "logps/chosen": -333.65643310546875, | |
| "logps/pi_response": -260.61004638671875, | |
| "logps/ref_response": -67.38578033447266, | |
| "logps/rejected": -375.87469482421875, | |
| "loss": 0.5634, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.1766176223754883, | |
| "rewards/margins": 0.4966951012611389, | |
| "rewards/rejected": -1.673312783241272, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "learning_rate": 2.7370891215954565e-07, | |
| "logits/chosen": -2.3725361824035645, | |
| "logits/rejected": -2.3253109455108643, | |
| "logps/chosen": -358.3512268066406, | |
| "logps/pi_response": -293.8153381347656, | |
| "logps/ref_response": -72.65675354003906, | |
| "logps/rejected": -365.43853759765625, | |
| "loss": 0.5653, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.2577649354934692, | |
| "rewards/margins": 0.4211507737636566, | |
| "rewards/rejected": -1.6789157390594482, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "learning_rate": 2.55479083351317e-07, | |
| "logits/chosen": -2.37805438041687, | |
| "logits/rejected": -2.32930326461792, | |
| "logps/chosen": -374.30377197265625, | |
| "logps/pi_response": -283.6889343261719, | |
| "logps/ref_response": -74.38246154785156, | |
| "logps/rejected": -373.2997741699219, | |
| "loss": 0.5588, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.15468430519104, | |
| "rewards/margins": 0.474844366312027, | |
| "rewards/rejected": -1.6295287609100342, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "learning_rate": 2.3722002126275822e-07, | |
| "logits/chosen": -2.2448177337646484, | |
| "logits/rejected": -2.2177035808563232, | |
| "logps/chosen": -346.1255187988281, | |
| "logps/pi_response": -286.4845275878906, | |
| "logps/ref_response": -76.11927032470703, | |
| "logps/rejected": -373.6055603027344, | |
| "loss": 0.5786, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.2877910137176514, | |
| "rewards/margins": 0.400728315114975, | |
| "rewards/rejected": -1.6885192394256592, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "learning_rate": 2.19029145890313e-07, | |
| "logits/chosen": -2.1976356506347656, | |
| "logits/rejected": -2.1738736629486084, | |
| "logps/chosen": -318.11578369140625, | |
| "logps/pi_response": -277.13848876953125, | |
| "logps/ref_response": -61.588165283203125, | |
| "logps/rejected": -371.4759216308594, | |
| "loss": 0.5439, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.2585313320159912, | |
| "rewards/margins": 0.4716874659061432, | |
| "rewards/rejected": -1.7302188873291016, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "learning_rate": 2.0100351342479216e-07, | |
| "logits/chosen": -2.267279863357544, | |
| "logits/rejected": -2.225353240966797, | |
| "logps/chosen": -341.911865234375, | |
| "logps/pi_response": -263.16766357421875, | |
| "logps/ref_response": -70.3861312866211, | |
| "logps/rejected": -358.21624755859375, | |
| "loss": 0.565, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.13209068775177, | |
| "rewards/margins": 0.480673611164093, | |
| "rewards/rejected": -1.6127643585205078, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "learning_rate": 1.8323929841460178e-07, | |
| "logits/chosen": -2.2511088848114014, | |
| "logits/rejected": -2.20157790184021, | |
| "logps/chosen": -369.5644226074219, | |
| "logps/pi_response": -284.96087646484375, | |
| "logps/ref_response": -79.78241729736328, | |
| "logps/rejected": -390.3699645996094, | |
| "loss": 0.5584, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.189987063407898, | |
| "rewards/margins": 0.4863070845603943, | |
| "rewards/rejected": -1.6762943267822266, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "learning_rate": 1.6583128063291573e-07, | |
| "logits/chosen": -2.2550597190856934, | |
| "logits/rejected": -2.2291643619537354, | |
| "logps/chosen": -391.4834289550781, | |
| "logps/pi_response": -303.56292724609375, | |
| "logps/ref_response": -84.91761016845703, | |
| "logps/rejected": -436.4339904785156, | |
| "loss": 0.5727, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.2904925346374512, | |
| "rewards/margins": 0.6019302606582642, | |
| "rewards/rejected": -1.8924226760864258, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "learning_rate": 1.488723393865766e-07, | |
| "logits/chosen": -2.292328357696533, | |
| "logits/rejected": -2.205026388168335, | |
| "logps/chosen": -375.3182678222656, | |
| "logps/pi_response": -294.6897277832031, | |
| "logps/ref_response": -75.90422821044922, | |
| "logps/rejected": -383.18994140625, | |
| "loss": 0.547, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.125930905342102, | |
| "rewards/margins": 0.6577755212783813, | |
| "rewards/rejected": -1.7837063074111938, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "learning_rate": 1.3245295796480788e-07, | |
| "logits/chosen": -2.238246202468872, | |
| "logits/rejected": -2.1797311305999756, | |
| "logps/chosen": -344.31427001953125, | |
| "logps/pi_response": -295.9718933105469, | |
| "logps/ref_response": -69.94654846191406, | |
| "logps/rejected": -386.03436279296875, | |
| "loss": 0.5343, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.2373181581497192, | |
| "rewards/margins": 0.531391978263855, | |
| "rewards/rejected": -1.7687098979949951, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "learning_rate": 1.1666074087171627e-07, | |
| "logits/chosen": -2.2410218715667725, | |
| "logits/rejected": -2.182896137237549, | |
| "logps/chosen": -320.0640563964844, | |
| "logps/pi_response": -265.65252685546875, | |
| "logps/ref_response": -72.93882751464844, | |
| "logps/rejected": -349.9189453125, | |
| "loss": 0.5352, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -1.1647237539291382, | |
| "rewards/margins": 0.5372185707092285, | |
| "rewards/rejected": -1.7019424438476562, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "learning_rate": 1.0157994641835734e-07, | |
| "logits/chosen": -2.214590549468994, | |
| "logits/rejected": -2.144507646560669, | |
| "logps/chosen": -331.2563171386719, | |
| "logps/pi_response": -289.6974182128906, | |
| "logps/ref_response": -74.75759887695312, | |
| "logps/rejected": -383.2954406738281, | |
| "loss": 0.5407, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.2160807847976685, | |
| "rewards/margins": 0.6268461346626282, | |
| "rewards/rejected": -1.8429267406463623, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "learning_rate": 8.729103716819111e-08, | |
| "logits/chosen": -2.2340056896209717, | |
| "logits/rejected": -2.1630749702453613, | |
| "logps/chosen": -363.3587341308594, | |
| "logps/pi_response": -291.95538330078125, | |
| "logps/ref_response": -75.47894287109375, | |
| "logps/rejected": -398.1706237792969, | |
| "loss": 0.5352, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.259597659111023, | |
| "rewards/margins": 0.6464335322380066, | |
| "rewards/rejected": -1.9060310125350952, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "learning_rate": 7.387025063449081e-08, | |
| "logits/chosen": -2.187648296356201, | |
| "logits/rejected": -2.1372499465942383, | |
| "logps/chosen": -342.2130432128906, | |
| "logps/pi_response": -296.50830078125, | |
| "logps/ref_response": -68.33134460449219, | |
| "logps/rejected": -397.110595703125, | |
| "loss": 0.5447, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -1.2424293756484985, | |
| "rewards/margins": 0.5943651795387268, | |
| "rewards/rejected": -1.8367946147918701, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "learning_rate": 6.138919252022435e-08, | |
| "logits/chosen": -2.223222255706787, | |
| "logits/rejected": -2.1696248054504395, | |
| "logps/chosen": -345.4095764160156, | |
| "logps/pi_response": -297.59759521484375, | |
| "logps/ref_response": -72.3606948852539, | |
| "logps/rejected": -421.50531005859375, | |
| "loss": 0.5508, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -1.3772599697113037, | |
| "rewards/margins": 0.516662061214447, | |
| "rewards/rejected": -1.8939220905303955, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "learning_rate": 4.991445467064689e-08, | |
| "logits/chosen": -2.2751657962799072, | |
| "logits/rejected": -2.190664529800415, | |
| "logps/chosen": -357.4648742675781, | |
| "logps/pi_response": -295.1226806640625, | |
| "logps/ref_response": -70.87272644042969, | |
| "logps/rejected": -415.9715270996094, | |
| "loss": 0.5434, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -1.3301585912704468, | |
| "rewards/margins": 0.6158719658851624, | |
| "rewards/rejected": -1.9460303783416748, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "learning_rate": 3.9507259776993954e-08, | |
| "logits/chosen": -2.254105806350708, | |
| "logits/rejected": -2.2081995010375977, | |
| "logps/chosen": -366.28436279296875, | |
| "logps/pi_response": -304.9342041015625, | |
| "logps/ref_response": -77.48291015625, | |
| "logps/rejected": -364.3297119140625, | |
| "loss": 0.5429, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.3418512344360352, | |
| "rewards/margins": 0.46248334646224976, | |
| "rewards/rejected": -1.8043344020843506, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "learning_rate": 3.022313472693447e-08, | |
| "logits/chosen": -2.232144594192505, | |
| "logits/rejected": -2.1603074073791504, | |
| "logps/chosen": -369.909423828125, | |
| "logps/pi_response": -314.9814147949219, | |
| "logps/ref_response": -76.4972915649414, | |
| "logps/rejected": -432.63824462890625, | |
| "loss": 0.5482, | |
| "rewards/accuracies": 0.7124999761581421, | |
| "rewards/chosen": -1.456408977508545, | |
| "rewards/margins": 0.5390293598175049, | |
| "rewards/rejected": -1.9954382181167603, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "learning_rate": 2.2111614344599684e-08, | |
| "logits/chosen": -2.268048048019409, | |
| "logits/rejected": -2.194788694381714, | |
| "logps/chosen": -351.391845703125, | |
| "logps/pi_response": -303.7065734863281, | |
| "logps/ref_response": -70.95718383789062, | |
| "logps/rejected": -420.17315673828125, | |
| "loss": 0.5353, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -1.3950040340423584, | |
| "rewards/margins": 0.5390046238899231, | |
| "rewards/rejected": -1.9340085983276367, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "learning_rate": 1.521597710086439e-08, | |
| "logits/chosen": -2.256483554840088, | |
| "logits/rejected": -2.182900905609131, | |
| "logps/chosen": -394.15948486328125, | |
| "logps/pi_response": -322.7268981933594, | |
| "logps/ref_response": -71.76502990722656, | |
| "logps/rejected": -406.4312438964844, | |
| "loss": 0.5666, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.330580711364746, | |
| "rewards/margins": 0.6264187693595886, | |
| "rewards/rejected": -1.9569995403289795, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "learning_rate": 9.57301420397924e-09, | |
| "logits/chosen": -2.261594772338867, | |
| "logits/rejected": -2.219043016433716, | |
| "logps/chosen": -363.95025634765625, | |
| "logps/pi_response": -315.3779296875, | |
| "logps/ref_response": -77.31925201416016, | |
| "logps/rejected": -394.43280029296875, | |
| "loss": 0.5349, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.391911268234253, | |
| "rewards/margins": 0.5445317029953003, | |
| "rewards/rejected": -1.9364430904388428, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "learning_rate": 5.212833302556258e-09, | |
| "logits/chosen": -2.2221949100494385, | |
| "logits/rejected": -2.142899513244629, | |
| "logps/chosen": -372.461181640625, | |
| "logps/pi_response": -308.6124572753906, | |
| "logps/ref_response": -75.69192504882812, | |
| "logps/rejected": -399.45892333984375, | |
| "loss": 0.5203, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -1.2634239196777344, | |
| "rewards/margins": 0.6645442247390747, | |
| "rewards/rejected": -1.9279682636260986, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "learning_rate": 2.158697848236607e-09, | |
| "logits/chosen": -2.2366528511047363, | |
| "logits/rejected": -2.166773796081543, | |
| "logps/chosen": -369.5625915527344, | |
| "logps/pi_response": -296.8544006347656, | |
| "logps/ref_response": -69.99818420410156, | |
| "logps/rejected": -397.776123046875, | |
| "loss": 0.5432, | |
| "rewards/accuracies": 0.7250000238418579, | |
| "rewards/chosen": -1.3708467483520508, | |
| "rewards/margins": 0.6627798676490784, | |
| "rewards/rejected": -2.0336265563964844, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "learning_rate": 4.269029751107489e-10, | |
| "logits/chosen": -2.2382636070251465, | |
| "logits/rejected": -2.1679019927978516, | |
| "logps/chosen": -345.45440673828125, | |
| "logps/pi_response": -300.89666748046875, | |
| "logps/ref_response": -71.26921081542969, | |
| "logps/rejected": -400.05804443359375, | |
| "loss": 0.5428, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.2617619037628174, | |
| "rewards/margins": 0.6952942609786987, | |
| "rewards/rejected": -1.9570560455322266, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 478, | |
| "total_flos": 0.0, | |
| "train_loss": 0.583395419260448, | |
| "train_runtime": 12448.4703, | |
| "train_samples_per_second": 4.911, | |
| "train_steps_per_second": 0.038 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 478, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |