{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 478,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -2.771872043609619,
      "logits/rejected": -2.836427688598633,
      "logps/chosen": -186.35714721679688,
      "logps/pi_response": -56.843544006347656,
      "logps/ref_response": -56.843544006347656,
      "logps/rejected": -159.2776336669922,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.812751054763794,
      "logits/rejected": -2.805162191390991,
      "logps/chosen": -250.5966796875,
      "logps/pi_response": -84.03253173828125,
      "logps/ref_response": -84.02335357666016,
      "logps/rejected": -215.7916717529297,
      "loss": 0.6932,
      "rewards/accuracies": 0.4444444477558136,
      "rewards/chosen": 0.0001410771074006334,
      "rewards/margins": 0.00040674934280104935,
      "rewards/rejected": -0.00026567222084850073,
      "step": 10
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.818342924118042,
      "logits/rejected": -2.815077304840088,
      "logps/chosen": -247.14089965820312,
      "logps/pi_response": -69.06600189208984,
      "logps/ref_response": -68.91864013671875,
      "logps/rejected": -225.91415405273438,
      "loss": 0.6927,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": 0.001155191333964467,
      "rewards/margins": -0.00012961139145772904,
      "rewards/rejected": 0.0012848025653511286,
      "step": 20
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.7623016834259033,
      "logits/rejected": -2.729628324508667,
      "logps/chosen": -243.77188110351562,
      "logps/pi_response": -83.35380554199219,
      "logps/ref_response": -82.44132995605469,
      "logps/rejected": -221.338134765625,
      "loss": 0.6902,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.006917249411344528,
      "rewards/margins": 0.005432372447103262,
      "rewards/rejected": 0.00148487719707191,
      "step": 30
    },
    {
      "epoch": 0.08,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.799405097961426,
      "logits/rejected": -2.7605228424072266,
      "logps/chosen": -235.19711303710938,
      "logps/pi_response": -72.81827545166016,
      "logps/ref_response": -70.3433837890625,
      "logps/rejected": -198.21481323242188,
      "loss": 0.6836,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.027682146057486534,
      "rewards/margins": 0.023167867213487625,
      "rewards/rejected": 0.004514278843998909,
      "step": 40
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.999733114418725e-07,
      "logits/chosen": -2.7923216819763184,
      "logits/rejected": -2.780092716217041,
      "logps/chosen": -244.62802124023438,
      "logps/pi_response": -87.8843002319336,
      "logps/ref_response": -81.94856262207031,
      "logps/rejected": -257.96270751953125,
      "loss": 0.6748,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.04818359389901161,
      "rewards/margins": 0.05035162717103958,
      "rewards/rejected": -0.0021680318750441074,
      "step": 50
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990398100856366e-07,
      "logits/chosen": -2.744097948074341,
      "logits/rejected": -2.7316837310791016,
      "logps/chosen": -204.2379608154297,
      "logps/pi_response": -71.70608520507812,
      "logps/ref_response": -57.30153274536133,
      "logps/rejected": -191.46578979492188,
      "loss": 0.6608,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": 0.0015774101484566927,
      "rewards/margins": 0.05679439380764961,
      "rewards/rejected": -0.055216990411281586,
      "step": 60
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.967775735898179e-07,
      "logits/chosen": -2.6771161556243896,
      "logits/rejected": -2.670588970184326,
      "logps/chosen": -212.1810302734375,
      "logps/pi_response": -96.94953155517578,
      "logps/ref_response": -65.7808609008789,
      "logps/rejected": -224.86795043945312,
      "loss": 0.6418,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.05775865167379379,
      "rewards/margins": 0.1283641755580902,
      "rewards/rejected": -0.1861228346824646,
      "step": 70
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.931986719649298e-07,
      "logits/chosen": -2.6362526416778564,
      "logits/rejected": -2.591085910797119,
      "logps/chosen": -237.90524291992188,
      "logps/pi_response": -120.31071472167969,
      "logps/ref_response": -76.52632141113281,
      "logps/rejected": -243.9879608154297,
      "loss": 0.6356,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10214054584503174,
      "rewards/margins": 0.2090938836336136,
      "rewards/rejected": -0.31123441457748413,
      "step": 80
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.883222001996351e-07,
      "logits/chosen": -2.662489414215088,
      "logits/rejected": -2.6212925910949707,
      "logps/chosen": -238.72500610351562,
      "logps/pi_response": -122.1798095703125,
      "logps/ref_response": -70.05473327636719,
      "logps/rejected": -236.9344024658203,
      "loss": 0.6331,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.16823504865169525,
      "rewards/margins": 0.17120732367038727,
      "rewards/rejected": -0.3394424021244049,
      "step": 90
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.821741763807186e-07,
      "logits/chosen": -2.6857962608337402,
      "logits/rejected": -2.6634621620178223,
      "logps/chosen": -234.5189208984375,
      "logps/pi_response": -132.98020935058594,
      "logps/ref_response": -72.16676330566406,
      "logps/rejected": -242.3022918701172,
      "loss": 0.6265,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.20184996724128723,
      "rewards/margins": 0.15526965260505676,
      "rewards/rejected": -0.357119619846344,
      "step": 100
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.747874028753375e-07,
      "logits/chosen": -2.6297690868377686,
      "logits/rejected": -2.594348430633545,
      "logps/chosen": -280.73272705078125,
      "logps/pi_response": -147.92088317871094,
      "logps/ref_response": -78.85017395019531,
      "logps/rejected": -264.87420654296875,
      "loss": 0.6288,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.22727267444133759,
      "rewards/margins": 0.32433170080184937,
      "rewards/rejected": -0.5516043901443481,
      "step": 110
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.662012913161997e-07,
      "logits/chosen": -2.6496646404266357,
      "logits/rejected": -2.619614839553833,
      "logps/chosen": -314.30682373046875,
      "logps/pi_response": -197.1916046142578,
      "logps/ref_response": -79.45616149902344,
      "logps/rejected": -317.90936279296875,
      "loss": 0.5952,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.5182690024375916,
      "rewards/margins": 0.3716340661048889,
      "rewards/rejected": -0.8899030685424805,
      "step": 120
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5646165232345103e-07,
      "logits/chosen": -2.669473648071289,
      "logits/rejected": -2.6336374282836914,
      "logps/chosen": -293.299072265625,
      "logps/pi_response": -204.230224609375,
      "logps/ref_response": -77.86495208740234,
      "logps/rejected": -316.03448486328125,
      "loss": 0.6044,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.616094708442688,
      "rewards/margins": 0.3894798159599304,
      "rewards/rejected": -1.0055744647979736,
      "step": 130
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.456204510851956e-07,
      "logits/chosen": -2.600029706954956,
      "logits/rejected": -2.552990198135376,
      "logps/chosen": -274.93023681640625,
      "logps/pi_response": -175.21990966796875,
      "logps/ref_response": -76.53099060058594,
      "logps/rejected": -297.7197570800781,
      "loss": 0.6096,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.3966221511363983,
      "rewards/margins": 0.4644131660461426,
      "rewards/rejected": -0.8610353469848633,
      "step": 140
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.337355301007335e-07,
      "logits/chosen": -2.451113224029541,
      "logits/rejected": -2.440601348876953,
      "logps/chosen": -244.29443359375,
      "logps/pi_response": -189.71490478515625,
      "logps/ref_response": -59.47999954223633,
      "logps/rejected": -276.4666442871094,
      "loss": 0.5883,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.521985650062561,
      "rewards/margins": 0.40682539343833923,
      "rewards/rejected": -0.9288110733032227,
      "step": 150
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.2087030056579986e-07,
      "logits/chosen": -2.392782688140869,
      "logits/rejected": -2.3590378761291504,
      "logps/chosen": -293.15496826171875,
      "logps/pi_response": -246.6798095703125,
      "logps/ref_response": -73.12691497802734,
      "logps/rejected": -316.5404968261719,
      "loss": 0.6101,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.7774425745010376,
      "rewards/margins": 0.32445091009140015,
      "rewards/rejected": -1.101893424987793,
      "step": 160
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.070934040463998e-07,
      "logits/chosen": -2.3637728691101074,
      "logits/rejected": -2.3082034587860107,
      "logps/chosen": -330.51300048828125,
      "logps/pi_response": -247.8793182373047,
      "logps/ref_response": -69.19013214111328,
      "logps/rejected": -340.6532287597656,
      "loss": 0.5765,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.8771257400512695,
      "rewards/margins": 0.37836703658103943,
      "rewards/rejected": -1.255492925643921,
      "step": 170
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.9247834624635404e-07,
      "logits/chosen": -2.3222761154174805,
      "logits/rejected": -2.290357828140259,
      "logps/chosen": -317.992919921875,
      "logps/pi_response": -246.22482299804688,
      "logps/ref_response": -64.53086853027344,
      "logps/rejected": -345.11138916015625,
      "loss": 0.5736,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8496101498603821,
      "rewards/margins": 0.627398669719696,
      "rewards/rejected": -1.4770088195800781,
      "step": 180
    },
    {
      "epoch": 0.4,
      "learning_rate": 3.7710310482256523e-07,
      "logits/chosen": -2.2988460063934326,
      "logits/rejected": -2.2790465354919434,
      "logps/chosen": -323.6756286621094,
      "logps/pi_response": -238.6248016357422,
      "logps/ref_response": -72.57078552246094,
      "logps/rejected": -355.2505798339844,
      "loss": 0.5828,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0282012224197388,
      "rewards/margins": 0.3955053389072418,
      "rewards/rejected": -1.4237064123153687,
      "step": 190
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.610497133404795e-07,
      "logits/chosen": -2.335862636566162,
      "logits/rejected": -2.277442693710327,
      "logps/chosen": -346.7106628417969,
      "logps/pi_response": -245.89291381835938,
      "logps/ref_response": -73.11415100097656,
      "logps/rejected": -370.8809814453125,
      "loss": 0.5449,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.9884869456291199,
      "rewards/margins": 0.5412154793739319,
      "rewards/rejected": -1.5297023057937622,
      "step": 200
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4440382358952115e-07,
      "logits/chosen": -2.2967655658721924,
      "logits/rejected": -2.263054609298706,
      "logps/chosen": -343.8060302734375,
      "logps/pi_response": -267.05694580078125,
      "logps/ref_response": -70.57121276855469,
      "logps/rejected": -349.5938415527344,
      "loss": 0.5706,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.133570671081543,
      "rewards/margins": 0.5091238617897034,
      "rewards/rejected": -1.6426944732666016,
      "step": 210
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": -2.383333921432495,
      "logits/rejected": -2.3383095264434814,
      "logps/chosen": -331.64178466796875,
      "logps/pi_response": -235.6920623779297,
      "logps/ref_response": -68.40555572509766,
      "logps/rejected": -365.8157043457031,
      "loss": 0.5675,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.9668046236038208,
      "rewards/margins": 0.5283567905426025,
      "rewards/rejected": -1.495161533355713,
      "step": 220
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.096924887558854e-07,
      "logits/chosen": -2.343796491622925,
      "logits/rejected": -2.309424877166748,
      "logps/chosen": -322.5975036621094,
      "logps/pi_response": -233.7900390625,
      "logps/ref_response": -67.48506164550781,
      "logps/rejected": -376.9325256347656,
      "loss": 0.5833,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9696311950683594,
      "rewards/margins": 0.49308356642723083,
      "rewards/rejected": -1.462714672088623,
      "step": 230
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9181224366319943e-07,
      "logits/chosen": -2.346538782119751,
      "logits/rejected": -2.2966227531433105,
      "logps/chosen": -333.65643310546875,
      "logps/pi_response": -260.61004638671875,
      "logps/ref_response": -67.38578033447266,
      "logps/rejected": -375.87469482421875,
      "loss": 0.5634,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.1766176223754883,
      "rewards/margins": 0.4966951012611389,
      "rewards/rejected": -1.673312783241272,
      "step": 240
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.7370891215954565e-07,
      "logits/chosen": -2.3725361824035645,
      "logits/rejected": -2.3253109455108643,
      "logps/chosen": -358.3512268066406,
      "logps/pi_response": -293.8153381347656,
      "logps/ref_response": -72.65675354003906,
      "logps/rejected": -365.43853759765625,
      "loss": 0.5653,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.2577649354934692,
      "rewards/margins": 0.4211507737636566,
      "rewards/rejected": -1.6789157390594482,
      "step": 250
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.55479083351317e-07,
      "logits/chosen": -2.37805438041687,
      "logits/rejected": -2.32930326461792,
      "logps/chosen": -374.30377197265625,
      "logps/pi_response": -283.6889343261719,
      "logps/ref_response": -74.38246154785156,
      "logps/rejected": -373.2997741699219,
      "loss": 0.5588,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.15468430519104,
      "rewards/margins": 0.474844366312027,
      "rewards/rejected": -1.6295287609100342,
      "step": 260
    },
    {
      "epoch": 0.56,
      "learning_rate": 2.3722002126275822e-07,
      "logits/chosen": -2.2448177337646484,
      "logits/rejected": -2.2177035808563232,
      "logps/chosen": -346.1255187988281,
      "logps/pi_response": -286.4845275878906,
      "logps/ref_response": -76.11927032470703,
      "logps/rejected": -373.6055603027344,
      "loss": 0.5786,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.2877910137176514,
      "rewards/margins": 0.400728315114975,
      "rewards/rejected": -1.6885192394256592,
      "step": 270
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.19029145890313e-07,
      "logits/chosen": -2.1976356506347656,
      "logits/rejected": -2.1738736629486084,
      "logps/chosen": -318.11578369140625,
      "logps/pi_response": -277.13848876953125,
      "logps/ref_response": -61.588165283203125,
      "logps/rejected": -371.4759216308594,
      "loss": 0.5439,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.2585313320159912,
      "rewards/margins": 0.4716874659061432,
      "rewards/rejected": -1.7302188873291016,
      "step": 280
    },
    {
      "epoch": 0.61,
      "learning_rate": 2.0100351342479216e-07,
      "logits/chosen": -2.267279863357544,
      "logits/rejected": -2.225353240966797,
      "logps/chosen": -341.911865234375,
      "logps/pi_response": -263.16766357421875,
      "logps/ref_response": -70.3861312866211,
      "logps/rejected": -358.21624755859375,
      "loss": 0.565,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.13209068775177,
      "rewards/margins": 0.480673611164093,
      "rewards/rejected": -1.6127643585205078,
      "step": 290
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8323929841460178e-07,
      "logits/chosen": -2.2511088848114014,
      "logits/rejected": -2.20157790184021,
      "logps/chosen": -369.5644226074219,
      "logps/pi_response": -284.96087646484375,
      "logps/ref_response": -79.78241729736328,
      "logps/rejected": -390.3699645996094,
      "loss": 0.5584,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.189987063407898,
      "rewards/margins": 0.4863070845603943,
      "rewards/rejected": -1.6762943267822266,
      "step": 300
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6583128063291573e-07,
      "logits/chosen": -2.2550597190856934,
      "logits/rejected": -2.2291643619537354,
      "logps/chosen": -391.4834289550781,
      "logps/pi_response": -303.56292724609375,
      "logps/ref_response": -84.91761016845703,
      "logps/rejected": -436.4339904785156,
      "loss": 0.5727,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2904925346374512,
      "rewards/margins": 0.6019302606582642,
      "rewards/rejected": -1.8924226760864258,
      "step": 310
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.488723393865766e-07,
      "logits/chosen": -2.292328357696533,
      "logits/rejected": -2.205026388168335,
      "logps/chosen": -375.3182678222656,
      "logps/pi_response": -294.6897277832031,
      "logps/ref_response": -75.90422821044922,
      "logps/rejected": -383.18994140625,
      "loss": 0.547,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.125930905342102,
      "rewards/margins": 0.6577755212783813,
      "rewards/rejected": -1.7837063074111938,
      "step": 320
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3245295796480788e-07,
      "logits/chosen": -2.238246202468872,
      "logits/rejected": -2.1797311305999756,
      "logps/chosen": -344.31427001953125,
      "logps/pi_response": -295.9718933105469,
      "logps/ref_response": -69.94654846191406,
      "logps/rejected": -386.03436279296875,
      "loss": 0.5343,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.2373181581497192,
      "rewards/margins": 0.531391978263855,
      "rewards/rejected": -1.7687098979949951,
      "step": 330
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1666074087171627e-07,
      "logits/chosen": -2.2410218715667725,
      "logits/rejected": -2.182896137237549,
      "logps/chosen": -320.0640563964844,
      "logps/pi_response": -265.65252685546875,
      "logps/ref_response": -72.93882751464844,
      "logps/rejected": -349.9189453125,
      "loss": 0.5352,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -1.1647237539291382,
      "rewards/margins": 0.5372185707092285,
      "rewards/rejected": -1.7019424438476562,
      "step": 340
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.0157994641835734e-07,
      "logits/chosen": -2.214590549468994,
      "logits/rejected": -2.144507646560669,
      "logps/chosen": -331.2563171386719,
      "logps/pi_response": -289.6974182128906,
      "logps/ref_response": -74.75759887695312,
      "logps/rejected": -383.2954406738281,
      "loss": 0.5407,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.2160807847976685,
      "rewards/margins": 0.6268461346626282,
      "rewards/rejected": -1.8429267406463623,
      "step": 350
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.729103716819111e-08,
      "logits/chosen": -2.2340056896209717,
      "logits/rejected": -2.1630749702453613,
      "logps/chosen": -363.3587341308594,
      "logps/pi_response": -291.95538330078125,
      "logps/ref_response": -75.47894287109375,
      "logps/rejected": -398.1706237792969,
      "loss": 0.5352,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.259597659111023,
      "rewards/margins": 0.6464335322380066,
      "rewards/rejected": -1.9060310125350952,
      "step": 360
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.387025063449081e-08,
      "logits/chosen": -2.187648296356201,
      "logits/rejected": -2.1372499465942383,
      "logps/chosen": -342.2130432128906,
      "logps/pi_response": -296.50830078125,
      "logps/ref_response": -68.33134460449219,
      "logps/rejected": -397.110595703125,
      "loss": 0.5447,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.2424293756484985,
      "rewards/margins": 0.5943651795387268,
      "rewards/rejected": -1.8367946147918701,
      "step": 370
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.138919252022435e-08,
      "logits/chosen": -2.223222255706787,
      "logits/rejected": -2.1696248054504395,
      "logps/chosen": -345.4095764160156,
      "logps/pi_response": -297.59759521484375,
      "logps/ref_response": -72.3606948852539,
      "logps/rejected": -421.50531005859375,
      "loss": 0.5508,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.3772599697113037,
      "rewards/margins": 0.516662061214447,
      "rewards/rejected": -1.8939220905303955,
      "step": 380
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.991445467064689e-08,
      "logits/chosen": -2.2751657962799072,
      "logits/rejected": -2.190664529800415,
      "logps/chosen": -357.4648742675781,
      "logps/pi_response": -295.1226806640625,
      "logps/ref_response": -70.87272644042969,
      "logps/rejected": -415.9715270996094,
      "loss": 0.5434,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -1.3301585912704468,
      "rewards/margins": 0.6158719658851624,
      "rewards/rejected": -1.9460303783416748,
      "step": 390
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.9507259776993954e-08,
      "logits/chosen": -2.254105806350708,
      "logits/rejected": -2.2081995010375977,
      "logps/chosen": -366.28436279296875,
      "logps/pi_response": -304.9342041015625,
      "logps/ref_response": -77.48291015625,
      "logps/rejected": -364.3297119140625,
      "loss": 0.5429,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.3418512344360352,
      "rewards/margins": 0.46248334646224976,
      "rewards/rejected": -1.8043344020843506,
      "step": 400
    },
    {
      "epoch": 0.86,
      "learning_rate": 3.022313472693447e-08,
      "logits/chosen": -2.232144594192505,
      "logits/rejected": -2.1603074073791504,
      "logps/chosen": -369.909423828125,
      "logps/pi_response": -314.9814147949219,
      "logps/ref_response": -76.4972915649414,
      "logps/rejected": -432.63824462890625,
      "loss": 0.5482,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.456408977508545,
      "rewards/margins": 0.5390293598175049,
      "rewards/rejected": -1.9954382181167603,
      "step": 410
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.2111614344599684e-08,
      "logits/chosen": -2.268048048019409,
      "logits/rejected": -2.194788694381714,
      "logps/chosen": -351.391845703125,
      "logps/pi_response": -303.7065734863281,
      "logps/ref_response": -70.95718383789062,
      "logps/rejected": -420.17315673828125,
      "loss": 0.5353,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.3950040340423584,
      "rewards/margins": 0.5390046238899231,
      "rewards/rejected": -1.9340085983276367,
      "step": 420
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.521597710086439e-08,
      "logits/chosen": -2.256483554840088,
      "logits/rejected": -2.182900905609131,
      "logps/chosen": -394.15948486328125,
      "logps/pi_response": -322.7268981933594,
      "logps/ref_response": -71.76502990722656,
      "logps/rejected": -406.4312438964844,
      "loss": 0.5666,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.330580711364746,
      "rewards/margins": 0.6264187693595886,
      "rewards/rejected": -1.9569995403289795,
      "step": 430
    },
    {
      "epoch": 0.92,
      "learning_rate": 9.57301420397924e-09,
      "logits/chosen": -2.261594772338867,
      "logits/rejected": -2.219043016433716,
      "logps/chosen": -363.95025634765625,
      "logps/pi_response": -315.3779296875,
      "logps/ref_response": -77.31925201416016,
      "logps/rejected": -394.43280029296875,
      "loss": 0.5349,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.391911268234253,
      "rewards/margins": 0.5445317029953003,
      "rewards/rejected": -1.9364430904388428,
      "step": 440
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.212833302556258e-09,
      "logits/chosen": -2.2221949100494385,
      "logits/rejected": -2.142899513244629,
      "logps/chosen": -372.461181640625,
      "logps/pi_response": -308.6124572753906,
      "logps/ref_response": -75.69192504882812,
      "logps/rejected": -399.45892333984375,
      "loss": 0.5203,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.2634239196777344,
      "rewards/margins": 0.6645442247390747,
      "rewards/rejected": -1.9279682636260986,
      "step": 450
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.158697848236607e-09,
      "logits/chosen": -2.2366528511047363,
      "logits/rejected": -2.166773796081543,
      "logps/chosen": -369.5625915527344,
      "logps/pi_response": -296.8544006347656,
      "logps/ref_response": -69.99818420410156,
      "logps/rejected": -397.776123046875,
      "loss": 0.5432,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.3708467483520508,
      "rewards/margins": 0.6627798676490784,
      "rewards/rejected": -2.0336265563964844,
      "step": 460
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.269029751107489e-10,
      "logits/chosen": -2.2382636070251465,
      "logits/rejected": -2.1679019927978516,
      "logps/chosen": -345.45440673828125,
      "logps/pi_response": -300.89666748046875,
      "logps/ref_response": -71.26921081542969,
      "logps/rejected": -400.05804443359375,
      "loss": 0.5428,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -1.2617619037628174,
      "rewards/margins": 0.6952942609786987,
      "rewards/rejected": -1.9570560455322266,
      "step": 470
    },
    {
      "epoch": 1.0,
      "step": 478,
      "total_flos": 0.0,
      "train_loss": 0.583395419260448,
      "train_runtime": 12448.4703,
      "train_samples_per_second": 4.911,
      "train_steps_per_second": 0.038
    }
  ],
  "logging_steps": 10,
  "max_steps": 478,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}