{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 311,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.003215434083601286,
      "grad_norm": 79.87917506057431,
      "learning_rate": 5.3191489361702125e-09,
      "logits/chosen": -4.125,
      "logits/rejected": -3.765625,
      "logps/chosen": -132.0,
      "logps/rejected": -141.0,
      "loss": 0.6914,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03215434083601286,
      "grad_norm": 118.20552774316917,
      "learning_rate": 5.3191489361702123e-08,
      "logits/chosen": -4.15625,
      "logits/rejected": -3.8125,
      "logps/chosen": -114.0,
      "logps/rejected": -178.0,
      "loss": 0.6858,
      "rewards/accuracies": 0.2638888955116272,
      "rewards/chosen": 0.0086669921875,
      "rewards/margins": 0.00555419921875,
      "rewards/rejected": 0.00311279296875,
      "step": 10
    },
    {
      "epoch": 0.06430868167202572,
      "grad_norm": 66.66640901080964,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": -4.0,
      "logits/rejected": -3.8125,
      "logps/chosen": -122.0,
      "logps/rejected": -203.0,
      "loss": 0.6479,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.05615234375,
      "rewards/margins": 0.09423828125,
      "rewards/rejected": -0.0380859375,
      "step": 20
    },
    {
      "epoch": 0.09646302250803858,
      "grad_norm": 81.48311869513411,
      "learning_rate": 1.5957446808510638e-07,
      "logits/chosen": -4.03125,
      "logits/rejected": -3.84375,
      "logps/chosen": -142.0,
      "logps/rejected": -224.0,
      "loss": 0.5284,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.1962890625,
      "rewards/margins": 0.474609375,
      "rewards/rejected": -0.279296875,
      "step": 30
    },
    {
      "epoch": 0.12861736334405144,
      "grad_norm": 483.03383426380265,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": -3.921875,
      "logits/rejected": -3.859375,
      "logps/chosen": -130.0,
      "logps/rejected": -196.0,
      "loss": 0.4049,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.04638671875,
      "rewards/margins": 0.9296875,
      "rewards/rejected": -0.8828125,
      "step": 40
    },
    {
      "epoch": 0.1607717041800643,
      "grad_norm": 521.8067650467351,
      "learning_rate": 2.659574468085106e-07,
      "logits/chosen": -3.875,
      "logits/rejected": -3.90625,
      "logps/chosen": -130.0,
      "logps/rejected": -214.0,
      "loss": 0.4117,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.259765625,
      "rewards/margins": 2.078125,
      "rewards/rejected": -1.8203125,
      "step": 50
    },
    {
      "epoch": 0.19292604501607716,
      "grad_norm": 122.5164041936752,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": -3.90625,
      "logits/rejected": -3.953125,
      "logps/chosen": -116.0,
      "logps/rejected": -214.0,
      "loss": 0.2465,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.248046875,
      "rewards/margins": 2.34375,
      "rewards/rejected": -2.59375,
      "step": 60
    },
    {
      "epoch": 0.22508038585209003,
      "grad_norm": 98.58564731221452,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": -3.84375,
      "logits/rejected": -3.90625,
      "logps/chosen": -130.0,
      "logps/rejected": -274.0,
      "loss": 0.2316,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.1923828125,
      "rewards/margins": 3.359375,
      "rewards/rejected": -3.546875,
      "step": 70
    },
    {
      "epoch": 0.2572347266881029,
      "grad_norm": 58.684567343268625,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -3.6875,
      "logits/rejected": -3.828125,
      "logps/chosen": -144.0,
      "logps/rejected": -209.0,
      "loss": 0.3018,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.50390625,
      "rewards/margins": 2.765625,
      "rewards/rejected": -3.265625,
      "step": 80
    },
    {
      "epoch": 0.28938906752411575,
      "grad_norm": 44.1285015532565,
      "learning_rate": 4.787234042553192e-07,
      "logits/chosen": -3.484375,
      "logits/rejected": -3.65625,
      "logps/chosen": -121.0,
      "logps/rejected": -198.0,
      "loss": 0.3009,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.181640625,
      "rewards/margins": 2.4375,
      "rewards/rejected": -2.625,
      "step": 90
    },
    {
      "epoch": 0.3215434083601286,
      "grad_norm": 50.263563110645,
      "learning_rate": 4.964243146603099e-07,
      "logits/chosen": -3.5,
      "logits/rejected": -3.609375,
      "logps/chosen": -150.0,
      "logps/rejected": -222.0,
      "loss": 0.2831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -4.887580871582031e-05,
      "rewards/margins": 3.09375,
      "rewards/rejected": -3.09375,
      "step": 100
    },
    {
      "epoch": 0.3536977491961415,
      "grad_norm": 77.17996002738214,
      "learning_rate": 4.904648390941597e-07,
      "logits/chosen": -3.578125,
      "logits/rejected": -3.703125,
      "logps/chosen": -114.5,
      "logps/rejected": -233.0,
      "loss": 0.1551,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.0654296875,
      "rewards/margins": 3.828125,
      "rewards/rejected": -3.890625,
      "step": 110
    },
    {
      "epoch": 0.3858520900321543,
      "grad_norm": 50.629644956690505,
      "learning_rate": 4.845053635280095e-07,
      "logits/chosen": -3.59375,
      "logits/rejected": -3.8125,
      "logps/chosen": -120.5,
      "logps/rejected": -262.0,
      "loss": 0.1768,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.388671875,
      "rewards/margins": 4.15625,
      "rewards/rejected": -4.53125,
      "step": 120
    },
    {
      "epoch": 0.4180064308681672,
      "grad_norm": 87.75654163889804,
      "learning_rate": 4.785458879618593e-07,
      "logits/chosen": -3.484375,
      "logits/rejected": -3.640625,
      "logps/chosen": -153.0,
      "logps/rejected": -280.0,
      "loss": 0.1777,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.66796875,
      "rewards/margins": 4.25,
      "rewards/rejected": -4.90625,
      "step": 130
    },
    {
      "epoch": 0.45016077170418006,
      "grad_norm": 48.30101230741645,
      "learning_rate": 4.7258641239570916e-07,
      "logits/chosen": -3.640625,
      "logits/rejected": -3.765625,
      "logps/chosen": -119.0,
      "logps/rejected": -233.0,
      "loss": 0.1792,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.142578125,
      "rewards/margins": 3.859375,
      "rewards/rejected": -3.71875,
      "step": 140
    },
    {
      "epoch": 0.48231511254019294,
      "grad_norm": 160.46320495134455,
      "learning_rate": 4.66626936829559e-07,
      "logits/chosen": -3.75,
      "logits/rejected": -3.875,
      "logps/chosen": -111.0,
      "logps/rejected": -248.0,
      "loss": 0.3135,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.1611328125,
      "rewards/margins": 4.03125,
      "rewards/rejected": -3.875,
      "step": 150
    },
    {
      "epoch": 0.5144694533762058,
      "grad_norm": 27.202159849500408,
      "learning_rate": 4.606674612634088e-07,
      "logits/chosen": -3.5625,
      "logits/rejected": -3.765625,
      "logps/chosen": -132.0,
      "logps/rejected": -260.0,
      "loss": 0.1584,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.181640625,
      "rewards/margins": 4.34375,
      "rewards/rejected": -4.15625,
      "step": 160
    },
    {
      "epoch": 0.5466237942122186,
      "grad_norm": 59.16977770326313,
      "learning_rate": 4.547079856972586e-07,
      "logits/chosen": -3.734375,
      "logits/rejected": -3.796875,
      "logps/chosen": -140.0,
      "logps/rejected": -241.0,
      "loss": 0.1732,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.1953125,
      "rewards/margins": 3.8125,
      "rewards/rejected": -4.0,
      "step": 170
    },
    {
      "epoch": 0.5787781350482315,
      "grad_norm": 54.881650170057746,
      "learning_rate": 4.487485101311084e-07,
      "logits/chosen": -3.625,
      "logits/rejected": -3.75,
      "logps/chosen": -122.0,
      "logps/rejected": -239.0,
      "loss": 0.2094,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.58203125,
      "rewards/margins": 3.734375,
      "rewards/rejected": -4.34375,
      "step": 180
    },
    {
      "epoch": 0.6109324758842444,
      "grad_norm": 46.393130621023865,
      "learning_rate": 4.4278903456495827e-07,
      "logits/chosen": -3.5,
      "logits/rejected": -3.734375,
      "logps/chosen": -105.5,
      "logps/rejected": -248.0,
      "loss": 0.1894,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.248046875,
      "rewards/margins": 4.0,
      "rewards/rejected": -3.765625,
      "step": 190
    },
    {
      "epoch": 0.6430868167202572,
      "grad_norm": 61.77038241219252,
      "learning_rate": 4.368295589988081e-07,
      "logits/chosen": -3.546875,
      "logits/rejected": -3.609375,
      "logps/chosen": -121.5,
      "logps/rejected": -246.0,
      "loss": 0.2004,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.43359375,
      "rewards/margins": 3.84375,
      "rewards/rejected": -3.421875,
      "step": 200
    },
    {
      "epoch": 0.6752411575562701,
      "grad_norm": 54.13127427585697,
      "learning_rate": 4.308700834326579e-07,
      "logits/chosen": -3.546875,
      "logits/rejected": -3.8125,
      "logps/chosen": -134.0,
      "logps/rejected": -258.0,
      "loss": 0.1556,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.259765625,
      "rewards/margins": 4.46875,
      "rewards/rejected": -4.1875,
      "step": 210
    },
    {
      "epoch": 0.707395498392283,
      "grad_norm": 39.76866338516343,
      "learning_rate": 4.249106078665077e-07,
      "logits/chosen": -3.734375,
      "logits/rejected": -3.84375,
      "logps/chosen": -114.5,
      "logps/rejected": -254.0,
      "loss": 0.1614,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.41796875,
      "rewards/margins": 4.9375,
      "rewards/rejected": -4.53125,
      "step": 220
    },
    {
      "epoch": 0.7395498392282959,
      "grad_norm": 19.376631378140395,
      "learning_rate": 4.1895113230035757e-07,
      "logits/chosen": -3.65625,
      "logits/rejected": -3.703125,
      "logps/chosen": -125.5,
      "logps/rejected": -266.0,
      "loss": 0.2333,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.96484375,
      "rewards/margins": 4.5,
      "rewards/rejected": -3.515625,
      "step": 230
    },
    {
      "epoch": 0.7717041800643086,
      "grad_norm": 38.972752947728395,
      "learning_rate": 4.129916567342074e-07,
      "logits/chosen": -3.53125,
      "logits/rejected": -3.5625,
      "logps/chosen": -133.0,
      "logps/rejected": -266.0,
      "loss": 0.2215,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 1.46875,
      "rewards/margins": 5.03125,
      "rewards/rejected": -3.578125,
      "step": 240
    },
    {
      "epoch": 0.8038585209003215,
      "grad_norm": 25.549122278044088,
      "learning_rate": 4.070321811680572e-07,
      "logits/chosen": -3.546875,
      "logits/rejected": -3.671875,
      "logps/chosen": -139.0,
      "logps/rejected": -238.0,
      "loss": 0.1699,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.59375,
      "rewards/margins": 4.46875,
      "rewards/rejected": -5.09375,
      "step": 250
    },
    {
      "epoch": 0.8360128617363344,
      "grad_norm": 36.7792748916066,
      "learning_rate": 4.0107270560190706e-07,
      "logits/chosen": -3.640625,
      "logits/rejected": -3.84375,
      "logps/chosen": -125.0,
      "logps/rejected": -248.0,
      "loss": 0.1649,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.51171875,
      "rewards/margins": 4.53125,
      "rewards/rejected": -4.03125,
      "step": 260
    },
    {
      "epoch": 0.8681672025723473,
      "grad_norm": 10.885169604932974,
      "learning_rate": 3.9511323003575687e-07,
      "logits/chosen": -3.59375,
      "logits/rejected": -3.640625,
      "logps/chosen": -145.0,
      "logps/rejected": -246.0,
      "loss": 0.1556,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.396484375,
      "rewards/margins": 4.53125,
      "rewards/rejected": -4.125,
      "step": 270
    },
    {
      "epoch": 0.9003215434083601,
      "grad_norm": 33.68141687625864,
      "learning_rate": 3.8915375446960663e-07,
      "logits/chosen": -3.5625,
      "logits/rejected": -3.734375,
      "logps/chosen": -132.0,
      "logps/rejected": -276.0,
      "loss": 0.13,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.203125,
      "rewards/margins": 5.125,
      "rewards/rejected": -5.34375,
      "step": 280
    },
    {
      "epoch": 0.932475884244373,
      "grad_norm": 38.54727132055094,
      "learning_rate": 3.8319427890345644e-07,
      "logits/chosen": -3.453125,
      "logits/rejected": -3.578125,
      "logps/chosen": -136.0,
      "logps/rejected": -266.0,
      "loss": 0.1626,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1279296875,
      "rewards/margins": 4.96875,
      "rewards/rejected": -5.09375,
      "step": 290
    },
    {
      "epoch": 0.9646302250803859,
      "grad_norm": 33.100977527402115,
      "learning_rate": 3.772348033373063e-07,
      "logits/chosen": -3.4375,
      "logits/rejected": -3.65625,
      "logps/chosen": -137.0,
      "logps/rejected": -256.0,
      "loss": 0.1537,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.031982421875,
      "rewards/margins": 4.96875,
      "rewards/rejected": -4.9375,
      "step": 300
    },
    {
      "epoch": 0.9967845659163987,
      "grad_norm": 67.60299631407663,
      "learning_rate": 3.712753277711561e-07,
      "logits/chosen": -3.484375,
      "logits/rejected": -3.703125,
      "logps/chosen": -143.0,
      "logps/rejected": -296.0,
      "loss": 0.1769,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.10888671875,
      "rewards/margins": 5.4375,
      "rewards/rejected": -5.34375,
      "step": 310
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": -3.78125,
      "eval_logits/rejected": -3.78125,
      "eval_logps/chosen": -206.0,
      "eval_logps/rejected": -234.0,
      "eval_loss": 0.5869876742362976,
      "eval_rewards/accuracies": 0.7678571343421936,
      "eval_rewards/chosen": -2.078125,
      "eval_rewards/margins": 2.265625,
      "eval_rewards/rejected": -4.34375,
      "eval_runtime": 14.2416,
      "eval_samples_per_second": 13.973,
      "eval_steps_per_second": 0.492,
      "step": 311
    }
  ],
  "logging_steps": 10,
  "max_steps": 933,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}