{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9989258861439313,
  "eval_steps": 100000,
  "global_step": 465,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010741138560687433,
      "grad_norm": 94.9524373482681,
      "learning_rate": 8.51063829787234e-08,
      "logits/chosen": -10.583884239196777,
      "logits/rejected": -10.455678939819336,
      "logps/chosen": -0.9052008390426636,
      "logps/rejected": -0.8782350420951843,
      "loss": 5.2312,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -9.052008628845215,
      "rewards/margins": -0.26965802907943726,
      "rewards/rejected": -8.782350540161133,
      "step": 5
    },
    {
      "epoch": 0.021482277121374866,
      "grad_norm": 128.03935080093092,
      "learning_rate": 1.702127659574468e-07,
      "logits/chosen": -10.708091735839844,
      "logits/rejected": -10.848196983337402,
      "logps/chosen": -1.0050404071807861,
      "logps/rejected": -0.8861092329025269,
      "loss": 4.9303,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -10.05040454864502,
      "rewards/margins": -1.1893106698989868,
      "rewards/rejected": -8.861093521118164,
      "step": 10
    },
    {
      "epoch": 0.0322234156820623,
      "grad_norm": 57.90329446305289,
      "learning_rate": 2.553191489361702e-07,
      "logits/chosen": -10.310507774353027,
      "logits/rejected": -10.244686126708984,
      "logps/chosen": -1.086784839630127,
      "logps/rejected": -1.1507618427276611,
      "loss": 5.3365,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -10.867849349975586,
      "rewards/margins": 0.6397695541381836,
      "rewards/rejected": -11.507619857788086,
      "step": 15
    },
    {
      "epoch": 0.04296455424274973,
      "grad_norm": 130.56050724808273,
      "learning_rate": 3.404255319148936e-07,
      "logits/chosen": -9.948221206665039,
      "logits/rejected": -10.049638748168945,
      "logps/chosen": -0.959194540977478,
      "logps/rejected": -1.131405234336853,
      "loss": 4.6485,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -9.59194564819336,
      "rewards/margins": 1.722106695175171,
      "rewards/rejected": -11.31405258178711,
      "step": 20
    },
    {
      "epoch": 0.05370569280343716,
      "grad_norm": 90.96527999913032,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -10.305929183959961,
      "logits/rejected": -10.031614303588867,
      "logps/chosen": -0.8211612701416016,
      "logps/rejected": -1.1483596563339233,
      "loss": 4.3342,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -8.211612701416016,
      "rewards/margins": 3.271984815597534,
      "rewards/rejected": -11.483596801757812,
      "step": 25
    },
    {
      "epoch": 0.0644468313641246,
      "grad_norm": 79.34736484694697,
      "learning_rate": 5.106382978723404e-07,
      "logits/chosen": -9.271533966064453,
      "logits/rejected": -9.081380844116211,
      "logps/chosen": -0.6570690870285034,
      "logps/rejected": -0.6504842042922974,
      "loss": 4.1249,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -6.570690155029297,
      "rewards/margins": -0.06584861129522324,
      "rewards/rejected": -6.5048418045043945,
      "step": 30
    },
    {
      "epoch": 0.07518796992481203,
      "grad_norm": 91.87407769385476,
      "learning_rate": 5.957446808510638e-07,
      "logits/chosen": -8.827609062194824,
      "logits/rejected": -8.880338668823242,
      "logps/chosen": -0.7122071981430054,
      "logps/rejected": -0.6885973215103149,
      "loss": 4.2705,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -7.122071743011475,
      "rewards/margins": -0.23609896004199982,
      "rewards/rejected": -6.885972499847412,
      "step": 35
    },
    {
      "epoch": 0.08592910848549946,
      "grad_norm": 83.00868397694008,
      "learning_rate": 6.808510638297872e-07,
      "logits/chosen": -7.915566444396973,
      "logits/rejected": -7.83048152923584,
      "logps/chosen": -0.7321688532829285,
      "logps/rejected": -0.7351454496383667,
      "loss": 3.9545,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -7.321688175201416,
      "rewards/margins": 0.02976653352379799,
      "rewards/rejected": -7.351454734802246,
      "step": 40
    },
    {
      "epoch": 0.0966702470461869,
      "grad_norm": 53.570140237355766,
      "learning_rate": 7.659574468085107e-07,
      "logits/chosen": -8.591641426086426,
      "logits/rejected": -8.103092193603516,
      "logps/chosen": -0.5240384936332703,
      "logps/rejected": -0.5919655561447144,
      "loss": 3.6825,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -5.240384101867676,
      "rewards/margins": 0.6792702674865723,
      "rewards/rejected": -5.9196553230285645,
      "step": 45
    },
    {
      "epoch": 0.10741138560687433,
      "grad_norm": 57.10547378807199,
      "learning_rate": 7.998983280184396e-07,
      "logits/chosen": -8.613755226135254,
      "logits/rejected": -8.360788345336914,
      "logps/chosen": -0.500317394733429,
      "logps/rejected": -0.5585881471633911,
      "loss": 3.7313,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -5.003174781799316,
      "rewards/margins": 0.5827071070671082,
      "rewards/rejected": -5.585881233215332,
      "step": 50
    },
    {
      "epoch": 0.11815252416756176,
      "grad_norm": 41.56979433489102,
      "learning_rate": 7.992771864078597e-07,
      "logits/chosen": -7.88674783706665,
      "logits/rejected": -7.897039890289307,
      "logps/chosen": -0.5727618336677551,
      "logps/rejected": -0.6441472172737122,
      "loss": 3.7718,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -5.7276177406311035,
      "rewards/margins": 0.7138547301292419,
      "rewards/rejected": -6.44147253036499,
      "step": 55
    },
    {
      "epoch": 0.1288936627282492,
      "grad_norm": 56.57522168163828,
      "learning_rate": 7.980922636120897e-07,
      "logits/chosen": -8.342988967895508,
      "logits/rejected": -7.979635715484619,
      "logps/chosen": -0.5451933145523071,
      "logps/rejected": -0.6341482400894165,
      "loss": 3.6498,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -5.451932907104492,
      "rewards/margins": 0.8895494341850281,
      "rewards/rejected": -6.341482639312744,
      "step": 60
    },
    {
      "epoch": 0.13963480128893663,
      "grad_norm": 47.24775440564087,
      "learning_rate": 7.963452327474534e-07,
      "logits/chosen": -8.7212553024292,
      "logits/rejected": -8.58852767944336,
      "logps/chosen": -0.5589169859886169,
      "logps/rejected": -0.6501928567886353,
      "loss": 3.7606,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -5.589169979095459,
      "rewards/margins": 0.9127582311630249,
      "rewards/rejected": -6.501928806304932,
      "step": 65
    },
    {
      "epoch": 0.15037593984962405,
      "grad_norm": 49.929835506693976,
      "learning_rate": 7.940385606293987e-07,
      "logits/chosen": -8.28922176361084,
      "logits/rejected": -8.039751052856445,
      "logps/chosen": -0.6125788688659668,
      "logps/rejected": -0.6985718011856079,
      "loss": 3.6725,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -6.125788688659668,
      "rewards/margins": 0.8599298596382141,
      "rewards/rejected": -6.9857177734375,
      "step": 70
    },
    {
      "epoch": 0.1611170784103115,
      "grad_norm": 56.54270530915161,
      "learning_rate": 7.911755042893434e-07,
      "logits/chosen": -8.184160232543945,
      "logits/rejected": -8.052042961120605,
      "logps/chosen": -0.6090260148048401,
      "logps/rejected": -0.6563765406608582,
      "loss": 3.5589,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -6.090260028839111,
      "rewards/margins": 0.4735049307346344,
      "rewards/rejected": -6.563765525817871,
      "step": 75
    },
    {
      "epoch": 0.17185821697099893,
      "grad_norm": 60.92867617326722,
      "learning_rate": 7.877601063757321e-07,
      "logits/chosen": -8.475130081176758,
      "logits/rejected": -7.986093997955322,
      "logps/chosen": -0.6018053889274597,
      "logps/rejected": -0.7064691185951233,
      "loss": 3.5592,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -6.0180535316467285,
      "rewards/margins": 1.0466376543045044,
      "rewards/rejected": -7.064691066741943,
      "step": 80
    },
    {
      "epoch": 0.18259935553168635,
      "grad_norm": 44.52469580065135,
      "learning_rate": 7.837971894457989e-07,
      "logits/chosen": -8.589648246765137,
      "logits/rejected": -8.194600105285645,
      "logps/chosen": -0.6048508882522583,
      "logps/rejected": -0.6801942586898804,
      "loss": 3.7328,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -6.048508644104004,
      "rewards/margins": 0.753433346748352,
      "rewards/rejected": -6.801942348480225,
      "step": 85
    },
    {
      "epoch": 0.1933404940923738,
      "grad_norm": 51.57800146630992,
      "learning_rate": 7.792923491560942e-07,
      "logits/chosen": -7.764379978179932,
      "logits/rejected": -7.69348669052124,
      "logps/chosen": -0.5881169438362122,
      "logps/rejected": -0.6489121317863464,
      "loss": 3.5784,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -5.881168842315674,
      "rewards/margins": 0.6079525351524353,
      "rewards/rejected": -6.489121437072754,
      "step": 90
    },
    {
      "epoch": 0.20408163265306123,
      "grad_norm": 53.35932423879679,
      "learning_rate": 7.742519463613926e-07,
      "logits/chosen": -8.414074897766113,
      "logits/rejected": -8.186460494995117,
      "logps/chosen": -0.625275731086731,
      "logps/rejected": -0.6843087077140808,
      "loss": 3.4558,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -6.2527570724487305,
      "rewards/margins": 0.5903297662734985,
      "rewards/rejected": -6.843087196350098,
      "step": 95
    },
    {
      "epoch": 0.21482277121374865,
      "grad_norm": 159.6429414740936,
      "learning_rate": 7.68683098133138e-07,
      "logits/chosen": -8.126518249511719,
      "logits/rejected": -7.86532735824585,
      "logps/chosen": -0.6598135232925415,
      "logps/rejected": -0.795958399772644,
      "loss": 3.6034,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -6.598135948181152,
      "rewards/margins": 1.3614475727081299,
      "rewards/rejected": -7.959583282470703,
      "step": 100
    },
    {
      "epoch": 0.22556390977443608,
      "grad_norm": 101.4724702229996,
      "learning_rate": 7.625936677101051e-07,
      "logits/chosen": -7.772393703460693,
      "logits/rejected": -7.7723541259765625,
      "logps/chosen": -0.7960762977600098,
      "logps/rejected": -0.7650221586227417,
      "loss": 3.5703,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -7.960763454437256,
      "rewards/margins": -0.3105413317680359,
      "rewards/rejected": -7.6502227783203125,
      "step": 105
    },
    {
      "epoch": 0.23630504833512353,
      "grad_norm": 46.86893681454666,
      "learning_rate": 7.559922533954731e-07,
      "logits/chosen": -8.542976379394531,
      "logits/rejected": -8.485472679138184,
      "logps/chosen": -0.6914218068122864,
      "logps/rejected": -0.7978275418281555,
      "loss": 3.459,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -6.914217948913574,
      "rewards/margins": 1.064057469367981,
      "rewards/rejected": -7.978275299072266,
      "step": 110
    },
    {
      "epoch": 0.24704618689581095,
      "grad_norm": 58.65489916034957,
      "learning_rate": 7.488881764159808e-07,
      "logits/chosen": -8.630939483642578,
      "logits/rejected": -8.360986709594727,
      "logps/chosen": -0.6410632729530334,
      "logps/rejected": -0.7169302701950073,
      "loss": 3.3782,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -6.410633087158203,
      "rewards/margins": 0.7586696147918701,
      "rewards/rejected": -7.169302463531494,
      "step": 115
    },
    {
      "epoch": 0.2577873254564984,
      "grad_norm": 84.35506532206263,
      "learning_rate": 7.412914677603135e-07,
      "logits/chosen": -8.622358322143555,
      "logits/rejected": -8.39331340789795,
      "logps/chosen": -0.8245999217033386,
      "logps/rejected": -0.8862274289131165,
      "loss": 3.4276,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -8.245999336242676,
      "rewards/margins": 0.6162742376327515,
      "rewards/rejected": -8.862272262573242,
      "step": 120
    },
    {
      "epoch": 0.26852846401718583,
      "grad_norm": 53.225499002407155,
      "learning_rate": 7.332128540153017e-07,
      "logits/chosen": -9.33182144165039,
      "logits/rejected": -9.169477462768555,
      "logps/chosen": -0.6887825727462769,
      "logps/rejected": -0.8196980357170105,
      "loss": 3.3163,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -6.887825012207031,
      "rewards/margins": 1.3091557025909424,
      "rewards/rejected": -8.196981430053711,
      "step": 125
    },
    {
      "epoch": 0.27926960257787325,
      "grad_norm": 50.64163791482278,
      "learning_rate": 7.246637422199322e-07,
      "logits/chosen": -9.222763061523438,
      "logits/rejected": -9.078741073608398,
      "logps/chosen": -0.7283066511154175,
      "logps/rejected": -0.8620139956474304,
      "loss": 3.3182,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -7.283067226409912,
      "rewards/margins": 1.3370740413665771,
      "rewards/rejected": -8.62014102935791,
      "step": 130
    },
    {
      "epoch": 0.2900107411385607,
      "grad_norm": 53.65783420701467,
      "learning_rate": 7.156562037585574e-07,
      "logits/chosen": -9.75479793548584,
      "logits/rejected": -9.333330154418945,
      "logps/chosen": -0.7047444581985474,
      "logps/rejected": -0.9013818502426147,
      "loss": 3.2484,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -7.0474443435668945,
      "rewards/margins": 1.9663751125335693,
      "rewards/rejected": -9.013819694519043,
      "step": 135
    },
    {
      "epoch": 0.3007518796992481,
      "grad_norm": 65.5532547328958,
      "learning_rate": 7.062029573160467e-07,
      "logits/chosen": -9.653564453125,
      "logits/rejected": -9.361381530761719,
      "logps/chosen": -0.7335777878761292,
      "logps/rejected": -0.8712660074234009,
      "loss": 3.0731,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -7.335777282714844,
      "rewards/margins": 1.3768823146820068,
      "rewards/rejected": -8.71265983581543,
      "step": 140
    },
    {
      "epoch": 0.31149301825993553,
      "grad_norm": 85.95758082442647,
      "learning_rate": 6.963173509189455e-07,
      "logits/chosen": -10.888960838317871,
      "logits/rejected": -10.63546371459961,
      "logps/chosen": -0.8133164644241333,
      "logps/rejected": -0.9737753868103027,
      "loss": 3.1647,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -8.13316535949707,
      "rewards/margins": 1.604588508605957,
      "rewards/rejected": -9.737752914428711,
      "step": 145
    },
    {
      "epoch": 0.322234156820623,
      "grad_norm": 75.69682236302825,
      "learning_rate": 6.860133430880024e-07,
      "logits/chosen": -11.62391471862793,
      "logits/rejected": -11.238265991210938,
      "logps/chosen": -0.9367789030075073,
      "logps/rejected": -1.1026126146316528,
      "loss": 3.1205,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -9.367789268493652,
      "rewards/margins": 1.6583366394042969,
      "rewards/rejected": -11.02612590789795,
      "step": 150
    },
    {
      "epoch": 0.33297529538131043,
      "grad_norm": 81.01141627000389,
      "learning_rate": 6.753054831286747e-07,
      "logits/chosen": -12.134136199951172,
      "logits/rejected": -11.972139358520508,
      "logps/chosen": -1.021984338760376,
      "logps/rejected": -1.1821359395980835,
      "loss": 3.2077,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -10.219841957092285,
      "rewards/margins": 1.6015164852142334,
      "rewards/rejected": -11.821358680725098,
      "step": 155
    },
    {
      "epoch": 0.34371643394199786,
      "grad_norm": 77.94551535345828,
      "learning_rate": 6.642088905874433e-07,
      "logits/chosen": -11.634748458862305,
      "logits/rejected": -11.425237655639648,
      "logps/chosen": -1.0143425464630127,
      "logps/rejected": -1.1674789190292358,
      "loss": 3.1419,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -10.143426895141602,
      "rewards/margins": 1.5313634872436523,
      "rewards/rejected": -11.674790382385254,
      "step": 160
    },
    {
      "epoch": 0.3544575725026853,
      "grad_norm": 70.1225669816486,
      "learning_rate": 6.527392339029455e-07,
      "logits/chosen": -11.960525512695312,
      "logits/rejected": -11.831192016601562,
      "logps/chosen": -0.9272716641426086,
      "logps/rejected": -1.1244293451309204,
      "loss": 2.9347,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -9.272717475891113,
      "rewards/margins": 1.9715774059295654,
      "rewards/rejected": -11.244295120239258,
      "step": 165
    },
    {
      "epoch": 0.3651987110633727,
      "grad_norm": 91.08392173832057,
      "learning_rate": 6.409127082820689e-07,
      "logits/chosen": -11.943336486816406,
      "logits/rejected": -11.900778770446777,
      "logps/chosen": -1.1345242261886597,
      "logps/rejected": -1.2922136783599854,
      "loss": 3.0417,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -11.345243453979492,
      "rewards/margins": 1.5768930912017822,
      "rewards/rejected": -12.922136306762695,
      "step": 170
    },
    {
      "epoch": 0.37593984962406013,
      "grad_norm": 96.01506406008006,
      "learning_rate": 6.287460128322457e-07,
      "logits/chosen": -12.159789085388184,
      "logits/rejected": -11.930963516235352,
      "logps/chosen": -1.180555820465088,
      "logps/rejected": -1.4784237146377563,
      "loss": 2.8224,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -11.805558204650879,
      "rewards/margins": 2.9786787033081055,
      "rewards/rejected": -14.7842378616333,
      "step": 175
    },
    {
      "epoch": 0.3866809881847476,
      "grad_norm": 134.327125951201,
      "learning_rate": 6.16256326982239e-07,
      "logits/chosen": -14.223356246948242,
      "logits/rejected": -14.133268356323242,
      "logps/chosen": -1.1939128637313843,
      "logps/rejected": -1.450350046157837,
      "loss": 2.8145,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -11.939129829406738,
      "rewards/margins": 2.5643723011016846,
      "rewards/rejected": -14.503501892089844,
      "step": 180
    },
    {
      "epoch": 0.39742212674543503,
      "grad_norm": 83.18554015763489,
      "learning_rate": 6.034612862247114e-07,
      "logits/chosen": -12.888773918151855,
      "logits/rejected": -12.586360931396484,
      "logps/chosen": -1.2202188968658447,
      "logps/rejected": -1.4362331628799438,
      "loss": 2.5759,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -12.202188491821289,
      "rewards/margins": 2.160142421722412,
      "rewards/rejected": -14.362332344055176,
      "step": 185
    },
    {
      "epoch": 0.40816326530612246,
      "grad_norm": 89.84068475608692,
      "learning_rate": 5.903789572148295e-07,
      "logits/chosen": -13.51233196258545,
      "logits/rejected": -12.938328742980957,
      "logps/chosen": -1.2647783756256104,
      "logps/rejected": -1.6234039068222046,
      "loss": 2.7467,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -12.647781372070312,
      "rewards/margins": 3.5862574577331543,
      "rewards/rejected": -16.234039306640625,
      "step": 190
    },
    {
      "epoch": 0.4189044038668099,
      "grad_norm": 99.44799935907999,
      "learning_rate": 5.770278122600662e-07,
      "logits/chosen": -13.629679679870605,
      "logits/rejected": -13.426033020019531,
      "logps/chosen": -1.2931153774261475,
      "logps/rejected": -1.5686538219451904,
      "loss": 2.8854,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -12.93115234375,
      "rewards/margins": 2.7553839683532715,
      "rewards/rejected": -15.68653678894043,
      "step": 195
    },
    {
      "epoch": 0.4296455424274973,
      "grad_norm": 89.61158465486228,
      "learning_rate": 5.634267032372192e-07,
      "logits/chosen": -13.519218444824219,
      "logits/rejected": -13.498291015625,
      "logps/chosen": -1.317453145980835,
      "logps/rejected": -1.579171895980835,
      "loss": 2.7892,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -13.174531936645508,
      "rewards/margins": 2.617187261581421,
      "rewards/rejected": -15.791720390319824,
      "step": 200
    },
    {
      "epoch": 0.44038668098818473,
      "grad_norm": 102.3200615771371,
      "learning_rate": 5.495948349734758e-07,
      "logits/chosen": -13.04889965057373,
      "logits/rejected": -12.813755989074707,
      "logps/chosen": -1.4210269451141357,
      "logps/rejected": -1.681862473487854,
      "loss": 2.5788,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -14.210270881652832,
      "rewards/margins": 2.6083550453186035,
      "rewards/rejected": -16.81862449645996,
      "step": 205
    },
    {
      "epoch": 0.45112781954887216,
      "grad_norm": 90.0090654846689,
      "learning_rate": 5.355517381291105e-07,
      "logits/chosen": -14.2084321975708,
      "logits/rejected": -13.401834487915039,
      "logps/chosen": -1.5185084342956543,
      "logps/rejected": -1.9036325216293335,
      "loss": 2.6169,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -15.185084342956543,
      "rewards/margins": 3.8512394428253174,
      "rewards/rejected": -19.03632354736328,
      "step": 210
    },
    {
      "epoch": 0.46186895810955964,
      "grad_norm": 119.47997160045188,
      "learning_rate": 5.21317241620105e-07,
      "logits/chosen": -15.774456977844238,
      "logits/rejected": -15.285980224609375,
      "logps/chosen": -1.56905198097229,
      "logps/rejected": -1.8608026504516602,
      "loss": 2.8024,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -15.690518379211426,
      "rewards/margins": 2.9175078868865967,
      "rewards/rejected": -18.608028411865234,
      "step": 215
    },
    {
      "epoch": 0.47261009667024706,
      "grad_norm": 103.69035293278027,
      "learning_rate": 5.069114446196291e-07,
      "logits/chosen": -13.057888984680176,
      "logits/rejected": -12.574568748474121,
      "logps/chosen": -1.5152817964553833,
      "logps/rejected": -1.970833420753479,
      "loss": 2.5514,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -15.152816772460938,
      "rewards/margins": 4.55551815032959,
      "rewards/rejected": -19.70833396911621,
      "step": 220
    },
    {
      "epoch": 0.4833512352309345,
      "grad_norm": 93.86943383156643,
      "learning_rate": 4.923546881779183e-07,
      "logits/chosen": -14.003133773803711,
      "logits/rejected": -13.830352783203125,
      "logps/chosen": -1.266321063041687,
      "logps/rejected": -1.6633758544921875,
      "loss": 2.5453,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -12.663209915161133,
      "rewards/margins": 3.9705474376678467,
      "rewards/rejected": -16.633756637573242,
      "step": 225
    },
    {
      "epoch": 0.4940923737916219,
      "grad_norm": 98.47277489243837,
      "learning_rate": 4.776675265006186e-07,
      "logits/chosen": -13.277814865112305,
      "logits/rejected": -13.189724922180176,
      "logps/chosen": -1.3579418659210205,
      "logps/rejected": -1.6851608753204346,
      "loss": 2.6281,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -13.57941722869873,
      "rewards/margins": 3.272188901901245,
      "rewards/rejected": -16.851604461669922,
      "step": 230
    },
    {
      "epoch": 0.5048335123523093,
      "grad_norm": 180.03604900264963,
      "learning_rate": 4.62870697926156e-07,
      "logits/chosen": -13.268026351928711,
      "logits/rejected": -13.710092544555664,
      "logps/chosen": -1.4047057628631592,
      "logps/rejected": -1.6883337497711182,
      "loss": 2.6966,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -14.047056198120117,
      "rewards/margins": 2.8362793922424316,
      "rewards/rejected": -16.883337020874023,
      "step": 235
    },
    {
      "epoch": 0.5155746509129968,
      "grad_norm": 97.18499449175745,
      "learning_rate": 4.479850956431092e-07,
      "logits/chosen": -12.822425842285156,
      "logits/rejected": -12.987892150878906,
      "logps/chosen": -1.384665846824646,
      "logps/rejected": -1.6498899459838867,
      "loss": 2.4794,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -13.846659660339355,
      "rewards/margins": 2.6522395610809326,
      "rewards/rejected": -16.4989013671875,
      "step": 240
    },
    {
      "epoch": 0.5263157894736842,
      "grad_norm": 82.02645776585123,
      "learning_rate": 4.33031738188933e-07,
      "logits/chosen": -13.671134948730469,
      "logits/rejected": -13.073837280273438,
      "logps/chosen": -1.3697092533111572,
      "logps/rejected": -1.6771332025527954,
      "loss": 2.6649,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -13.69709300994873,
      "rewards/margins": 3.0742411613464355,
      "rewards/rejected": -16.771333694458008,
      "step": 245
    },
    {
      "epoch": 0.5370569280343717,
      "grad_norm": 94.06073499890805,
      "learning_rate": 4.180317397716889e-07,
      "logits/chosen": -13.844853401184082,
      "logits/rejected": -13.441442489624023,
      "logps/chosen": -1.3420655727386475,
      "logps/rejected": -1.8319076299667358,
      "loss": 2.5168,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -13.420656204223633,
      "rewards/margins": 4.898420810699463,
      "rewards/rejected": -18.319074630737305,
      "step": 250
    },
    {
      "epoch": 0.547798066595059,
      "grad_norm": 88.92677213863453,
      "learning_rate": 4.030062804566888e-07,
      "logits/chosen": -13.760513305664062,
      "logits/rejected": -13.63580322265625,
      "logps/chosen": -1.3302006721496582,
      "logps/rejected": -1.5523545742034912,
      "loss": 2.454,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -13.30200481414795,
      "rewards/margins": 2.2215399742126465,
      "rewards/rejected": -15.52354621887207,
      "step": 255
    },
    {
      "epoch": 0.5585392051557465,
      "grad_norm": 98.66187124678643,
      "learning_rate": 3.8797657626014614e-07,
      "logits/chosen": -14.272377014160156,
      "logits/rejected": -14.081184387207031,
      "logps/chosen": -1.380049467086792,
      "logps/rejected": -1.664563775062561,
      "loss": 2.6009,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -13.800496101379395,
      "rewards/margins": 2.8451404571533203,
      "rewards/rejected": -16.6456356048584,
      "step": 260
    },
    {
      "epoch": 0.569280343716434,
      "grad_norm": 100.86333922679928,
      "learning_rate": 3.729638491920669e-07,
      "logits/chosen": -12.846217155456543,
      "logits/rejected": -12.883219718933105,
      "logps/chosen": -1.294693946838379,
      "logps/rejected": -1.537477731704712,
      "loss": 2.5505,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -12.946939468383789,
      "rewards/margins": 2.4278392791748047,
      "rewards/rejected": -15.374776840209961,
      "step": 265
    },
    {
      "epoch": 0.5800214822771214,
      "grad_norm": 68.23752369485578,
      "learning_rate": 3.5798929729067464e-07,
      "logits/chosen": -14.66405963897705,
      "logits/rejected": -14.210983276367188,
      "logps/chosen": -1.4762803316116333,
      "logps/rejected": -1.8023189306259155,
      "loss": 2.2357,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -14.76280403137207,
      "rewards/margins": 3.260385513305664,
      "rewards/rejected": -18.023189544677734,
      "step": 270
    },
    {
      "epoch": 0.5907626208378088,
      "grad_norm": 91.0514861494684,
      "learning_rate": 3.4307406469068595e-07,
      "logits/chosen": -14.60252571105957,
      "logits/rejected": -14.498603820800781,
      "logps/chosen": -1.5057706832885742,
      "logps/rejected": -1.9537569284439087,
      "loss": 2.3592,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -15.057708740234375,
      "rewards/margins": 4.47986364364624,
      "rewards/rejected": -19.53757095336914,
      "step": 275
    },
    {
      "epoch": 0.6015037593984962,
      "grad_norm": 83.64718344441454,
      "learning_rate": 3.282392117676968e-07,
      "logits/chosen": -14.025052070617676,
      "logits/rejected": -14.093378067016602,
      "logps/chosen": -1.6209542751312256,
      "logps/rejected": -2.0730650424957275,
      "loss": 2.388,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -16.209543228149414,
      "rewards/margins": 4.5211076736450195,
      "rewards/rejected": -20.730648040771484,
      "step": 280
    },
    {
      "epoch": 0.6122448979591837,
      "grad_norm": 93.65673600303997,
      "learning_rate": 3.135056854008371e-07,
      "logits/chosen": -15.082783699035645,
      "logits/rejected": -14.919405937194824,
      "logps/chosen": -1.5184903144836426,
      "logps/rejected": -1.8643596172332764,
      "loss": 2.3229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -15.184903144836426,
      "rewards/margins": 3.458693742752075,
      "rewards/rejected": -18.643596649169922,
      "step": 285
    },
    {
      "epoch": 0.6229860365198711,
      "grad_norm": 94.14225077358178,
      "learning_rate": 2.988942893956833e-07,
      "logits/chosen": -14.259645462036133,
      "logits/rejected": -14.12261962890625,
      "logps/chosen": -1.5351895093917847,
      "logps/rejected": -1.9333627223968506,
      "loss": 2.4519,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -15.351896286010742,
      "rewards/margins": 3.9817326068878174,
      "rewards/rejected": -19.333629608154297,
      "step": 290
    },
    {
      "epoch": 0.6337271750805585,
      "grad_norm": 91.51342112280236,
      "learning_rate": 2.844256551091911e-07,
      "logits/chosen": -15.610522270202637,
      "logits/rejected": -15.578115463256836,
      "logps/chosen": -1.5930521488189697,
      "logps/rejected": -2.037524700164795,
      "loss": 2.322,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -15.930521965026855,
      "rewards/margins": 4.444725036621094,
      "rewards/rejected": -20.375247955322266,
      "step": 295
    },
    {
      "epoch": 0.644468313641246,
      "grad_norm": 126.04770140963556,
      "learning_rate": 2.7012021231812664e-07,
      "logits/chosen": -15.719934463500977,
      "logits/rejected": -15.443829536437988,
      "logps/chosen": -1.6209602355957031,
      "logps/rejected": -1.8999830484390259,
      "loss": 2.6214,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -16.209604263305664,
      "rewards/margins": 2.7902257442474365,
      "rewards/rejected": -18.999828338623047,
      "step": 300
    },
    {
      "epoch": 0.6552094522019334,
      "grad_norm": 92.42623141388283,
      "learning_rate": 2.5599816037212954e-07,
      "logits/chosen": -13.97626781463623,
      "logits/rejected": -13.821372985839844,
      "logps/chosen": -1.5213415622711182,
      "logps/rejected": -1.976249098777771,
      "loss": 2.3217,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -15.213415145874023,
      "rewards/margins": 4.549073219299316,
      "rewards/rejected": -19.76249122619629,
      "step": 305
    },
    {
      "epoch": 0.6659505907626209,
      "grad_norm": 107.38236836802639,
      "learning_rate": 2.4207943967214064e-07,
      "logits/chosen": -15.55937671661377,
      "logits/rejected": -15.249128341674805,
      "logps/chosen": -1.672916054725647,
      "logps/rejected": -2.119063138961792,
      "loss": 2.4341,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -16.72916030883789,
      "rewards/margins": 4.461469650268555,
      "rewards/rejected": -21.190631866455078,
      "step": 310
    },
    {
      "epoch": 0.6766917293233082,
      "grad_norm": 82.15179993575198,
      "learning_rate": 2.2838370351446547e-07,
      "logits/chosen": -15.349655151367188,
      "logits/rejected": -15.129667282104492,
      "logps/chosen": -1.478963851928711,
      "logps/rejected": -1.8934085369110107,
      "loss": 2.2252,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -14.789637565612793,
      "rewards/margins": 4.1444501876831055,
      "rewards/rejected": -18.9340877532959,
      "step": 315
    },
    {
      "epoch": 0.6874328678839957,
      "grad_norm": 97.61594439863894,
      "learning_rate": 2.1493029034023188e-07,
      "logits/chosen": -14.922358512878418,
      "logits/rejected": -14.824201583862305,
      "logps/chosen": -1.480419635772705,
      "logps/rejected": -1.897552490234375,
      "loss": 2.3422,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -14.804197311401367,
      "rewards/margins": 4.171327114105225,
      "rewards/rejected": -18.97552490234375,
      "step": 320
    },
    {
      "epoch": 0.6981740064446831,
      "grad_norm": 129.6217645662466,
      "learning_rate": 2.0173819642942376e-07,
      "logits/chosen": -14.275545120239258,
      "logits/rejected": -14.210858345031738,
      "logps/chosen": -1.6476881504058838,
      "logps/rejected": -2.2318155765533447,
      "loss": 2.4467,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -16.476879119873047,
      "rewards/margins": 5.841276168823242,
      "rewards/rejected": -22.318157196044922,
      "step": 325
    },
    {
      "epoch": 0.7089151450053706,
      "grad_norm": 102.73687963051421,
      "learning_rate": 1.888260490780485e-07,
      "logits/chosen": -13.992073059082031,
      "logits/rejected": -14.150375366210938,
      "logps/chosen": -1.4686782360076904,
      "logps/rejected": -1.8977546691894531,
      "loss": 2.4571,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -14.68678092956543,
      "rewards/margins": 4.29076623916626,
      "rewards/rejected": -18.977548599243164,
      "step": 330
    },
    {
      "epoch": 0.719656283566058,
      "grad_norm": 96.36528482116944,
      "learning_rate": 1.7621208029631078e-07,
      "logits/chosen": -14.44189167022705,
      "logits/rejected": -14.381067276000977,
      "logps/chosen": -1.6128368377685547,
      "logps/rejected": -2.1022796630859375,
      "loss": 2.3562,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -16.128366470336914,
      "rewards/margins": 4.894428253173828,
      "rewards/rejected": -21.022794723510742,
      "step": 335
    },
    {
      "epoch": 0.7303974221267454,
      "grad_norm": 94.96432834836168,
      "learning_rate": 1.6391410106493227e-07,
      "logits/chosen": -14.606742858886719,
      "logits/rejected": -14.417033195495605,
      "logps/chosen": -1.666783332824707,
      "logps/rejected": -2.13419508934021,
      "loss": 2.3525,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -16.66783332824707,
      "rewards/margins": 4.674118995666504,
      "rewards/rejected": -21.341951370239258,
      "step": 340
    },
    {
      "epoch": 0.7411385606874329,
      "grad_norm": 84.8009752822123,
      "learning_rate": 1.5194947618596673e-07,
      "logits/chosen": -14.718609809875488,
      "logits/rejected": -14.481607437133789,
      "logps/chosen": -1.585702657699585,
      "logps/rejected": -1.9948770999908447,
      "loss": 2.3799,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -15.857027053833008,
      "rewards/margins": 4.0917463302612305,
      "rewards/rejected": -19.948772430419922,
      "step": 345
    },
    {
      "epoch": 0.7518796992481203,
      "grad_norm": 91.67894623013964,
      "learning_rate": 1.4033509976362083e-07,
      "logits/chosen": -15.412130355834961,
      "logits/rejected": -15.329202651977539,
      "logps/chosen": -1.5967485904693604,
      "logps/rejected": -1.9570953845977783,
      "loss": 2.3651,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -15.967483520507812,
      "rewards/margins": 3.6034674644470215,
      "rewards/rejected": -19.570951461791992,
      "step": 350
    },
    {
      "epoch": 0.7626208378088077,
      "grad_norm": 90.74677011249962,
      "learning_rate": 1.2908737134970363e-07,
      "logits/chosen": -13.989651679992676,
      "logits/rejected": -14.033552169799805,
      "logps/chosen": -1.5832325220108032,
      "logps/rejected": -2.131143808364868,
      "loss": 2.4523,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -15.832324028015137,
      "rewards/margins": 5.479111671447754,
      "rewards/rejected": -21.31143569946289,
      "step": 355
    },
    {
      "epoch": 0.7733619763694952,
      "grad_norm": 167.14346795521232,
      "learning_rate": 1.1822217278738515e-07,
      "logits/chosen": -15.21629810333252,
      "logits/rejected": -15.203242301940918,
      "logps/chosen": -1.64801824092865,
      "logps/rejected": -2.112056255340576,
      "loss": 2.4498,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -16.480182647705078,
      "rewards/margins": 4.640378952026367,
      "rewards/rejected": -21.120563507080078,
      "step": 360
    },
    {
      "epoch": 0.7841031149301826,
      "grad_norm": 118.08524484358777,
      "learning_rate": 1.0775484578596241e-07,
      "logits/chosen": -15.109689712524414,
      "logits/rejected": -15.07580280303955,
      "logps/chosen": -1.6311914920806885,
      "logps/rejected": -2.166144609451294,
      "loss": 2.1476,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -16.311912536621094,
      "rewards/margins": 5.349532604217529,
      "rewards/rejected": -21.661447525024414,
      "step": 365
    },
    {
      "epoch": 0.7948442534908701,
      "grad_norm": 106.39619751867055,
      "learning_rate": 9.770017025829673e-08,
      "logits/chosen": -15.397229194641113,
      "logits/rejected": -15.390848159790039,
      "logps/chosen": -1.798893690109253,
      "logps/rejected": -2.2561848163604736,
      "loss": 2.1214,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -17.988937377929688,
      "rewards/margins": 4.572911262512207,
      "rewards/rejected": -22.561847686767578,
      "step": 370
    },
    {
      "epoch": 0.8055853920515574,
      "grad_norm": 93.15779349982272,
      "learning_rate": 8.807234345151027e-08,
      "logits/chosen": -14.388795852661133,
      "logits/rejected": -14.405484199523926,
      "logps/chosen": -1.7484766244888306,
      "logps/rejected": -2.2885029315948486,
      "loss": 2.309,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -17.484766006469727,
      "rewards/margins": 5.40026330947876,
      "rewards/rejected": -22.885028839111328,
      "step": 375
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 167.5169304145404,
      "learning_rate": 7.888495990040924e-08,
      "logits/chosen": -13.293497085571289,
      "logits/rejected": -13.353901863098145,
      "logps/chosen": -1.68817937374115,
      "logps/rejected": -2.3241219520568848,
      "loss": 2.4447,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -16.881792068481445,
      "rewards/margins": 6.3594279289245605,
      "rewards/rejected": -23.241220474243164,
      "step": 380
    },
    {
      "epoch": 0.8270676691729323,
      "grad_norm": 111.15424994986633,
      "learning_rate": 7.015099223193943e-08,
      "logits/chosen": -15.060186386108398,
      "logits/rejected": -15.154025077819824,
      "logps/chosen": -1.6439955234527588,
      "logps/rejected": -2.1140053272247314,
      "loss": 2.2619,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -16.439952850341797,
      "rewards/margins": 4.700098991394043,
      "rewards/rejected": -21.140052795410156,
      "step": 385
    },
    {
      "epoch": 0.8378088077336198,
      "grad_norm": 101.55885007809343,
      "learning_rate": 6.188277284777857e-08,
      "logits/chosen": -14.038566589355469,
      "logits/rejected": -13.341252326965332,
      "logps/chosen": -1.694256067276001,
      "logps/rejected": -2.257657766342163,
      "loss": 2.1908,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -16.942562103271484,
      "rewards/margins": 5.634018898010254,
      "rewards/rejected": -22.576580047607422,
      "step": 390
    },
    {
      "epoch": 0.8485499462943072,
      "grad_norm": 106.94728529442926,
      "learning_rate": 5.409197651092965e-08,
      "logits/chosen": -15.331846237182617,
      "logits/rejected": -15.203790664672852,
      "logps/chosen": -1.8188066482543945,
      "logps/rejected": -2.247000217437744,
      "loss": 2.207,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -18.188068389892578,
      "rewards/margins": 4.281935691833496,
      "rewards/rejected": -22.47000503540039,
      "step": 395
    },
    {
      "epoch": 0.8592910848549946,
      "grad_norm": 98.96501941427364,
      "learning_rate": 4.678960386090298e-08,
      "logits/chosen": -14.686403274536133,
      "logits/rejected": -14.618772506713867,
      "logps/chosen": -1.6232935190200806,
      "logps/rejected": -2.1728694438934326,
      "loss": 2.1576,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -16.232933044433594,
      "rewards/margins": 5.495760440826416,
      "rewards/rejected": -21.728694915771484,
      "step": 400
    },
    {
      "epoch": 0.8700322234156821,
      "grad_norm": 105.08036258776926,
      "learning_rate": 3.998596588076366e-08,
      "logits/chosen": -13.175695419311523,
      "logits/rejected": -13.03925609588623,
      "logps/chosen": -1.7160142660140991,
      "logps/rejected": -2.047438621520996,
      "loss": 2.4737,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -17.16014289855957,
      "rewards/margins": 3.314243793487549,
      "rewards/rejected": -20.47438621520996,
      "step": 405
    },
    {
      "epoch": 0.8807733619763695,
      "grad_norm": 111.72734645657039,
      "learning_rate": 3.3690669337976996e-08,
      "logits/chosen": -14.5875883102417,
      "logits/rejected": -14.465304374694824,
      "logps/chosen": -1.566119909286499,
      "logps/rejected": -1.9680745601654053,
      "loss": 2.1427,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -15.661198616027832,
      "rewards/margins": 4.019545078277588,
      "rewards/rejected": -19.680744171142578,
      "step": 410
    },
    {
      "epoch": 0.8915145005370569,
      "grad_norm": 109.73987252534666,
      "learning_rate": 2.7912603219609798e-08,
      "logits/chosen": -15.08208179473877,
      "logits/rejected": -15.053112983703613,
      "logps/chosen": -1.7169599533081055,
      "logps/rejected": -2.092921733856201,
      "loss": 2.162,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -17.169597625732422,
      "rewards/margins": 3.759619951248169,
      "rewards/rejected": -20.929218292236328,
      "step": 415
    },
    {
      "epoch": 0.9022556390977443,
      "grad_norm": 109.40686784832717,
      "learning_rate": 2.265992618104029e-08,
      "logits/chosen": -15.328409194946289,
      "logits/rejected": -15.404211044311523,
      "logps/chosen": -1.7850490808486938,
      "logps/rejected": -2.2638847827911377,
      "loss": 2.2059,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -17.85049057006836,
      "rewards/margins": 4.788357734680176,
      "rewards/rejected": -22.63884735107422,
      "step": 420
    },
    {
      "epoch": 0.9129967776584318,
      "grad_norm": 116.29528387151561,
      "learning_rate": 1.7940055025900304e-08,
      "logits/chosen": -13.697789192199707,
      "logits/rejected": -13.522615432739258,
      "logps/chosen": -1.7403781414031982,
      "logps/rejected": -2.0932869911193848,
      "loss": 2.4036,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -17.40378189086914,
      "rewards/margins": 3.529087781906128,
      "rewards/rejected": -20.932870864868164,
      "step": 425
    },
    {
      "epoch": 0.9237379162191193,
      "grad_norm": 96.25495308785165,
      "learning_rate": 1.3759654233514817e-08,
      "logits/chosen": -14.403491020202637,
      "logits/rejected": -14.36598014831543,
      "logps/chosen": -1.6513020992279053,
      "logps/rejected": -2.098128318786621,
      "loss": 2.2317,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -16.513019561767578,
      "rewards/margins": 4.468262672424316,
      "rewards/rejected": -20.981285095214844,
      "step": 430
    },
    {
      "epoch": 0.9344790547798066,
      "grad_norm": 97.45998752235127,
      "learning_rate": 1.0124626548627402e-08,
      "logits/chosen": -14.93018913269043,
      "logits/rejected": -15.036625862121582,
      "logps/chosen": -1.7160346508026123,
      "logps/rejected": -2.3369088172912598,
      "loss": 2.1682,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -17.16034507751465,
      "rewards/margins": 6.208739280700684,
      "rewards/rejected": -23.36908531188965,
      "step": 435
    },
    {
      "epoch": 0.9452201933404941,
      "grad_norm": 79.90824462915343,
      "learning_rate": 7.040104646698042e-09,
      "logits/chosen": -13.55671215057373,
      "logits/rejected": -13.5731201171875,
      "logps/chosen": -1.8288761377334595,
      "logps/rejected": -2.3549463748931885,
      "loss": 2.0965,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -18.288761138916016,
      "rewards/margins": 5.260700702667236,
      "rewards/rejected": -23.549463272094727,
      "step": 440
    },
    {
      "epoch": 0.9559613319011815,
      "grad_norm": 89.15905627877201,
      "learning_rate": 4.510443886542114e-09,
      "logits/chosen": -14.841180801391602,
      "logits/rejected": -14.82856559753418,
      "logps/chosen": -1.677901029586792,
      "logps/rejected": -2.1570167541503906,
      "loss": 2.2728,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -16.779010772705078,
      "rewards/margins": 4.7911577224731445,
      "rewards/rejected": -21.57016944885254,
      "step": 445
    },
    {
      "epoch": 0.966702470461869,
      "grad_norm": 108.85959629416836,
      "learning_rate": 2.539216160544333e-09,
      "logits/chosen": -14.823649406433105,
      "logits/rejected": -14.479968070983887,
      "logps/chosen": -1.819780707359314,
      "logps/rejected": -2.2197606563568115,
      "loss": 2.3086,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -18.19780921936035,
      "rewards/margins": 3.9997971057891846,
      "rewards/rejected": -22.197607040405273,
      "step": 450
    },
    {
      "epoch": 0.9774436090225563,
      "grad_norm": 120.19579580927282,
      "learning_rate": 1.1292048511303054e-09,
      "logits/chosen": -14.466642379760742,
      "logits/rejected": -14.534013748168945,
      "logps/chosen": -1.6585572957992554,
      "logps/rejected": -2.1007118225097656,
      "loss": 2.3314,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -16.585573196411133,
      "rewards/margins": 4.421543598175049,
      "rewards/rejected": -21.007116317749023,
      "step": 455
    },
    {
      "epoch": 0.9881847475832438,
      "grad_norm": 85.79550968404143,
      "learning_rate": 2.82400900618418e-10,
      "logits/chosen": -14.590263366699219,
      "logits/rejected": -14.626635551452637,
      "logps/chosen": -1.6538581848144531,
      "logps/rejected": -2.2225067615509033,
      "loss": 1.979,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -16.538583755493164,
      "rewards/margins": 5.686484336853027,
      "rewards/rejected": -22.225067138671875,
      "step": 460
    },
    {
      "epoch": 0.9989258861439313,
      "grad_norm": 102.57324448541058,
      "learning_rate": 0.0,
      "logits/chosen": -14.932310104370117,
      "logits/rejected": -14.83561897277832,
      "logps/chosen": -1.7759675979614258,
      "logps/rejected": -2.2065768241882324,
      "loss": 1.9975,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -17.759674072265625,
      "rewards/margins": 4.306093215942383,
      "rewards/rejected": -22.065767288208008,
      "step": 465
    },
    {
      "epoch": 0.9989258861439313,
      "step": 465,
      "total_flos": 0.0,
      "train_loss": 2.883953002191359,
      "train_runtime": 6137.844,
      "train_samples_per_second": 9.705,
      "train_steps_per_second": 0.076
    }
  ],
  "logging_steps": 5,
  "max_steps": 465,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}