{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9981298423724285,
  "eval_steps": 400,
  "global_step": 467,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0021373230029388193,
      "grad_norm": 42.099143078937416,
      "learning_rate": 1.0638297872340425e-08,
      "logits/chosen": 2.2472424507141113,
      "logits/rejected": 2.264739513397217,
      "logps/chosen": -0.8019136786460876,
      "logps/rejected": -0.7812027335166931,
      "loss": 1.7463,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -2.004784345626831,
      "rewards/margins": -0.05177761986851692,
      "rewards/rejected": -1.953006625175476,
      "step": 1
    },
    {
      "epoch": 0.010686615014694095,
      "grad_norm": 43.49437439863032,
      "learning_rate": 5.3191489361702123e-08,
      "logits/chosen": 2.1162009239196777,
      "logits/rejected": 2.0480892658233643,
      "logps/chosen": -1.0361735820770264,
      "logps/rejected": -1.1913045644760132,
      "loss": 1.9184,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": -2.5904340744018555,
      "rewards/margins": 0.38782745599746704,
      "rewards/rejected": -2.9782614707946777,
      "step": 5
    },
    {
      "epoch": 0.02137323002938819,
      "grad_norm": 73.58193785863429,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": 2.0012879371643066,
      "logits/rejected": 1.9172722101211548,
      "logps/chosen": -1.5639755725860596,
      "logps/rejected": -1.5803353786468506,
      "loss": 1.9077,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -3.9099388122558594,
      "rewards/margins": 0.04089919477701187,
      "rewards/rejected": -3.950838088989258,
      "step": 10
    },
    {
      "epoch": 0.03205984504408229,
      "grad_norm": 18.887588182169267,
      "learning_rate": 1.5957446808510638e-07,
      "logits/chosen": 1.9121309518814087,
      "logits/rejected": 1.8687114715576172,
      "logps/chosen": -1.1899975538253784,
      "logps/rejected": -1.2847260236740112,
      "loss": 1.832,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.974993944168091,
      "rewards/margins": 0.2368215024471283,
      "rewards/rejected": -3.211815357208252,
      "step": 15
    },
    {
      "epoch": 0.04274646005877638,
      "grad_norm": 8.819209234657059,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": 1.8210818767547607,
      "logits/rejected": 1.8469772338867188,
      "logps/chosen": -1.21903657913208,
      "logps/rejected": -1.7520463466644287,
      "loss": 1.7601,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -3.0475919246673584,
      "rewards/margins": 1.3325239419937134,
      "rewards/rejected": -4.380115509033203,
      "step": 20
    },
    {
      "epoch": 0.053433075073470476,
      "grad_norm": 33.18790384628351,
      "learning_rate": 2.659574468085106e-07,
      "logits/chosen": 2.0562968254089355,
      "logits/rejected": 2.0691914558410645,
      "logps/chosen": -1.0988093614578247,
      "logps/rejected": -1.1313331127166748,
      "loss": 1.815,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.747023344039917,
      "rewards/margins": 0.08130944520235062,
      "rewards/rejected": -2.8283333778381348,
      "step": 25
    },
    {
      "epoch": 0.06411969008816458,
      "grad_norm": 35.16962019463958,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": 1.9061098098754883,
      "logits/rejected": 1.9218835830688477,
      "logps/chosen": -1.2370525598526,
      "logps/rejected": -1.4245789051055908,
      "loss": 1.6802,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -3.0926311016082764,
      "rewards/margins": 0.46881628036499023,
      "rewards/rejected": -3.5614471435546875,
      "step": 30
    },
    {
      "epoch": 0.07480630510285867,
      "grad_norm": 44.11653715682163,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": 1.9321657419204712,
      "logits/rejected": 1.955640435218811,
      "logps/chosen": -1.344245433807373,
      "logps/rejected": -1.36150062084198,
      "loss": 1.8113,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -3.3606135845184326,
      "rewards/margins": 0.0431380569934845,
      "rewards/rejected": -3.4037513732910156,
      "step": 35
    },
    {
      "epoch": 0.08549292011755276,
      "grad_norm": 36.76549905569564,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": 1.8915932178497314,
      "logits/rejected": 1.9206931591033936,
      "logps/chosen": -1.2968499660491943,
      "logps/rejected": -1.3007665872573853,
      "loss": 1.8425,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -3.2421250343322754,
      "rewards/margins": 0.009791326709091663,
      "rewards/rejected": -3.2519161701202393,
      "step": 40
    },
    {
      "epoch": 0.09617953513224686,
      "grad_norm": 17.026888436798433,
      "learning_rate": 4.787234042553192e-07,
      "logits/chosen": 1.8335082530975342,
      "logits/rejected": 1.803972601890564,
      "logps/chosen": -1.1560873985290527,
      "logps/rejected": -1.391876459121704,
      "loss": 1.748,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.8902182579040527,
      "rewards/margins": 0.5894726514816284,
      "rewards/rejected": -3.4796910285949707,
      "step": 45
    },
    {
      "epoch": 0.10686615014694095,
      "grad_norm": 12.410804374401861,
      "learning_rate": 4.999370587356267e-07,
      "logits/chosen": 1.864829659461975,
      "logits/rejected": 1.8258002996444702,
      "logps/chosen": -1.0499851703643799,
      "logps/rejected": -1.454998254776001,
      "loss": 1.5838,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.6249632835388184,
      "rewards/margins": 1.0125324726104736,
      "rewards/rejected": -3.637495517730713,
      "step": 50
    },
    {
      "epoch": 0.11755276516163506,
      "grad_norm": 16.97890172462393,
      "learning_rate": 4.995525324419337e-07,
      "logits/chosen": 1.8395353555679321,
      "logits/rejected": 1.8361022472381592,
      "logps/chosen": -0.9045565724372864,
      "logps/rejected": -1.1536071300506592,
      "loss": 1.6796,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.2613916397094727,
      "rewards/margins": 0.6226258873939514,
      "rewards/rejected": -2.8840174674987793,
      "step": 55
    },
    {
      "epoch": 0.12823938017632916,
      "grad_norm": 18.13814698710649,
      "learning_rate": 4.988189843662815e-07,
      "logits/chosen": 1.9378535747528076,
      "logits/rejected": 1.9659252166748047,
      "logps/chosen": -1.1384503841400146,
      "logps/rejected": -1.0124545097351074,
      "loss": 1.7608,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.846126079559326,
      "rewards/margins": -0.3149900436401367,
      "rewards/rejected": -2.5311360359191895,
      "step": 60
    },
    {
      "epoch": 0.13892599519102325,
      "grad_norm": 31.166552691677705,
      "learning_rate": 4.977374404419837e-07,
      "logits/chosen": 2.1393017768859863,
      "logits/rejected": 2.103208303451538,
      "logps/chosen": -0.8744574785232544,
      "logps/rejected": -0.9523324966430664,
      "loss": 1.6552,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.186143636703491,
      "rewards/margins": 0.19468756020069122,
      "rewards/rejected": -2.380831241607666,
      "step": 65
    },
    {
      "epoch": 0.14961261020571734,
      "grad_norm": 52.728695245981314,
      "learning_rate": 4.963094133060148e-07,
      "logits/chosen": 2.0067527294158936,
      "logits/rejected": 1.9597012996673584,
      "logps/chosen": -0.9174124598503113,
      "logps/rejected": -1.1131672859191895,
      "loss": 1.7212,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.2935314178466797,
      "rewards/margins": 0.4893869459629059,
      "rewards/rejected": -2.7829184532165527,
      "step": 70
    },
    {
      "epoch": 0.16029922522041143,
      "grad_norm": 46.624386507512185,
      "learning_rate": 4.945369001834514e-07,
      "logits/chosen": 1.8767681121826172,
      "logits/rejected": 1.8939720392227173,
      "logps/chosen": -0.9977117776870728,
      "logps/rejected": -1.1487284898757935,
      "loss": 1.6147,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.494279384613037,
      "rewards/margins": 0.3775419294834137,
      "rewards/rejected": -2.8718209266662598,
      "step": 75
    },
    {
      "epoch": 0.17098584023510552,
      "grad_norm": 13.45137638802328,
      "learning_rate": 4.924223800941717e-07,
      "logits/chosen": 1.912676453590393,
      "logits/rejected": 1.8263626098632812,
      "logps/chosen": -0.9859241247177124,
      "logps/rejected": -1.0560874938964844,
      "loss": 1.651,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -2.464810609817505,
      "rewards/margins": 0.17540840804576874,
      "rewards/rejected": -2.640218734741211,
      "step": 80
    },
    {
      "epoch": 0.18167245524979964,
      "grad_norm": 23.876540575284714,
      "learning_rate": 4.899688103857222e-07,
      "logits/chosen": 1.9252907037734985,
      "logits/rejected": 1.9396631717681885,
      "logps/chosen": -0.9312021136283875,
      "logps/rejected": -1.1102412939071655,
      "loss": 1.618,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.328005313873291,
      "rewards/margins": 0.44759804010391235,
      "rewards/rejected": -2.7756032943725586,
      "step": 85
    },
    {
      "epoch": 0.19235907026449373,
      "grad_norm": 7.705041238913054,
      "learning_rate": 4.871796225971999e-07,
      "logits/chosen": 1.9238191843032837,
      "logits/rejected": 1.8918044567108154,
      "logps/chosen": -0.9711843729019165,
      "logps/rejected": -1.096145749092102,
      "loss": 1.6624,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.4279608726501465,
      "rewards/margins": 0.3124033510684967,
      "rewards/rejected": -2.7403643131256104,
      "step": 90
    },
    {
      "epoch": 0.20304568527918782,
      "grad_norm": 16.98405162222745,
      "learning_rate": 4.840587176599343e-07,
      "logits/chosen": 2.0161519050598145,
      "logits/rejected": 2.037721872329712,
      "logps/chosen": -0.9618253707885742,
      "logps/rejected": -1.1678836345672607,
      "loss": 1.6553,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.4045631885528564,
      "rewards/margins": 0.5151457786560059,
      "rewards/rejected": -2.9197089672088623,
      "step": 95
    },
    {
      "epoch": 0.2137323002938819,
      "grad_norm": 26.59166513674174,
      "learning_rate": 4.806104604416823e-07,
      "logits/chosen": 1.9551931619644165,
      "logits/rejected": 1.9663063287734985,
      "logps/chosen": -0.9686329960823059,
      "logps/rejected": -0.964672863483429,
      "loss": 1.6952,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.4215824604034424,
      "rewards/margins": -0.009900185279548168,
      "rewards/rejected": -2.41168212890625,
      "step": 100
    },
    {
      "epoch": 0.224418915308576,
      "grad_norm": 22.597015867331866,
      "learning_rate": 4.768396736419662e-07,
      "logits/chosen": 2.0987541675567627,
      "logits/rejected": 2.1704182624816895,
      "logps/chosen": -0.8102301359176636,
      "logps/rejected": -1.0246448516845703,
      "loss": 1.6353,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.0255753993988037,
      "rewards/margins": 0.5360367894172668,
      "rewards/rejected": -2.561612367630005,
      "step": 105
    },
    {
      "epoch": 0.2351055303232701,
      "grad_norm": 7.710202530561847,
      "learning_rate": 4.7275163104709194e-07,
      "logits/chosen": 1.8211323022842407,
      "logits/rejected": 1.8467600345611572,
      "logps/chosen": -0.8923318982124329,
      "logps/rejected": -0.9886472821235657,
      "loss": 1.6826,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.2308297157287598,
      "rewards/margins": 0.24078825116157532,
      "rewards/rejected": -2.4716179370880127,
      "step": 110
    },
    {
      "epoch": 0.2457921453379642,
      "grad_norm": 9.211974257959199,
      "learning_rate": 4.683520501542824e-07,
      "logits/chosen": 1.822178840637207,
      "logits/rejected": 1.8460540771484375,
      "logps/chosen": -0.937783420085907,
      "logps/rejected": -0.9862428903579712,
      "loss": 1.6087,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.34445858001709,
      "rewards/margins": 0.12114912271499634,
      "rewards/rejected": -2.4656076431274414,
      "step": 115
    },
    {
      "epoch": 0.2564787603526583,
      "grad_norm": 17.97332526477704,
      "learning_rate": 4.636470841752404e-07,
      "logits/chosen": 1.9925601482391357,
      "logits/rejected": 2.0132079124450684,
      "logps/chosen": -0.8938210606575012,
      "logps/rejected": -0.9105901718139648,
      "loss": 1.6647,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.2345528602600098,
      "rewards/margins": 0.04192250221967697,
      "rewards/rejected": -2.276475429534912,
      "step": 120
    },
    {
      "epoch": 0.2671653753673524,
      "grad_norm": 21.241131663632817,
      "learning_rate": 4.5864331343032565e-07,
      "logits/chosen": 1.821298360824585,
      "logits/rejected": 1.8526639938354492,
      "logps/chosen": -0.9126847386360168,
      "logps/rejected": -1.0501748323440552,
      "loss": 1.7558,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.281712055206299,
      "rewards/margins": 0.3437250852584839,
      "rewards/rejected": -2.6254372596740723,
      "step": 125
    },
    {
      "epoch": 0.2778519903820465,
      "grad_norm": 18.947973877308787,
      "learning_rate": 4.533477361453819e-07,
      "logits/chosen": 2.178891658782959,
      "logits/rejected": 2.143307685852051,
      "logps/chosen": -0.9617658853530884,
      "logps/rejected": -1.223284125328064,
      "loss": 1.6892,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.404414653778076,
      "rewards/margins": 0.6537958383560181,
      "rewards/rejected": -3.0582103729248047,
      "step": 130
    },
    {
      "epoch": 0.2885386053967406,
      "grad_norm": 13.76012227966358,
      "learning_rate": 4.4776775866408533e-07,
      "logits/chosen": 1.8531124591827393,
      "logits/rejected": 1.8338654041290283,
      "logps/chosen": -0.8667293787002563,
      "logps/rejected": -0.9339718818664551,
      "loss": 1.658,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.166823148727417,
      "rewards/margins": 0.16810639202594757,
      "rewards/rejected": -2.334929943084717,
      "step": 135
    },
    {
      "epoch": 0.2992252204114347,
      "grad_norm": 12.832812020829884,
      "learning_rate": 4.4191118508950277e-07,
      "logits/chosen": 1.8038040399551392,
      "logits/rejected": 1.7551212310791016,
      "logps/chosen": -0.8944159746170044,
      "logps/rejected": -1.0033025741577148,
      "loss": 1.6771,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.236039638519287,
      "rewards/margins": 0.2722164988517761,
      "rewards/rejected": -2.508256435394287,
      "step": 140
    },
    {
      "epoch": 0.30991183542612877,
      "grad_norm": 9.09964498419542,
      "learning_rate": 4.357862063693485e-07,
      "logits/chosen": 2.027944564819336,
      "logits/rejected": 2.005831718444824,
      "logps/chosen": -0.8879717588424683,
      "logps/rejected": -0.9542590975761414,
      "loss": 1.7133,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.2199294567108154,
      "rewards/margins": 0.16571831703186035,
      "rewards/rejected": -2.3856475353240967,
      "step": 145
    },
    {
      "epoch": 0.32059845044082286,
      "grad_norm": 24.685014270210495,
      "learning_rate": 4.294013888402029e-07,
      "logits/chosen": 2.138366222381592,
      "logits/rejected": 2.2007527351379395,
      "logps/chosen": -0.8750768899917603,
      "logps/rejected": -1.0953787565231323,
      "loss": 1.6625,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.187692165374756,
      "rewards/margins": 0.5507549047470093,
      "rewards/rejected": -2.7384471893310547,
      "step": 150
    },
    {
      "epoch": 0.33128506545551695,
      "grad_norm": 24.61662782262514,
      "learning_rate": 4.227656622467162e-07,
      "logits/chosen": 1.911320686340332,
      "logits/rejected": 1.9191503524780273,
      "logps/chosen": -0.8599516153335571,
      "logps/rejected": -1.1553843021392822,
      "loss": 1.6495,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.149878978729248,
      "rewards/margins": 0.7385817766189575,
      "rewards/rejected": -2.888460874557495,
      "step": 155
    },
    {
      "epoch": 0.34197168047021104,
      "grad_norm": 9.846236022517108,
      "learning_rate": 4.158883072525528e-07,
      "logits/chosen": 2.061549425125122,
      "logits/rejected": 2.0822834968566895,
      "logps/chosen": -0.8990328907966614,
      "logps/rejected": -0.985998809337616,
      "loss": 1.6348,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -2.247582197189331,
      "rewards/margins": 0.21741452813148499,
      "rewards/rejected": -2.464996814727783,
      "step": 160
    },
    {
      "epoch": 0.3526582954849052,
      "grad_norm": 19.106656482086205,
      "learning_rate": 4.087789424605447e-07,
      "logits/chosen": 2.265204906463623,
      "logits/rejected": 2.245617151260376,
      "logps/chosen": -0.818160355091095,
      "logps/rejected": -0.9623705744743347,
      "loss": 1.618,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.045400857925415,
      "rewards/margins": 0.3605257570743561,
      "rewards/rejected": -2.405926465988159,
      "step": 165
    },
    {
      "epoch": 0.36334491049959927,
      "grad_norm": 14.919110153191296,
      "learning_rate": 4.0144751096020497e-07,
      "logits/chosen": 1.9928451776504517,
      "logits/rejected": 1.9697974920272827,
      "logps/chosen": -1.013695478439331,
      "logps/rejected": -1.2301123142242432,
      "loss": 1.5848,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.534238815307617,
      "rewards/margins": 0.5410416722297668,
      "rewards/rejected": -3.0752806663513184,
      "step": 170
    },
    {
      "epoch": 0.37403152551429336,
      "grad_norm": 11.830869809507767,
      "learning_rate": 3.939042664214184e-07,
      "logits/chosen": 2.031292676925659,
      "logits/rejected": 2.066436767578125,
      "logps/chosen": -0.97381991147995,
      "logps/rejected": -1.2112761735916138,
      "loss": 1.5954,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.4345498085021973,
      "rewards/margins": 0.5936408638954163,
      "rewards/rejected": -3.0281906127929688,
      "step": 175
    },
    {
      "epoch": 0.38471814052898745,
      "grad_norm": 14.628170167780459,
      "learning_rate": 3.8615975875375676e-07,
      "logits/chosen": 2.258387804031372,
      "logits/rejected": 2.3218982219696045,
      "logps/chosen": -1.0137016773223877,
      "logps/rejected": -1.1529831886291504,
      "loss": 1.6695,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.534254312515259,
      "rewards/margins": 0.3482036888599396,
      "rewards/rejected": -2.882457971572876,
      "step": 180
    },
    {
      "epoch": 0.39540475554368154,
      "grad_norm": 16.49947715615614,
      "learning_rate": 3.7822481935147655e-07,
      "logits/chosen": 2.0560050010681152,
      "logits/rejected": 2.1167311668395996,
      "logps/chosen": -0.9939271211624146,
      "logps/rejected": -1.1564804315567017,
      "loss": 1.6754,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.4848177433013916,
      "rewards/margins": 0.40638333559036255,
      "rewards/rejected": -2.8912010192871094,
      "step": 185
    },
    {
      "epoch": 0.40609137055837563,
      "grad_norm": 13.5081717360946,
      "learning_rate": 3.7011054594483443e-07,
      "logits/chosen": 2.1290013790130615,
      "logits/rejected": 2.223310947418213,
      "logps/chosen": -0.9123891592025757,
      "logps/rejected": -1.1287665367126465,
      "loss": 1.5831,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.280972957611084,
      "rewards/margins": 0.5409437417984009,
      "rewards/rejected": -2.8219165802001953,
      "step": 190
    },
    {
      "epoch": 0.4167779855730697,
      "grad_norm": 17.61863203205812,
      "learning_rate": 3.618282870789081e-07,
      "logits/chosen": 2.1478934288024902,
      "logits/rejected": 2.1439719200134277,
      "logps/chosen": -0.974346935749054,
      "logps/rejected": -1.1090627908706665,
      "loss": 1.64,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.4358673095703125,
      "rewards/margins": 0.33678966760635376,
      "rewards/rejected": -2.7726569175720215,
      "step": 195
    },
    {
      "epoch": 0.4274646005877638,
      "grad_norm": 12.457201720784246,
      "learning_rate": 3.5338962624163016e-07,
      "logits/chosen": 2.2304863929748535,
      "logits/rejected": 2.2417056560516357,
      "logps/chosen": -0.9161917567253113,
      "logps/rejected": -1.0349229574203491,
      "loss": 1.672,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -2.2904791831970215,
      "rewards/margins": 0.29682818055152893,
      "rewards/rejected": -2.5873074531555176,
      "step": 200
    },
    {
      "epoch": 0.4381512156024579,
      "grad_norm": 25.419906192801392,
      "learning_rate": 3.448063656632321e-07,
      "logits/chosen": 2.2168259620666504,
      "logits/rejected": 2.2494800090789795,
      "logps/chosen": -0.9495501518249512,
      "logps/rejected": -1.0588171482086182,
      "loss": 1.6462,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.373875379562378,
      "rewards/margins": 0.2731671929359436,
      "rewards/rejected": -2.647042751312256,
      "step": 205
    },
    {
      "epoch": 0.448837830617152,
      "grad_norm": 31.43495532901771,
      "learning_rate": 3.360905098097587e-07,
      "logits/chosen": 2.0355324745178223,
      "logits/rejected": 2.0058634281158447,
      "logps/chosen": -0.9030116200447083,
      "logps/rejected": -1.100672960281372,
      "loss": 1.7332,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.2575290203094482,
      "rewards/margins": 0.49415332078933716,
      "rewards/rejected": -2.751682758331299,
      "step": 210
    },
    {
      "epoch": 0.45952444563184613,
      "grad_norm": 9.866165199153304,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": 2.188046932220459,
      "logits/rejected": 2.209195375442505,
      "logps/chosen": -0.8452509045600891,
      "logps/rejected": -0.9856406450271606,
      "loss": 1.6173,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.1131272315979004,
      "rewards/margins": 0.3509744107723236,
      "rewards/rejected": -2.464101791381836,
      "step": 215
    },
    {
      "epoch": 0.4702110606465402,
      "grad_norm": 11.268271977292386,
      "learning_rate": 3.1830994032548e-07,
      "logits/chosen": 2.2558753490448,
      "logits/rejected": 2.271909475326538,
      "logps/chosen": -0.9621235728263855,
      "logps/rejected": -1.0470136404037476,
      "loss": 1.6623,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.405308961868286,
      "rewards/margins": 0.21222510933876038,
      "rewards/rejected": -2.6175341606140137,
      "step": 220
    },
    {
      "epoch": 0.4808976756612343,
      "grad_norm": 8.220154130520008,
      "learning_rate": 3.0927009442887437e-07,
      "logits/chosen": 2.033316135406494,
      "logits/rejected": 2.0451745986938477,
      "logps/chosen": -0.9348015785217285,
      "logps/rejected": -0.9307538866996765,
      "loss": 1.7025,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.3370039463043213,
      "rewards/margins": -0.010119226761162281,
      "rewards/rejected": -2.3268847465515137,
      "step": 225
    },
    {
      "epoch": 0.4915842906759284,
      "grad_norm": 8.761484125412297,
      "learning_rate": 3.001473539458182e-07,
      "logits/chosen": 2.039243221282959,
      "logits/rejected": 2.0847630500793457,
      "logps/chosen": -0.9077224731445312,
      "logps/rejected": -0.9697052240371704,
      "loss": 1.6369,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -2.269306182861328,
      "rewards/margins": 0.15495693683624268,
      "rewards/rejected": -2.4242632389068604,
      "step": 230
    },
    {
      "epoch": 0.5022709056906225,
      "grad_norm": 21.069821104434805,
      "learning_rate": 2.909544778537844e-07,
      "logits/chosen": 2.1515793800354004,
      "logits/rejected": 2.0550239086151123,
      "logps/chosen": -0.9554904699325562,
      "logps/rejected": -1.0575495958328247,
      "loss": 1.6471,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -2.3887264728546143,
      "rewards/margins": 0.2551475167274475,
      "rewards/rejected": -2.643873691558838,
      "step": 235
    },
    {
      "epoch": 0.5129575207053166,
      "grad_norm": 18.45689011337618,
      "learning_rate": 2.817043232212371e-07,
      "logits/chosen": 2.3277316093444824,
      "logits/rejected": 2.359060287475586,
      "logps/chosen": -0.9175651669502258,
      "logps/rejected": -1.075629472732544,
      "loss": 1.5723,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.2939131259918213,
      "rewards/margins": 0.39516064524650574,
      "rewards/rejected": -2.6890735626220703,
      "step": 240
    },
    {
      "epoch": 0.5236441357200107,
      "grad_norm": 5.945491870174955,
      "learning_rate": 2.7240982722585837e-07,
      "logits/chosen": 2.2272980213165283,
      "logits/rejected": 2.216254472732544,
      "logps/chosen": -0.887627124786377,
      "logps/rejected": -1.0360788106918335,
      "loss": 1.6259,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.2190678119659424,
      "rewards/margins": 0.3711293637752533,
      "rewards/rejected": -2.5901970863342285,
      "step": 245
    },
    {
      "epoch": 0.5343307507347048,
      "grad_norm": 10.042684286997996,
      "learning_rate": 2.63083989060736e-07,
      "logits/chosen": 2.162696361541748,
      "logits/rejected": 2.178349494934082,
      "logps/chosen": -0.9883350133895874,
      "logps/rejected": -1.1994340419769287,
      "loss": 1.6589,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.4708378314971924,
      "rewards/margins": 0.5277475118637085,
      "rewards/rejected": -2.9985852241516113,
      "step": 250
    },
    {
      "epoch": 0.5450173657493989,
      "grad_norm": 10.520900876521566,
      "learning_rate": 2.537398517538159e-07,
      "logits/chosen": 2.2908027172088623,
      "logits/rejected": 2.235353708267212,
      "logps/chosen": -0.9431841969490051,
      "logps/rejected": -1.015700101852417,
      "loss": 1.6741,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.3579602241516113,
      "rewards/margins": 0.18129023909568787,
      "rewards/rejected": -2.539250612258911,
      "step": 255
    },
    {
      "epoch": 0.555703980764093,
      "grad_norm": 9.313778052567603,
      "learning_rate": 2.4439048392604877e-07,
      "logits/chosen": 2.1532225608825684,
      "logits/rejected": 2.2637267112731934,
      "logps/chosen": -0.9131342768669128,
      "logps/rejected": -1.2010549306869507,
      "loss": 1.5779,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -2.2828357219696045,
      "rewards/margins": 0.7198014259338379,
      "rewards/rejected": -3.0026373863220215,
      "step": 260
    },
    {
      "epoch": 0.566390595778787,
      "grad_norm": 17.127300405069718,
      "learning_rate": 2.3504896151374144e-07,
      "logits/chosen": 2.2980525493621826,
      "logits/rejected": 2.2274222373962402,
      "logps/chosen": -0.9766885042190552,
      "logps/rejected": -1.0872061252593994,
      "loss": 1.6741,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.4417214393615723,
      "rewards/margins": 0.2762938141822815,
      "rewards/rejected": -2.718015193939209,
      "step": 265
    },
    {
      "epoch": 0.5770772107934812,
      "grad_norm": 14.096322545468484,
      "learning_rate": 2.2572834948067795e-07,
      "logits/chosen": 2.13866925239563,
      "logits/rejected": 2.09075665473938,
      "logps/chosen": -0.866176962852478,
      "logps/rejected": -1.0079139471054077,
      "loss": 1.6445,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.165442705154419,
      "rewards/margins": 0.3543422520160675,
      "rewards/rejected": -2.519784688949585,
      "step": 270
    },
    {
      "epoch": 0.5877638258081752,
      "grad_norm": 11.519350142557252,
      "learning_rate": 2.164416835455862e-07,
      "logits/chosen": 1.9599807262420654,
      "logits/rejected": 1.9166367053985596,
      "logps/chosen": -0.9016801714897156,
      "logps/rejected": -1.060001015663147,
      "loss": 1.6308,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.2542004585266113,
      "rewards/margins": 0.39580219984054565,
      "rewards/rejected": -2.6500024795532227,
      "step": 275
    },
    {
      "epoch": 0.5984504408228694,
      "grad_norm": 9.94425188158257,
      "learning_rate": 2.072019519505062e-07,
      "logits/chosen": 2.006683349609375,
      "logits/rejected": 1.9877275228500366,
      "logps/chosen": -0.9040509462356567,
      "logps/rejected": -1.0043412446975708,
      "loss": 1.6201,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.260127305984497,
      "rewards/margins": 0.2507258355617523,
      "rewards/rejected": -2.5108530521392822,
      "step": 280
    },
    {
      "epoch": 0.6091370558375635,
      "grad_norm": 17.126362799060438,
      "learning_rate": 1.980220772955602e-07,
      "logits/chosen": 2.040419816970825,
      "logits/rejected": 1.9795433282852173,
      "logps/chosen": -1.0711162090301514,
      "logps/rejected": -1.1054927110671997,
      "loss": 1.6696,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.677790641784668,
      "rewards/margins": 0.08594076335430145,
      "rewards/rejected": -2.7637314796447754,
      "step": 285
    },
    {
      "epoch": 0.6198236708522575,
      "grad_norm": 13.492215975630849,
      "learning_rate": 1.8891489846552644e-07,
      "logits/chosen": 2.3427670001983643,
      "logits/rejected": 2.3189902305603027,
      "logps/chosen": -0.9356976747512817,
      "logps/rejected": -1.0561234951019287,
      "loss": 1.6313,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.3392443656921387,
      "rewards/margins": 0.30106478929519653,
      "rewards/rejected": -2.6403088569641113,
      "step": 290
    },
    {
      "epoch": 0.6305102858669517,
      "grad_norm": 6.893975837082491,
      "learning_rate": 1.7989315267349933e-07,
      "logits/chosen": 2.3545644283294678,
      "logits/rejected": 2.3726115226745605,
      "logps/chosen": -0.9937087893486023,
      "logps/rejected": -1.0849647521972656,
      "loss": 1.6816,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.484271764755249,
      "rewards/margins": 0.22813992202281952,
      "rewards/rejected": -2.712411642074585,
      "step": 295
    },
    {
      "epoch": 0.6411969008816457,
      "grad_norm": 6.6767839699246405,
      "learning_rate": 1.7096945764674398e-07,
      "logits/chosen": 2.0134410858154297,
      "logits/rejected": 1.9672603607177734,
      "logps/chosen": -1.038081169128418,
      "logps/rejected": -1.0271114110946655,
      "loss": 1.6598,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -2.595202922821045,
      "rewards/margins": -0.027424562722444534,
      "rewards/rejected": -2.5677783489227295,
      "step": 300
    },
    {
      "epoch": 0.6518835158963399,
      "grad_norm": 8.402566753074327,
      "learning_rate": 1.621562939796643e-07,
      "logits/chosen": 2.099931240081787,
      "logits/rejected": 2.131023406982422,
      "logps/chosen": -0.9801462888717651,
      "logps/rejected": -1.1142746210098267,
      "loss": 1.6239,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.4503657817840576,
      "rewards/margins": 0.33532071113586426,
      "rewards/rejected": -2.785686492919922,
      "step": 305
    },
    {
      "epoch": 0.6625701309110339,
      "grad_norm": 13.484008579562406,
      "learning_rate": 1.5346598767856345e-07,
      "logits/chosen": 2.3738818168640137,
      "logits/rejected": 2.2915918827056885,
      "logps/chosen": -0.9194884300231934,
      "logps/rejected": -1.1123372316360474,
      "loss": 1.6509,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.2987208366394043,
      "rewards/margins": 0.4821222722530365,
      "rewards/rejected": -2.7808432579040527,
      "step": 310
    },
    {
      "epoch": 0.673256745925728,
      "grad_norm": 11.276062854222866,
      "learning_rate": 1.4491069292260866e-07,
      "logits/chosen": 2.0688905715942383,
      "logits/rejected": 2.0706334114074707,
      "logps/chosen": -0.9416360855102539,
      "logps/rejected": -0.9889582395553589,
      "loss": 1.6263,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.354090452194214,
      "rewards/margins": 0.11830528825521469,
      "rewards/rejected": -2.472395658493042,
      "step": 315
    },
    {
      "epoch": 0.6839433609404221,
      "grad_norm": 12.406779732367644,
      "learning_rate": 1.365023750651133e-07,
      "logits/chosen": 2.029076337814331,
      "logits/rejected": 2.07525634765625,
      "logps/chosen": -0.8669158220291138,
      "logps/rejected": -1.1443169116973877,
      "loss": 1.5861,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.1672892570495605,
      "rewards/margins": 0.6935025453567505,
      "rewards/rejected": -2.8607916831970215,
      "step": 320
    },
    {
      "epoch": 0.6946299759551162,
      "grad_norm": 9.410463016483124,
      "learning_rate": 1.2825279389890818e-07,
      "logits/chosen": 2.2680742740631104,
      "logits/rejected": 2.2626852989196777,
      "logps/chosen": -0.9127257466316223,
      "logps/rejected": -1.036811113357544,
      "loss": 1.5807,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.2818140983581543,
      "rewards/margins": 0.31021344661712646,
      "rewards/rejected": -2.5920276641845703,
      "step": 325
    },
    {
      "epoch": 0.7053165909698104,
      "grad_norm": 13.015902838409264,
      "learning_rate": 1.201734872092077e-07,
      "logits/chosen": 2.0204429626464844,
      "logits/rejected": 1.9889049530029297,
      "logps/chosen": -0.9624814987182617,
      "logps/rejected": -1.4148118495941162,
      "loss": 1.6247,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.4062037467956543,
      "rewards/margins": 1.130826473236084,
      "rewards/rejected": -3.537029981613159,
      "step": 330
    },
    {
      "epoch": 0.7160032059845044,
      "grad_norm": 10.457412577224535,
      "learning_rate": 1.1227575463697439e-07,
      "logits/chosen": 2.1002488136291504,
      "logits/rejected": 2.1578989028930664,
      "logps/chosen": -0.9219167828559875,
      "logps/rejected": -1.0541021823883057,
      "loss": 1.6126,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.3047919273376465,
      "rewards/margins": 0.33046382665634155,
      "rewards/rejected": -2.635255813598633,
      "step": 335
    },
    {
      "epoch": 0.7266898209991985,
      "grad_norm": 13.23573306912224,
      "learning_rate": 1.0457064187534861e-07,
      "logits/chosen": 2.2117228507995605,
      "logits/rejected": 2.291081666946411,
      "logps/chosen": -0.8333222270011902,
      "logps/rejected": -1.0662933588027954,
      "loss": 1.611,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.083305597305298,
      "rewards/margins": 0.5824280977249146,
      "rewards/rejected": -2.665733575820923,
      "step": 340
    },
    {
      "epoch": 0.7373764360138926,
      "grad_norm": 7.317920448430426,
      "learning_rate": 9.706892522124838e-08,
      "logits/chosen": 2.1216635704040527,
      "logits/rejected": 2.178093194961548,
      "logps/chosen": -1.0588462352752686,
      "logps/rejected": -1.0970056056976318,
      "loss": 1.5811,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.647115468978882,
      "rewards/margins": 0.09539877623319626,
      "rewards/rejected": -2.742514133453369,
      "step": 345
    },
    {
      "epoch": 0.7480630510285867,
      "grad_norm": 14.193162676817792,
      "learning_rate": 8.978109650374396e-08,
      "logits/chosen": 2.2546029090881348,
      "logits/rejected": 2.294689893722534,
      "logps/chosen": -0.9023507833480835,
      "logps/rejected": -1.162570595741272,
      "loss": 1.6171,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.2558767795562744,
      "rewards/margins": 0.6505497694015503,
      "rewards/rejected": -2.9064266681671143,
      "step": 350
    },
    {
      "epoch": 0.7587496660432808,
      "grad_norm": 12.52863776377329,
      "learning_rate": 8.271734841028552e-08,
      "logits/chosen": 2.245272636413574,
      "logits/rejected": 2.2576751708984375,
      "logps/chosen": -1.005053997039795,
      "logps/rejected": -1.0007752180099487,
      "loss": 1.6536,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -2.512634754180908,
      "rewards/margins": -0.010696577839553356,
      "rewards/rejected": -2.5019383430480957,
      "step": 355
    },
    {
      "epoch": 0.7694362810579749,
      "grad_norm": 10.369189534811127,
      "learning_rate": 7.588756023130833e-08,
      "logits/chosen": 2.318972587585449,
      "logits/rejected": 2.281419515609741,
      "logps/chosen": -1.0578750371932983,
      "logps/rejected": -1.3604905605316162,
      "loss": 1.6205,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -2.6446874141693115,
      "rewards/margins": 0.7565388679504395,
      "rewards/rejected": -3.40122652053833,
      "step": 360
    },
    {
      "epoch": 0.7801228960726689,
      "grad_norm": 6.075365195709392,
      "learning_rate": 6.930128404315214e-08,
      "logits/chosen": 2.0289673805236816,
      "logits/rejected": 2.0680978298187256,
      "logps/chosen": -0.976359486579895,
      "logps/rejected": -1.0074145793914795,
      "loss": 1.6246,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.440898895263672,
      "rewards/margins": 0.07763750851154327,
      "rewards/rejected": -2.518536329269409,
      "step": 365
    },
    {
      "epoch": 0.7908095110873631,
      "grad_norm": 16.214823251732284,
      "learning_rate": 6.296773134861824e-08,
      "logits/chosen": 2.2002553939819336,
      "logits/rejected": 2.2343714237213135,
      "logps/chosen": -0.9719416499137878,
      "logps/rejected": -1.023166537284851,
      "loss": 1.6894,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.429853916168213,
      "rewards/margins": 0.12806224822998047,
      "rewards/rejected": -2.5579161643981934,
      "step": 370
    },
    {
      "epoch": 0.8014961261020572,
      "grad_norm": 10.840080143128308,
      "learning_rate": 5.6895760193850145e-08,
      "logits/chosen": 2.1092734336853027,
      "logits/rejected": 2.168839693069458,
      "logps/chosen": -1.0623986721038818,
      "logps/rejected": -1.1893246173858643,
      "loss": 1.6248,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.6559970378875732,
      "rewards/margins": 0.31731492280960083,
      "rewards/rejected": -2.9733119010925293,
      "step": 375
    },
    {
      "epoch": 0.8121827411167513,
      "grad_norm": 14.986128731201607,
      "learning_rate": 5.109386277955477e-08,
      "logits/chosen": 2.400944232940674,
      "logits/rejected": 2.424126625061035,
      "logps/chosen": -1.0006974935531616,
      "logps/rejected": -1.0918558835983276,
      "loss": 1.6152,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -2.501743793487549,
      "rewards/margins": 0.22789612412452698,
      "rewards/rejected": -2.729639768600464,
      "step": 380
    },
    {
      "epoch": 0.8228693561314454,
      "grad_norm": 11.04509407630343,
      "learning_rate": 4.557015358389216e-08,
      "logits/chosen": 2.2161147594451904,
      "logits/rejected": 2.255049705505371,
      "logps/chosen": -0.9763911962509155,
      "logps/rejected": -1.2085295915603638,
      "loss": 1.6246,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -2.4409780502319336,
      "rewards/margins": 0.5803458094596863,
      "rewards/rejected": -3.0213241577148438,
      "step": 385
    },
    {
      "epoch": 0.8335559711461394,
      "grad_norm": 17.08993723906078,
      "learning_rate": 4.0332358013644015e-08,
      "logits/chosen": 2.3219351768493652,
      "logits/rejected": 2.316385507583618,
      "logps/chosen": -0.8949235081672668,
      "logps/rejected": -1.2672293186187744,
      "loss": 1.6161,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -2.2373085021972656,
      "rewards/margins": 0.9307647943496704,
      "rewards/rejected": -3.1680736541748047,
      "step": 390
    },
    {
      "epoch": 0.8442425861608336,
      "grad_norm": 7.486937950213667,
      "learning_rate": 3.538780159953347e-08,
      "logits/chosen": 2.171618700027466,
      "logits/rejected": 2.133547067642212,
      "logps/chosen": -0.913979172706604,
      "logps/rejected": -1.011819839477539,
      "loss": 1.6306,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.284947633743286,
      "rewards/margins": 0.24460192024707794,
      "rewards/rejected": -2.5295493602752686,
      "step": 395
    },
    {
      "epoch": 0.8549292011755276,
      "grad_norm": 12.731226626352624,
      "learning_rate": 3.074339975080836e-08,
      "logits/chosen": 2.207425594329834,
      "logits/rejected": 2.2549290657043457,
      "logps/chosen": -1.030874490737915,
      "logps/rejected": -1.2776422500610352,
      "loss": 1.6417,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.577186107635498,
      "rewards/margins": 0.6169193983078003,
      "rewards/rejected": -3.194105863571167,
      "step": 400
    },
    {
      "epoch": 0.8549292011755276,
      "eval_logits/chosen": 2.0145604610443115,
      "eval_logits/rejected": 2.0190441608428955,
      "eval_logps/chosen": -0.8955881595611572,
      "eval_logps/rejected": -1.058284044265747,
      "eval_loss": 1.6235809326171875,
      "eval_rewards/accuracies": 0.6209677457809448,
      "eval_rewards/chosen": -2.2389702796936035,
      "eval_rewards/margins": 0.40673962235450745,
      "eval_rewards/rejected": -2.645709753036499,
      "eval_runtime": 44.336,
      "eval_samples_per_second": 44.23,
      "eval_steps_per_second": 1.398,
      "step": 400
    },
    {
      "epoch": 0.8656158161902218,
      "grad_norm": 10.867554141726604,
      "learning_rate": 2.6405648083415833e-08,
      "logits/chosen": 2.151744842529297,
      "logits/rejected": 2.200650691986084,
      "logps/chosen": -0.980237603187561,
      "logps/rejected": -1.0543644428253174,
      "loss": 1.7482,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.450594186782837,
      "rewards/margins": 0.18531720340251923,
      "rewards/rejected": -2.635911226272583,
      "step": 405
    },
    {
      "epoch": 0.8763024312049158,
      "grad_norm": 8.929503879536968,
      "learning_rate": 2.2380613335296033e-08,
      "logits/chosen": 2.186063289642334,
      "logits/rejected": 2.2187790870666504,
      "logps/chosen": -0.9089058041572571,
      "logps/rejected": -1.0911632776260376,
      "loss": 1.5847,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -2.272264242172241,
      "rewards/margins": 0.45564380288124084,
      "rewards/rejected": -2.727908134460449,
      "step": 410
    },
    {
      "epoch": 0.88698904621961,
      "grad_norm": 7.217288770833436,
      "learning_rate": 1.8673924881500823e-08,
      "logits/chosen": 2.505709409713745,
      "logits/rejected": 2.470183849334717,
      "logps/chosen": -0.8387972116470337,
      "logps/rejected": -1.0221149921417236,
      "loss": 1.6029,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.0969929695129395,
      "rewards/margins": 0.45829471945762634,
      "rewards/rejected": -2.5552875995635986,
      "step": 415
    },
    {
      "epoch": 0.897675661234304,
      "grad_norm": 8.941360081202905,
      "learning_rate": 1.5290766861003475e-08,
      "logits/chosen": 2.191061019897461,
      "logits/rejected": 2.173102378845215,
      "logps/chosen": -0.9252120852470398,
      "logps/rejected": -1.03108811378479,
      "loss": 1.6027,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -2.31303071975708,
      "rewards/margins": 0.2646896243095398,
      "rewards/rejected": -2.5777201652526855,
      "step": 420
    },
    {
      "epoch": 0.9083622762489981,
      "grad_norm": 14.568857335011407,
      "learning_rate": 1.2235870926211616e-08,
      "logits/chosen": 2.0643928050994873,
      "logits/rejected": 2.0896859169006348,
      "logps/chosen": -0.9114057421684265,
      "logps/rejected": -1.0619531869888306,
      "loss": 1.6274,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -2.2785143852233887,
      "rewards/margins": 0.37636831402778625,
      "rewards/rejected": -2.6548826694488525,
      "step": 425
    },
    {
      "epoch": 0.9190488912636923,
      "grad_norm": 12.761707033542287,
      "learning_rate": 9.513509625323518e-09,
      "logits/chosen": 2.1255576610565186,
      "logits/rejected": 2.181107759475708,
      "logps/chosen": -0.85987389087677,
      "logps/rejected": -1.0060930252075195,
      "loss": 1.5736,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -2.149684429168701,
      "rewards/margins": 0.3655479848384857,
      "rewards/rejected": -2.515232563018799,
      "step": 430
    },
    {
      "epoch": 0.9297355062783863,
      "grad_norm": 16.68383920608418,
      "learning_rate": 7.127490426783123e-09,
      "logits/chosen": 2.2676453590393066,
      "logits/rejected": 2.224987506866455,
      "logps/chosen": -0.9531320333480835,
      "logps/rejected": -1.1150267124176025,
      "loss": 1.6026,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.3828301429748535,
      "rewards/margins": 0.4047369956970215,
      "rewards/rejected": -2.787567138671875,
      "step": 435
    },
    {
      "epoch": 0.9404221212930804,
      "grad_norm": 11.429700658399854,
      "learning_rate": 5.08115039419113e-09,
      "logits/chosen": 2.2096710205078125,
      "logits/rejected": 2.2330925464630127,
      "logps/chosen": -0.8609305620193481,
      "logps/rejected": -0.9673101305961609,
      "loss": 1.5978,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -2.1523263454437256,
      "rewards/margins": 0.265949010848999,
      "rewards/rejected": -2.4182753562927246,
      "step": 440
    },
    {
      "epoch": 0.9511087363077745,
      "grad_norm": 7.426544303240404,
      "learning_rate": 3.3773515191196646e-09,
      "logits/chosen": 2.20963716506958,
      "logits/rejected": 2.201622724533081,
      "logps/chosen": -0.8218202590942383,
      "logps/rejected": -0.8649258613586426,
      "loss": 1.6414,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -2.0545506477355957,
      "rewards/margins": 0.10776408016681671,
      "rewards/rejected": -2.1623148918151855,
      "step": 445
    },
    {
      "epoch": 0.9617953513224686,
      "grad_norm": 12.349821491663924,
      "learning_rate": 2.0184767183584474e-09,
      "logits/chosen": 2.325861930847168,
      "logits/rejected": 2.2664694786071777,
      "logps/chosen": -0.9952027201652527,
      "logps/rejected": -1.3025867938995361,
      "loss": 1.6015,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -2.488007068634033,
      "rewards/margins": 0.7684597969055176,
      "rewards/rejected": -3.2564666271209717,
      "step": 450
    },
    {
      "epoch": 0.9724819663371627,
      "grad_norm": 10.41544260252462,
      "learning_rate": 1.0064265011902328e-09,
      "logits/chosen": 2.0219030380249023,
      "logits/rejected": 1.9868360757827759,
      "logps/chosen": -1.0134050846099854,
      "logps/rejected": -1.0333858728408813,
      "loss": 1.6377,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -2.533512592315674,
      "rewards/margins": 0.049951858818531036,
      "rewards/rejected": -2.5834643840789795,
      "step": 455
    },
    {
      "epoch": 0.9831685813518568,
      "grad_norm": 9.044209064035641,
      "learning_rate": 3.4261631135654167e-10,
      "logits/chosen": 2.2376327514648438,
      "logits/rejected": 2.1572484970092773,
      "logps/chosen": -1.0239425897598267,
      "logps/rejected": -1.3165332078933716,
      "loss": 1.5875,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -2.559856414794922,
      "rewards/margins": 0.7314767241477966,
      "rewards/rejected": -3.291332960128784,
      "step": 460
    },
    {
      "epoch": 0.9938551963665508,
      "grad_norm": 11.3294414057108,
      "learning_rate": 2.797454743164174e-11,
      "logits/chosen": 2.186420202255249,
      "logits/rejected": 2.1705265045166016,
      "logps/chosen": -0.875818133354187,
      "logps/rejected": -1.0201952457427979,
      "loss": 1.5967,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -2.1895456314086914,
      "rewards/margins": 0.36094242334365845,
      "rewards/rejected": -2.550488233566284,
      "step": 465
    },
    {
      "epoch": 0.9981298423724285,
      "step": 467,
      "total_flos": 0.0,
      "train_loss": 1.6566840494470372,
      "train_runtime": 5417.7619,
      "train_samples_per_second": 11.052,
      "train_steps_per_second": 0.086
    }
  ],
  "logging_steps": 5,
  "max_steps": 467,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}