Model save

README.md

@@ -0,0 +1,73 @@
+---
+license: mit
+base_model: microsoft/Phi-3-mini-128k-instruct
+tags:
+- trl
+- dpo
+- generated_from_trainer
+model-index:
+- name: phi-3-mini-128k-instruct-simpo-lr-5e-07-gamma-1.5
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# phi-3-mini-128k-instruct-simpo-lr-5e-07-gamma-1.5
+
+This model is a fine-tuned version of [microsoft/Phi-3-mini-128k-instruct](https://huggingface.co/microsoft/Phi-3-mini-128k-instruct) on an unknown dataset.
+It achieves the following results on the evaluation set:
+- Loss: 1.6236
+- Rewards/chosen: -2.2390
+- Rewards/rejected: -2.6457
+- Rewards/accuracies: 0.6210
+- Rewards/margins: 0.4067
+- Logps/rejected: -1.0583
+- Logps/chosen: -0.8956
+- Logits/rejected: 2.0190
+- Logits/chosen: 2.0146
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-07
+- train_batch_size: 2
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
+|:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 1.6417        | 0.8549 | 400  | 1.6236          | -2.2390        | -2.6457          | 0.6210             | 0.4067          | -1.0583        | -0.8956      | 2.0190          | 2.0146        |
+
+
+### Framework versions
+
+- Transformers 4.41.2
+- Pytorch 2.3.1+cu121
+- Datasets 2.20.0
+- Tokenizers 0.19.1

added_tokens.json

@@ -0,0 +1,13 @@
+{
+  "<|assistant|>": 32001,
+  "<|endoftext|>": 32000,
+  "<|end|>": 32007,
+  "<|placeholder1|>": 32002,
+  "<|placeholder2|>": 32003,
+  "<|placeholder3|>": 32004,
+  "<|placeholder4|>": 32005,
+  "<|placeholder5|>": 32008,
+  "<|placeholder6|>": 32009,
+  "<|system|>": 32006,
+  "<|user|>": 32010
+}

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9981298423724285,
+    "total_flos": 0.0,
+    "train_loss": 1.6566840494470372,
+    "train_runtime": 5417.7619,
+    "train_samples": 59876,
+    "train_samples_per_second": 11.052,
+    "train_steps_per_second": 0.086
+}

config.json

@@ -0,0 +1,140 @@
+{
+  "_name_or_path": "microsoft/Phi-3-mini-128k-instruct",
+  "architectures": [
+    "Phi3ForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "auto_map": {
+    "AutoConfig": "microsoft/Phi-3-mini-128k-instruct--configuration_phi3.Phi3Config",
+    "AutoModelForCausalLM": "microsoft/Phi-3-mini-128k-instruct--modeling_phi3.Phi3ForCausalLM",
+    "AutoModelForSequenceClassification": "microsoft/Phi-3-mini-128k-instruct--modeling_phi3.Phi3ForSequenceClassification",
+    "AutoModelForTokenClassification": "microsoft/Phi-3-mini-128k-instruct--modeling_phi3.Phi3ForTokenClassification"
+  },
+  "bos_token_id": 1,
+  "embd_pdrop": 0.0,
+  "eos_token_id": 32000,
+  "hidden_act": "silu",
+  "hidden_size": 3072,
+  "initializer_range": 0.02,
+  "intermediate_size": 8192,
+  "max_position_embeddings": 131072,
+  "model_type": "phi3",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "original_max_position_embeddings": 4096,
+  "pad_token_id": 32000,
+  "resid_pdrop": 0.0,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": {
+    "long_factor": [
+      1.0299999713897705,
+      1.0499999523162842,
+      1.0499999523162842,
+      1.0799999237060547,
+      1.2299998998641968,
+      1.2299998998641968,
+      1.2999999523162842,
+      1.4499999284744263,
+      1.5999999046325684,
+      1.6499998569488525,
+      1.8999998569488525,
+      2.859999895095825,
+      3.68999981880188,
+      5.419999599456787,
+      5.489999771118164,
+      5.489999771118164,
+      9.09000015258789,
+      11.579999923706055,
+      15.65999984741211,
+      15.769999504089355,
+      15.789999961853027,
+      18.360000610351562,
+      21.989999771118164,
+      23.079999923706055,
+      30.009998321533203,
+      32.35000228881836,
+      32.590003967285156,
+      35.56000518798828,
+      39.95000457763672,
+      53.840003967285156,
+      56.20000457763672,
+      57.95000457763672,
+      59.29000473022461,
+      59.77000427246094,
+      59.920005798339844,
+      61.190006256103516,
+      61.96000671386719,
+      62.50000762939453,
+      63.3700065612793,
+      63.48000717163086,
+      63.48000717163086,
+      63.66000747680664,
+      63.850006103515625,
+      64.08000946044922,
+      64.760009765625,
+      64.80001068115234,
+      64.81001281738281,
+      64.81001281738281
+    ],
+    "short_factor": [
+      1.05,
+      1.05,
+      1.05,
+      1.1,
+      1.1,
+      1.1500000000000001,
+      1.2000000000000002,
+      1.2500000000000002,
+      1.3000000000000003,
+      1.3500000000000003,
+      1.5000000000000004,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.000000000000001,
+      2.0500000000000007,
+      2.0500000000000007,
+      2.0500000000000007,
+      2.1000000000000005,
+      2.1000000000000005,
+      2.1000000000000005,
+      2.1500000000000004,
+      2.1500000000000004,
+      2.3499999999999996,
+      2.549999999999999,
+      2.5999999999999988,
+      2.5999999999999988,
+      2.7499999999999982,
+      2.849999999999998,
+      2.849999999999998,
+      2.9499999999999975
+    ],
+    "type": "su"
+  },
+  "rope_theta": 10000.0,
+  "sliding_window": 262144,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.41.2",
+  "use_cache": false,
+  "vocab_size": 32064
+}

generation_config.json

@@ -0,0 +1,11 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": [
+    32000,
+    32001,
+    32007
+  ],
+  "pad_token_id": 32000,
+  "transformers_version": "4.41.2"
+}

model-00001-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d5e43f01963969b46efa3bb114a99bfe20876daf51668f39d7e18ad4288d2a8d
+size 4972489328

model-00002-of-00002.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ac11a3fc0a84be8028c27a527073d4bcb64a3884def72eaa84043491956c8f3
+size 2669692552

model.safetensors.index.json

@@ -0,0 +1,202 @@
+{
+  "metadata": {
+    "total_size": 7642159104
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00002-of-00002.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.0.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.1.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.10.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.11.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.12.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.13.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.14.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.15.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.16.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.17.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.18.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.19.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.2.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.20.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.21.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.22.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.23.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.24.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.25.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.26.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.27.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.28.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.29.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.3.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.30.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.mlp.gate_up_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.31.self_attn.qkv_proj.weight": "model-00002-of-00002.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.4.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.5.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.6.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.7.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.8.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.mlp.gate_up_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
+    "model.layers.9.self_attn.qkv_proj.weight": "model-00001-of-00002.safetensors",
+    "model.norm.weight": "model-00002-of-00002.safetensors"
+  }
+}

runs/Jun21_16-34-46_131-186-38-174/events.out.tfevents.1718988272.131-186-38-174.27809.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d7d0e76205db49909994d392502c92b444b4f2a5399db0d71639e5e88863b958
+size 73664

special_tokens_map.json

@@ -0,0 +1,30 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,130 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": false
+    },
+    "32000": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "32001": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32002": {
+      "content": "<|placeholder1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32003": {
+      "content": "<|placeholder2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32004": {
+      "content": "<|placeholder3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32005": {
+      "content": "<|placeholder4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32006": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32007": {
+      "content": "<|end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32008": {
+      "content": "<|placeholder5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32009": {
+      "content": "<|placeholder6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    },
+    "32010": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": true,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{{ bos_token }}{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|user|>' + '\n' + message['content'] + '<|end|>' + '\n' + '<|assistant|>' + '\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|end|>' + '\n'}}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<|endoftext|>",
+  "legacy": false,
+  "model_max_length": 2048,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "left",
+  "sp_model_kwargs": {},
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 0.9981298423724285,
+    "total_flos": 0.0,
+    "train_loss": 1.6566840494470372,
+    "train_runtime": 5417.7619,
+    "train_samples": 59876,
+    "train_samples_per_second": 11.052,
+    "train_steps_per_second": 0.086
+}

trainer_state.json

@@ -0,0 +1,1468 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9981298423724285,
+  "eval_steps": 400,
+  "global_step": 467,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0021373230029388193,
+      "grad_norm": 42.099143078937416,
+      "learning_rate": 1.0638297872340425e-08,
+      "logits/chosen": 2.2472424507141113,
+      "logits/rejected": 2.264739513397217,
+      "logps/chosen": -0.8019136786460876,
+      "logps/rejected": -0.7812027335166931,
+      "loss": 1.7463,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -2.004784345626831,
+      "rewards/margins": -0.05177761986851692,
+      "rewards/rejected": -1.953006625175476,
+      "step": 1
+    },
+    {
+      "epoch": 0.010686615014694095,
+      "grad_norm": 43.49437439863032,
+      "learning_rate": 5.3191489361702123e-08,
+      "logits/chosen": 2.1162009239196777,
+      "logits/rejected": 2.0480892658233643,
+      "logps/chosen": -1.0361735820770264,
+      "logps/rejected": -1.1913045644760132,
+      "loss": 1.9184,
+      "rewards/accuracies": 0.453125,
+      "rewards/chosen": -2.5904340744018555,
+      "rewards/margins": 0.38782745599746704,
+      "rewards/rejected": -2.9782614707946777,
+      "step": 5
+    },
+    {
+      "epoch": 0.02137323002938819,
+      "grad_norm": 73.58193785863429,
+      "learning_rate": 1.0638297872340425e-07,
+      "logits/chosen": 2.0012879371643066,
+      "logits/rejected": 1.9172722101211548,
+      "logps/chosen": -1.5639755725860596,
+      "logps/rejected": -1.5803353786468506,
+      "loss": 1.9077,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -3.9099388122558594,
+      "rewards/margins": 0.04089919477701187,
+      "rewards/rejected": -3.950838088989258,
+      "step": 10
+    },
+    {
+      "epoch": 0.03205984504408229,
+      "grad_norm": 18.887588182169267,
+      "learning_rate": 1.5957446808510638e-07,
+      "logits/chosen": 1.9121309518814087,
+      "logits/rejected": 1.8687114715576172,
+      "logps/chosen": -1.1899975538253784,
+      "logps/rejected": -1.2847260236740112,
+      "loss": 1.832,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.974993944168091,
+      "rewards/margins": 0.2368215024471283,
+      "rewards/rejected": -3.211815357208252,
+      "step": 15
+    },
+    {
+      "epoch": 0.04274646005877638,
+      "grad_norm": 8.819209234657059,
+      "learning_rate": 2.127659574468085e-07,
+      "logits/chosen": 1.8210818767547607,
+      "logits/rejected": 1.8469772338867188,
+      "logps/chosen": -1.21903657913208,
+      "logps/rejected": -1.7520463466644287,
+      "loss": 1.7601,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.0475919246673584,
+      "rewards/margins": 1.3325239419937134,
+      "rewards/rejected": -4.380115509033203,
+      "step": 20
+    },
+    {
+      "epoch": 0.053433075073470476,
+      "grad_norm": 33.18790384628351,
+      "learning_rate": 2.659574468085106e-07,
+      "logits/chosen": 2.0562968254089355,
+      "logits/rejected": 2.0691914558410645,
+      "logps/chosen": -1.0988093614578247,
+      "logps/rejected": -1.1313331127166748,
+      "loss": 1.815,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.747023344039917,
+      "rewards/margins": 0.08130944520235062,
+      "rewards/rejected": -2.8283333778381348,
+      "step": 25
+    },
+    {
+      "epoch": 0.06411969008816458,
+      "grad_norm": 35.16962019463958,
+      "learning_rate": 3.1914893617021275e-07,
+      "logits/chosen": 1.9061098098754883,
+      "logits/rejected": 1.9218835830688477,
+      "logps/chosen": -1.2370525598526,
+      "logps/rejected": -1.4245789051055908,
+      "loss": 1.6802,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -3.0926311016082764,
+      "rewards/margins": 0.46881628036499023,
+      "rewards/rejected": -3.5614471435546875,
+      "step": 30
+    },
+    {
+      "epoch": 0.07480630510285867,
+      "grad_norm": 44.11653715682163,
+      "learning_rate": 3.7234042553191484e-07,
+      "logits/chosen": 1.9321657419204712,
+      "logits/rejected": 1.955640435218811,
+      "logps/chosen": -1.344245433807373,
+      "logps/rejected": -1.36150062084198,
+      "loss": 1.8113,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -3.3606135845184326,
+      "rewards/margins": 0.0431380569934845,
+      "rewards/rejected": -3.4037513732910156,
+      "step": 35
+    },
+    {
+      "epoch": 0.08549292011755276,
+      "grad_norm": 36.76549905569564,
+      "learning_rate": 4.25531914893617e-07,
+      "logits/chosen": 1.8915932178497314,
+      "logits/rejected": 1.9206931591033936,
+      "logps/chosen": -1.2968499660491943,
+      "logps/rejected": -1.3007665872573853,
+      "loss": 1.8425,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -3.2421250343322754,
+      "rewards/margins": 0.009791326709091663,
+      "rewards/rejected": -3.2519161701202393,
+      "step": 40
+    },
+    {
+      "epoch": 0.09617953513224686,
+      "grad_norm": 17.026888436798433,
+      "learning_rate": 4.787234042553192e-07,
+      "logits/chosen": 1.8335082530975342,
+      "logits/rejected": 1.803972601890564,
+      "logps/chosen": -1.1560873985290527,
+      "logps/rejected": -1.391876459121704,
+      "loss": 1.748,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.8902182579040527,
+      "rewards/margins": 0.5894726514816284,
+      "rewards/rejected": -3.4796910285949707,
+      "step": 45
+    },
+    {
+      "epoch": 0.10686615014694095,
+      "grad_norm": 12.410804374401861,
+      "learning_rate": 4.999370587356267e-07,
+      "logits/chosen": 1.864829659461975,
+      "logits/rejected": 1.8258002996444702,
+      "logps/chosen": -1.0499851703643799,
+      "logps/rejected": -1.454998254776001,
+      "loss": 1.5838,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.6249632835388184,
+      "rewards/margins": 1.0125324726104736,
+      "rewards/rejected": -3.637495517730713,
+      "step": 50
+    },
+    {
+      "epoch": 0.11755276516163506,
+      "grad_norm": 16.97890172462393,
+      "learning_rate": 4.995525324419337e-07,
+      "logits/chosen": 1.8395353555679321,
+      "logits/rejected": 1.8361022472381592,
+      "logps/chosen": -0.9045565724372864,
+      "logps/rejected": -1.1536071300506592,
+      "loss": 1.6796,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.2613916397094727,
+      "rewards/margins": 0.6226258873939514,
+      "rewards/rejected": -2.8840174674987793,
+      "step": 55
+    },
+    {
+      "epoch": 0.12823938017632916,
+      "grad_norm": 18.13814698710649,
+      "learning_rate": 4.988189843662815e-07,
+      "logits/chosen": 1.9378535747528076,
+      "logits/rejected": 1.9659252166748047,
+      "logps/chosen": -1.1384503841400146,
+      "logps/rejected": -1.0124545097351074,
+      "loss": 1.7608,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.846126079559326,
+      "rewards/margins": -0.3149900436401367,
+      "rewards/rejected": -2.5311360359191895,
+      "step": 60
+    },
+    {
+      "epoch": 0.13892599519102325,
+      "grad_norm": 31.166552691677705,
+      "learning_rate": 4.977374404419837e-07,
+      "logits/chosen": 2.1393017768859863,
+      "logits/rejected": 2.103208303451538,
+      "logps/chosen": -0.8744574785232544,
+      "logps/rejected": -0.9523324966430664,
+      "loss": 1.6552,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.186143636703491,
+      "rewards/margins": 0.19468756020069122,
+      "rewards/rejected": -2.380831241607666,
+      "step": 65
+    },
+    {
+      "epoch": 0.14961261020571734,
+      "grad_norm": 52.728695245981314,
+      "learning_rate": 4.963094133060148e-07,
+      "logits/chosen": 2.0067527294158936,
+      "logits/rejected": 1.9597012996673584,
+      "logps/chosen": -0.9174124598503113,
+      "logps/rejected": -1.1131672859191895,
+      "loss": 1.7212,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.2935314178466797,
+      "rewards/margins": 0.4893869459629059,
+      "rewards/rejected": -2.7829184532165527,
+      "step": 70
+    },
+    {
+      "epoch": 0.16029922522041143,
+      "grad_norm": 46.624386507512185,
+      "learning_rate": 4.945369001834514e-07,
+      "logits/chosen": 1.8767681121826172,
+      "logits/rejected": 1.8939720392227173,
+      "logps/chosen": -0.9977117776870728,
+      "logps/rejected": -1.1487284898757935,
+      "loss": 1.6147,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.494279384613037,
+      "rewards/margins": 0.3775419294834137,
+      "rewards/rejected": -2.8718209266662598,
+      "step": 75
+    },
+    {
+      "epoch": 0.17098584023510552,
+      "grad_norm": 13.45137638802328,
+      "learning_rate": 4.924223800941717e-07,
+      "logits/chosen": 1.912676453590393,
+      "logits/rejected": 1.8263626098632812,
+      "logps/chosen": -0.9859241247177124,
+      "logps/rejected": -1.0560874938964844,
+      "loss": 1.651,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.464810609817505,
+      "rewards/margins": 0.17540840804576874,
+      "rewards/rejected": -2.640218734741211,
+      "step": 80
+    },
+    {
+      "epoch": 0.18167245524979964,
+      "grad_norm": 23.876540575284714,
+      "learning_rate": 4.899688103857222e-07,
+      "logits/chosen": 1.9252907037734985,
+      "logits/rejected": 1.9396631717681885,
+      "logps/chosen": -0.9312021136283875,
+      "logps/rejected": -1.1102412939071655,
+      "loss": 1.618,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.328005313873291,
+      "rewards/margins": 0.44759804010391235,
+      "rewards/rejected": -2.7756032943725586,
+      "step": 85
+    },
+    {
+      "epoch": 0.19235907026449373,
+      "grad_norm": 7.705041238913054,
+      "learning_rate": 4.871796225971999e-07,
+      "logits/chosen": 1.9238191843032837,
+      "logits/rejected": 1.8918044567108154,
+      "logps/chosen": -0.9711843729019165,
+      "logps/rejected": -1.096145749092102,
+      "loss": 1.6624,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.4279608726501465,
+      "rewards/margins": 0.3124033510684967,
+      "rewards/rejected": -2.7403643131256104,
+      "step": 90
+    },
+    {
+      "epoch": 0.20304568527918782,
+      "grad_norm": 16.98405162222745,
+      "learning_rate": 4.840587176599343e-07,
+      "logits/chosen": 2.0161519050598145,
+      "logits/rejected": 2.037721872329712,
+      "logps/chosen": -0.9618253707885742,
+      "logps/rejected": -1.1678836345672607,
+      "loss": 1.6553,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.4045631885528564,
+      "rewards/margins": 0.5151457786560059,
+      "rewards/rejected": -2.9197089672088623,
+      "step": 95
+    },
+    {
+      "epoch": 0.2137323002938819,
+      "grad_norm": 26.59166513674174,
+      "learning_rate": 4.806104604416823e-07,
+      "logits/chosen": 1.9551931619644165,
+      "logits/rejected": 1.9663063287734985,
+      "logps/chosen": -0.9686329960823059,
+      "logps/rejected": -0.964672863483429,
+      "loss": 1.6952,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.4215824604034424,
+      "rewards/margins": -0.009900185279548168,
+      "rewards/rejected": -2.41168212890625,
+      "step": 100
+    },
+    {
+      "epoch": 0.224418915308576,
+      "grad_norm": 22.597015867331866,
+      "learning_rate": 4.768396736419662e-07,
+      "logits/chosen": 2.0987541675567627,
+      "logits/rejected": 2.1704182624816895,
+      "logps/chosen": -0.8102301359176636,
+      "logps/rejected": -1.0246448516845703,
+      "loss": 1.6353,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.0255753993988037,
+      "rewards/margins": 0.5360367894172668,
+      "rewards/rejected": -2.561612367630005,
+      "step": 105
+    },
+    {
+      "epoch": 0.2351055303232701,
+      "grad_norm": 7.710202530561847,
+      "learning_rate": 4.7275163104709194e-07,
+      "logits/chosen": 1.8211323022842407,
+      "logits/rejected": 1.8467600345611572,
+      "logps/chosen": -0.8923318982124329,
+      "logps/rejected": -0.9886472821235657,
+      "loss": 1.6826,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.2308297157287598,
+      "rewards/margins": 0.24078825116157532,
+      "rewards/rejected": -2.4716179370880127,
+      "step": 110
+    },
+    {
+      "epoch": 0.2457921453379642,
+      "grad_norm": 9.211974257959199,
+      "learning_rate": 4.683520501542824e-07,
+      "logits/chosen": 1.822178840637207,
+      "logits/rejected": 1.8460540771484375,
+      "logps/chosen": -0.937783420085907,
+      "logps/rejected": -0.9862428903579712,
+      "loss": 1.6087,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.34445858001709,
+      "rewards/margins": 0.12114912271499634,
+      "rewards/rejected": -2.4656076431274414,
+      "step": 115
+    },
+    {
+      "epoch": 0.2564787603526583,
+      "grad_norm": 17.97332526477704,
+      "learning_rate": 4.636470841752404e-07,
+      "logits/chosen": 1.9925601482391357,
+      "logits/rejected": 2.0132079124450684,
+      "logps/chosen": -0.8938210606575012,
+      "logps/rejected": -0.9105901718139648,
+      "loss": 1.6647,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -2.2345528602600098,
+      "rewards/margins": 0.04192250221967697,
+      "rewards/rejected": -2.276475429534912,
+      "step": 120
+    },
+    {
+      "epoch": 0.2671653753673524,
+      "grad_norm": 21.241131663632817,
+      "learning_rate": 4.5864331343032565e-07,
+      "logits/chosen": 1.821298360824585,
+      "logits/rejected": 1.8526639938354492,
+      "logps/chosen": -0.9126847386360168,
+      "logps/rejected": -1.0501748323440552,
+      "loss": 1.7558,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.281712055206299,
+      "rewards/margins": 0.3437250852584839,
+      "rewards/rejected": -2.6254372596740723,
+      "step": 125
+    },
+    {
+      "epoch": 0.2778519903820465,
+      "grad_norm": 18.947973877308787,
+      "learning_rate": 4.533477361453819e-07,
+      "logits/chosen": 2.178891658782959,
+      "logits/rejected": 2.143307685852051,
+      "logps/chosen": -0.9617658853530884,
+      "logps/rejected": -1.223284125328064,
+      "loss": 1.6892,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.404414653778076,
+      "rewards/margins": 0.6537958383560181,
+      "rewards/rejected": -3.0582103729248047,
+      "step": 130
+    },
+    {
+      "epoch": 0.2885386053967406,
+      "grad_norm": 13.76012227966358,
+      "learning_rate": 4.4776775866408533e-07,
+      "logits/chosen": 1.8531124591827393,
+      "logits/rejected": 1.8338654041290283,
+      "logps/chosen": -0.8667293787002563,
+      "logps/rejected": -0.9339718818664551,
+      "loss": 1.658,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.166823148727417,
+      "rewards/margins": 0.16810639202594757,
+      "rewards/rejected": -2.334929943084717,
+      "step": 135
+    },
+    {
+      "epoch": 0.2992252204114347,
+      "grad_norm": 12.832812020829884,
+      "learning_rate": 4.4191118508950277e-07,
+      "logits/chosen": 1.8038040399551392,
+      "logits/rejected": 1.7551212310791016,
+      "logps/chosen": -0.8944159746170044,
+      "logps/rejected": -1.0033025741577148,
+      "loss": 1.6771,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -2.236039638519287,
+      "rewards/margins": 0.2722164988517761,
+      "rewards/rejected": -2.508256435394287,
+      "step": 140
+    },
+    {
+      "epoch": 0.30991183542612877,
+      "grad_norm": 9.09964498419542,
+      "learning_rate": 4.357862063693485e-07,
+      "logits/chosen": 2.027944564819336,
+      "logits/rejected": 2.005831718444824,
+      "logps/chosen": -0.8879717588424683,
+      "logps/rejected": -0.9542590975761414,
+      "loss": 1.7133,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.2199294567108154,
+      "rewards/margins": 0.16571831703186035,
+      "rewards/rejected": -2.3856475353240967,
+      "step": 145
+    },
+    {
+      "epoch": 0.32059845044082286,
+      "grad_norm": 24.685014270210495,
+      "learning_rate": 4.294013888402029e-07,
+      "logits/chosen": 2.138366222381592,
+      "logits/rejected": 2.2007527351379395,
+      "logps/chosen": -0.8750768899917603,
+      "logps/rejected": -1.0953787565231323,
+      "loss": 1.6625,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.187692165374756,
+      "rewards/margins": 0.5507549047470093,
+      "rewards/rejected": -2.7384471893310547,
+      "step": 150
+    },
+    {
+      "epoch": 0.33128506545551695,
+      "grad_norm": 24.61662782262514,
+      "learning_rate": 4.227656622467162e-07,
+      "logits/chosen": 1.911320686340332,
+      "logits/rejected": 1.9191503524780273,
+      "logps/chosen": -0.8599516153335571,
+      "logps/rejected": -1.1553843021392822,
+      "loss": 1.6495,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.149878978729248,
+      "rewards/margins": 0.7385817766189575,
+      "rewards/rejected": -2.888460874557495,
+      "step": 155
+    },
+    {
+      "epoch": 0.34197168047021104,
+      "grad_norm": 9.846236022517108,
+      "learning_rate": 4.158883072525528e-07,
+      "logits/chosen": 2.061549425125122,
+      "logits/rejected": 2.0822834968566895,
+      "logps/chosen": -0.8990328907966614,
+      "logps/rejected": -0.985998809337616,
+      "loss": 1.6348,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -2.247582197189331,
+      "rewards/margins": 0.21741452813148499,
+      "rewards/rejected": -2.464996814727783,
+      "step": 160
+    },
+    {
+      "epoch": 0.3526582954849052,
+      "grad_norm": 19.106656482086205,
+      "learning_rate": 4.087789424605447e-07,
+      "logits/chosen": 2.265204906463623,
+      "logits/rejected": 2.245617151260376,
+      "logps/chosen": -0.818160355091095,
+      "logps/rejected": -0.9623705744743347,
+      "loss": 1.618,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.045400857925415,
+      "rewards/margins": 0.3605257570743561,
+      "rewards/rejected": -2.405926465988159,
+      "step": 165
+    },
+    {
+      "epoch": 0.36334491049959927,
+      "grad_norm": 14.919110153191296,
+      "learning_rate": 4.0144751096020497e-07,
+      "logits/chosen": 1.9928451776504517,
+      "logits/rejected": 1.9697974920272827,
+      "logps/chosen": -1.013695478439331,
+      "logps/rejected": -1.2301123142242432,
+      "loss": 1.5848,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.534238815307617,
+      "rewards/margins": 0.5410416722297668,
+      "rewards/rejected": -3.0752806663513184,
+      "step": 170
+    },
+    {
+      "epoch": 0.37403152551429336,
+      "grad_norm": 11.830869809507767,
+      "learning_rate": 3.939042664214184e-07,
+      "logits/chosen": 2.031292676925659,
+      "logits/rejected": 2.066436767578125,
+      "logps/chosen": -0.97381991147995,
+      "logps/rejected": -1.2112761735916138,
+      "loss": 1.5954,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -2.4345498085021973,
+      "rewards/margins": 0.5936408638954163,
+      "rewards/rejected": -3.0281906127929688,
+      "step": 175
+    },
+    {
+      "epoch": 0.38471814052898745,
+      "grad_norm": 14.628170167780459,
+      "learning_rate": 3.8615975875375676e-07,
+      "logits/chosen": 2.258387804031372,
+      "logits/rejected": 2.3218982219696045,
+      "logps/chosen": -1.0137016773223877,
+      "logps/rejected": -1.1529831886291504,
+      "loss": 1.6695,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.534254312515259,
+      "rewards/margins": 0.3482036888599396,
+      "rewards/rejected": -2.882457971572876,
+      "step": 180
+    },
+    {
+      "epoch": 0.39540475554368154,
+      "grad_norm": 16.49947715615614,
+      "learning_rate": 3.7822481935147655e-07,
+      "logits/chosen": 2.0560050010681152,
+      "logits/rejected": 2.1167311668395996,
+      "logps/chosen": -0.9939271211624146,
+      "logps/rejected": -1.1564804315567017,
+      "loss": 1.6754,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.4848177433013916,
+      "rewards/margins": 0.40638333559036255,
+      "rewards/rejected": -2.8912010192871094,
+      "step": 185
+    },
+    {
+      "epoch": 0.40609137055837563,
+      "grad_norm": 13.5081717360946,
+      "learning_rate": 3.7011054594483443e-07,
+      "logits/chosen": 2.1290013790130615,
+      "logits/rejected": 2.223310947418213,
+      "logps/chosen": -0.9123891592025757,
+      "logps/rejected": -1.1287665367126465,
+      "loss": 1.5831,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.280972957611084,
+      "rewards/margins": 0.5409437417984009,
+      "rewards/rejected": -2.8219165802001953,
+      "step": 190
+    },
+    {
+      "epoch": 0.4167779855730697,
+      "grad_norm": 17.61863203205812,
+      "learning_rate": 3.618282870789081e-07,
+      "logits/chosen": 2.1478934288024902,
+      "logits/rejected": 2.1439719200134277,
+      "logps/chosen": -0.974346935749054,
+      "logps/rejected": -1.1090627908706665,
+      "loss": 1.64,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.4358673095703125,
+      "rewards/margins": 0.33678966760635376,
+      "rewards/rejected": -2.7726569175720215,
+      "step": 195
+    },
+    {
+      "epoch": 0.4274646005877638,
+      "grad_norm": 12.457201720784246,
+      "learning_rate": 3.5338962624163016e-07,
+      "logits/chosen": 2.2304863929748535,
+      "logits/rejected": 2.2417056560516357,
+      "logps/chosen": -0.9161917567253113,
+      "logps/rejected": -1.0349229574203491,
+      "loss": 1.672,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -2.2904791831970215,
+      "rewards/margins": 0.29682818055152893,
+      "rewards/rejected": -2.5873074531555176,
+      "step": 200
+    },
+    {
+      "epoch": 0.4381512156024579,
+      "grad_norm": 25.419906192801392,
+      "learning_rate": 3.448063656632321e-07,
+      "logits/chosen": 2.2168259620666504,
+      "logits/rejected": 2.2494800090789795,
+      "logps/chosen": -0.9495501518249512,
+      "logps/rejected": -1.0588171482086182,
+      "loss": 1.6462,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.373875379562378,
+      "rewards/margins": 0.2731671929359436,
+      "rewards/rejected": -2.647042751312256,
+      "step": 205
+    },
+    {
+      "epoch": 0.448837830617152,
+      "grad_norm": 31.43495532901771,
+      "learning_rate": 3.360905098097587e-07,
+      "logits/chosen": 2.0355324745178223,
+      "logits/rejected": 2.0058634281158447,
+      "logps/chosen": -0.9030116200447083,
+      "logps/rejected": -1.100672960281372,
+      "loss": 1.7332,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.2575290203094482,
+      "rewards/margins": 0.49415332078933716,
+      "rewards/rejected": -2.751682758331299,
+      "step": 210
+    },
+    {
+      "epoch": 0.45952444563184613,
+      "grad_norm": 9.866165199153304,
+      "learning_rate": 3.272542485937368e-07,
+      "logits/chosen": 2.188046932220459,
+      "logits/rejected": 2.209195375442505,
+      "logps/chosen": -0.8452509045600891,
+      "logps/rejected": -0.9856406450271606,
+      "loss": 1.6173,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.1131272315979004,
+      "rewards/margins": 0.3509744107723236,
+      "rewards/rejected": -2.464101791381836,
+      "step": 215
+    },
+    {
+      "epoch": 0.4702110606465402,
+      "grad_norm": 11.268271977292386,
+      "learning_rate": 3.1830994032548e-07,
+      "logits/chosen": 2.2558753490448,
+      "logits/rejected": 2.271909475326538,
+      "logps/chosen": -0.9621235728263855,
+      "logps/rejected": -1.0470136404037476,
+      "loss": 1.6623,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.405308961868286,
+      "rewards/margins": 0.21222510933876038,
+      "rewards/rejected": -2.6175341606140137,
+      "step": 220
+    },
+    {
+      "epoch": 0.4808976756612343,
+      "grad_norm": 8.220154130520008,
+      "learning_rate": 3.0927009442887437e-07,
+      "logits/chosen": 2.033316135406494,
+      "logits/rejected": 2.0451745986938477,
+      "logps/chosen": -0.9348015785217285,
+      "logps/rejected": -0.9307538866996765,
+      "loss": 1.7025,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.3370039463043213,
+      "rewards/margins": -0.010119226761162281,
+      "rewards/rejected": -2.3268847465515137,
+      "step": 225
+    },
+    {
+      "epoch": 0.4915842906759284,
+      "grad_norm": 8.761484125412297,
+      "learning_rate": 3.001473539458182e-07,
+      "logits/chosen": 2.039243221282959,
+      "logits/rejected": 2.0847630500793457,
+      "logps/chosen": -0.9077224731445312,
+      "logps/rejected": -0.9697052240371704,
+      "loss": 1.6369,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.269306182861328,
+      "rewards/margins": 0.15495693683624268,
+      "rewards/rejected": -2.4242632389068604,
+      "step": 230
+    },
+    {
+      "epoch": 0.5022709056906225,
+      "grad_norm": 21.069821104434805,
+      "learning_rate": 2.909544778537844e-07,
+      "logits/chosen": 2.1515793800354004,
+      "logits/rejected": 2.0550239086151123,
+      "logps/chosen": -0.9554904699325562,
+      "logps/rejected": -1.0575495958328247,
+      "loss": 1.6471,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -2.3887264728546143,
+      "rewards/margins": 0.2551475167274475,
+      "rewards/rejected": -2.643873691558838,
+      "step": 235
+    },
+    {
+      "epoch": 0.5129575207053166,
+      "grad_norm": 18.45689011337618,
+      "learning_rate": 2.817043232212371e-07,
+      "logits/chosen": 2.3277316093444824,
+      "logits/rejected": 2.359060287475586,
+      "logps/chosen": -0.9175651669502258,
+      "logps/rejected": -1.075629472732544,
+      "loss": 1.5723,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.2939131259918213,
+      "rewards/margins": 0.39516064524650574,
+      "rewards/rejected": -2.6890735626220703,
+      "step": 240
+    },
+    {
+      "epoch": 0.5236441357200107,
+      "grad_norm": 5.945491870174955,
+      "learning_rate": 2.7240982722585837e-07,
+      "logits/chosen": 2.2272980213165283,
+      "logits/rejected": 2.216254472732544,
+      "logps/chosen": -0.887627124786377,
+      "logps/rejected": -1.0360788106918335,
+      "loss": 1.6259,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.2190678119659424,
+      "rewards/margins": 0.3711293637752533,
+      "rewards/rejected": -2.5901970863342285,
+      "step": 245
+    },
+    {
+      "epoch": 0.5343307507347048,
+      "grad_norm": 10.042684286997996,
+      "learning_rate": 2.63083989060736e-07,
+      "logits/chosen": 2.162696361541748,
+      "logits/rejected": 2.178349494934082,
+      "logps/chosen": -0.9883350133895874,
+      "logps/rejected": -1.1994340419769287,
+      "loss": 1.6589,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.4708378314971924,
+      "rewards/margins": 0.5277475118637085,
+      "rewards/rejected": -2.9985852241516113,
+      "step": 250
+    },
+    {
+      "epoch": 0.5450173657493989,
+      "grad_norm": 10.520900876521566,
+      "learning_rate": 2.537398517538159e-07,
+      "logits/chosen": 2.2908027172088623,
+      "logits/rejected": 2.235353708267212,
+      "logps/chosen": -0.9431841969490051,
+      "logps/rejected": -1.015700101852417,
+      "loss": 1.6741,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.3579602241516113,
+      "rewards/margins": 0.18129023909568787,
+      "rewards/rejected": -2.539250612258911,
+      "step": 255
+    },
+    {
+      "epoch": 0.555703980764093,
+      "grad_norm": 9.313778052567603,
+      "learning_rate": 2.4439048392604877e-07,
+      "logits/chosen": 2.1532225608825684,
+      "logits/rejected": 2.2637267112731934,
+      "logps/chosen": -0.9131342768669128,
+      "logps/rejected": -1.2010549306869507,
+      "loss": 1.5779,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -2.2828357219696045,
+      "rewards/margins": 0.7198014259338379,
+      "rewards/rejected": -3.0026373863220215,
+      "step": 260
+    },
+    {
+      "epoch": 0.566390595778787,
+      "grad_norm": 17.127300405069718,
+      "learning_rate": 2.3504896151374144e-07,
+      "logits/chosen": 2.2980525493621826,
+      "logits/rejected": 2.2274222373962402,
+      "logps/chosen": -0.9766885042190552,
+      "logps/rejected": -1.0872061252593994,
+      "loss": 1.6741,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.4417214393615723,
+      "rewards/margins": 0.2762938141822815,
+      "rewards/rejected": -2.718015193939209,
+      "step": 265
+    },
+    {
+      "epoch": 0.5770772107934812,
+      "grad_norm": 14.096322545468484,
+      "learning_rate": 2.2572834948067795e-07,
+      "logits/chosen": 2.13866925239563,
+      "logits/rejected": 2.09075665473938,
+      "logps/chosen": -0.866176962852478,
+      "logps/rejected": -1.0079139471054077,
+      "loss": 1.6445,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.165442705154419,
+      "rewards/margins": 0.3543422520160675,
+      "rewards/rejected": -2.519784688949585,
+      "step": 270
+    },
+    {
+      "epoch": 0.5877638258081752,
+      "grad_norm": 11.519350142557252,
+      "learning_rate": 2.164416835455862e-07,
+      "logits/chosen": 1.9599807262420654,
+      "logits/rejected": 1.9166367053985596,
+      "logps/chosen": -0.9016801714897156,
+      "logps/rejected": -1.060001015663147,
+      "loss": 1.6308,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.2542004585266113,
+      "rewards/margins": 0.39580219984054565,
+      "rewards/rejected": -2.6500024795532227,
+      "step": 275
+    },
+    {
+      "epoch": 0.5984504408228694,
+      "grad_norm": 9.94425188158257,
+      "learning_rate": 2.072019519505062e-07,
+      "logits/chosen": 2.006683349609375,
+      "logits/rejected": 1.9877275228500366,
+      "logps/chosen": -0.9040509462356567,
+      "logps/rejected": -1.0043412446975708,
+      "loss": 1.6201,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.260127305984497,
+      "rewards/margins": 0.2507258355617523,
+      "rewards/rejected": -2.5108530521392822,
+      "step": 280
+    },
+    {
+      "epoch": 0.6091370558375635,
+      "grad_norm": 17.126362799060438,
+      "learning_rate": 1.980220772955602e-07,
+      "logits/chosen": 2.040419816970825,
+      "logits/rejected": 1.9795433282852173,
+      "logps/chosen": -1.0711162090301514,
+      "logps/rejected": -1.1054927110671997,
+      "loss": 1.6696,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.677790641784668,
+      "rewards/margins": 0.08594076335430145,
+      "rewards/rejected": -2.7637314796447754,
+      "step": 285
+    },
+    {
+      "epoch": 0.6198236708522575,
+      "grad_norm": 13.492215975630849,
+      "learning_rate": 1.8891489846552644e-07,
+      "logits/chosen": 2.3427670001983643,
+      "logits/rejected": 2.3189902305603027,
+      "logps/chosen": -0.9356976747512817,
+      "logps/rejected": -1.0561234951019287,
+      "loss": 1.6313,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.3392443656921387,
+      "rewards/margins": 0.30106478929519653,
+      "rewards/rejected": -2.6403088569641113,
+      "step": 290
+    },
+    {
+      "epoch": 0.6305102858669517,
+      "grad_norm": 6.893975837082491,
+      "learning_rate": 1.7989315267349933e-07,
+      "logits/chosen": 2.3545644283294678,
+      "logits/rejected": 2.3726115226745605,
+      "logps/chosen": -0.9937087893486023,
+      "logps/rejected": -1.0849647521972656,
+      "loss": 1.6816,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.484271764755249,
+      "rewards/margins": 0.22813992202281952,
+      "rewards/rejected": -2.712411642074585,
+      "step": 295
+    },
+    {
+      "epoch": 0.6411969008816457,
+      "grad_norm": 6.6767839699246405,
+      "learning_rate": 1.7096945764674398e-07,
+      "logits/chosen": 2.0134410858154297,
+      "logits/rejected": 1.9672603607177734,
+      "logps/chosen": -1.038081169128418,
+      "logps/rejected": -1.0271114110946655,
+      "loss": 1.6598,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -2.595202922821045,
+      "rewards/margins": -0.027424562722444534,
+      "rewards/rejected": -2.5677783489227295,
+      "step": 300
+    },
+    {
+      "epoch": 0.6518835158963399,
+      "grad_norm": 8.402566753074327,
+      "learning_rate": 1.621562939796643e-07,
+      "logits/chosen": 2.099931240081787,
+      "logits/rejected": 2.131023406982422,
+      "logps/chosen": -0.9801462888717651,
+      "logps/rejected": -1.1142746210098267,
+      "loss": 1.6239,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -2.4503657817840576,
+      "rewards/margins": 0.33532071113586426,
+      "rewards/rejected": -2.785686492919922,
+      "step": 305
+    },
+    {
+      "epoch": 0.6625701309110339,
+      "grad_norm": 13.484008579562406,
+      "learning_rate": 1.5346598767856345e-07,
+      "logits/chosen": 2.3738818168640137,
+      "logits/rejected": 2.2915918827056885,
+      "logps/chosen": -0.9194884300231934,
+      "logps/rejected": -1.1123372316360474,
+      "loss": 1.6509,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.2987208366394043,
+      "rewards/margins": 0.4821222722530365,
+      "rewards/rejected": -2.7808432579040527,
+      "step": 310
+    },
+    {
+      "epoch": 0.673256745925728,
+      "grad_norm": 11.276062854222866,
+      "learning_rate": 1.4491069292260866e-07,
+      "logits/chosen": 2.0688905715942383,
+      "logits/rejected": 2.0706334114074707,
+      "logps/chosen": -0.9416360855102539,
+      "logps/rejected": -0.9889582395553589,
+      "loss": 1.6263,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.354090452194214,
+      "rewards/margins": 0.11830528825521469,
+      "rewards/rejected": -2.472395658493042,
+      "step": 315
+    },
+    {
+      "epoch": 0.6839433609404221,
+      "grad_norm": 12.406779732367644,
+      "learning_rate": 1.365023750651133e-07,
+      "logits/chosen": 2.029076337814331,
+      "logits/rejected": 2.07525634765625,
+      "logps/chosen": -0.8669158220291138,
+      "logps/rejected": -1.1443169116973877,
+      "loss": 1.5861,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.1672892570495605,
+      "rewards/margins": 0.6935025453567505,
+      "rewards/rejected": -2.8607916831970215,
+      "step": 320
+    },
+    {
+      "epoch": 0.6946299759551162,
+      "grad_norm": 9.410463016483124,
+      "learning_rate": 1.2825279389890818e-07,
+      "logits/chosen": 2.2680742740631104,
+      "logits/rejected": 2.2626852989196777,
+      "logps/chosen": -0.9127257466316223,
+      "logps/rejected": -1.036811113357544,
+      "loss": 1.5807,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.2818140983581543,
+      "rewards/margins": 0.31021344661712646,
+      "rewards/rejected": -2.5920276641845703,
+      "step": 325
+    },
+    {
+      "epoch": 0.7053165909698104,
+      "grad_norm": 13.015902838409264,
+      "learning_rate": 1.201734872092077e-07,
+      "logits/chosen": 2.0204429626464844,
+      "logits/rejected": 1.9889049530029297,
+      "logps/chosen": -0.9624814987182617,
+      "logps/rejected": -1.4148118495941162,
+      "loss": 1.6247,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.4062037467956543,
+      "rewards/margins": 1.130826473236084,
+      "rewards/rejected": -3.537029981613159,
+      "step": 330
+    },
+    {
+      "epoch": 0.7160032059845044,
+      "grad_norm": 10.457412577224535,
+      "learning_rate": 1.1227575463697439e-07,
+      "logits/chosen": 2.1002488136291504,
+      "logits/rejected": 2.1578989028930664,
+      "logps/chosen": -0.9219167828559875,
+      "logps/rejected": -1.0541021823883057,
+      "loss": 1.6126,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.3047919273376465,
+      "rewards/margins": 0.33046382665634155,
+      "rewards/rejected": -2.635255813598633,
+      "step": 335
+    },
+    {
+      "epoch": 0.7266898209991985,
+      "grad_norm": 13.23573306912224,
+      "learning_rate": 1.0457064187534861e-07,
+      "logits/chosen": 2.2117228507995605,
+      "logits/rejected": 2.291081666946411,
+      "logps/chosen": -0.8333222270011902,
+      "logps/rejected": -1.0662933588027954,
+      "loss": 1.611,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.083305597305298,
+      "rewards/margins": 0.5824280977249146,
+      "rewards/rejected": -2.665733575820923,
+      "step": 340
+    },
+    {
+      "epoch": 0.7373764360138926,
+      "grad_norm": 7.317920448430426,
+      "learning_rate": 9.706892522124838e-08,
+      "logits/chosen": 2.1216635704040527,
+      "logits/rejected": 2.178093194961548,
+      "logps/chosen": -1.0588462352752686,
+      "logps/rejected": -1.0970056056976318,
+      "loss": 1.5811,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.647115468978882,
+      "rewards/margins": 0.09539877623319626,
+      "rewards/rejected": -2.742514133453369,
+      "step": 345
+    },
+    {
+      "epoch": 0.7480630510285867,
+      "grad_norm": 14.193162676817792,
+      "learning_rate": 8.978109650374396e-08,
+      "logits/chosen": 2.2546029090881348,
+      "logits/rejected": 2.294689893722534,
+      "logps/chosen": -0.9023507833480835,
+      "logps/rejected": -1.162570595741272,
+      "loss": 1.6171,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.2558767795562744,
+      "rewards/margins": 0.6505497694015503,
+      "rewards/rejected": -2.9064266681671143,
+      "step": 350
+    },
+    {
+      "epoch": 0.7587496660432808,
+      "grad_norm": 12.52863776377329,
+      "learning_rate": 8.271734841028552e-08,
+      "logits/chosen": 2.245272636413574,
+      "logits/rejected": 2.2576751708984375,
+      "logps/chosen": -1.005053997039795,
+      "logps/rejected": -1.0007752180099487,
+      "loss": 1.6536,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -2.512634754180908,
+      "rewards/margins": -0.010696577839553356,
+      "rewards/rejected": -2.5019383430480957,
+      "step": 355
+    },
+    {
+      "epoch": 0.7694362810579749,
+      "grad_norm": 10.369189534811127,
+      "learning_rate": 7.588756023130833e-08,
+      "logits/chosen": 2.318972587585449,
+      "logits/rejected": 2.281419515609741,
+      "logps/chosen": -1.0578750371932983,
+      "logps/rejected": -1.3604905605316162,
+      "loss": 1.6205,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.6446874141693115,
+      "rewards/margins": 0.7565388679504395,
+      "rewards/rejected": -3.40122652053833,
+      "step": 360
+    },
+    {
+      "epoch": 0.7801228960726689,
+      "grad_norm": 6.075365195709392,
+      "learning_rate": 6.930128404315214e-08,
+      "logits/chosen": 2.0289673805236816,
+      "logits/rejected": 2.0680978298187256,
+      "logps/chosen": -0.976359486579895,
+      "logps/rejected": -1.0074145793914795,
+      "loss": 1.6246,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.440898895263672,
+      "rewards/margins": 0.07763750851154327,
+      "rewards/rejected": -2.518536329269409,
+      "step": 365
+    },
+    {
+      "epoch": 0.7908095110873631,
+      "grad_norm": 16.214823251732284,
+      "learning_rate": 6.296773134861824e-08,
+      "logits/chosen": 2.2002553939819336,
+      "logits/rejected": 2.2343714237213135,
+      "logps/chosen": -0.9719416499137878,
+      "logps/rejected": -1.023166537284851,
+      "loss": 1.6894,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.429853916168213,
+      "rewards/margins": 0.12806224822998047,
+      "rewards/rejected": -2.5579161643981934,
+      "step": 370
+    },
+    {
+      "epoch": 0.8014961261020572,
+      "grad_norm": 10.840080143128308,
+      "learning_rate": 5.6895760193850145e-08,
+      "logits/chosen": 2.1092734336853027,
+      "logits/rejected": 2.168839693069458,
+      "logps/chosen": -1.0623986721038818,
+      "logps/rejected": -1.1893246173858643,
+      "loss": 1.6248,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.6559970378875732,
+      "rewards/margins": 0.31731492280960083,
+      "rewards/rejected": -2.9733119010925293,
+      "step": 375
+    },
+    {
+      "epoch": 0.8121827411167513,
+      "grad_norm": 14.986128731201607,
+      "learning_rate": 5.109386277955477e-08,
+      "logits/chosen": 2.400944232940674,
+      "logits/rejected": 2.424126625061035,
+      "logps/chosen": -1.0006974935531616,
+      "logps/rejected": -1.0918558835983276,
+      "loss": 1.6152,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.501743793487549,
+      "rewards/margins": 0.22789612412452698,
+      "rewards/rejected": -2.729639768600464,
+      "step": 380
+    },
+    {
+      "epoch": 0.8228693561314454,
+      "grad_norm": 11.04509407630343,
+      "learning_rate": 4.557015358389216e-08,
+      "logits/chosen": 2.2161147594451904,
+      "logits/rejected": 2.255049705505371,
+      "logps/chosen": -0.9763911962509155,
+      "logps/rejected": -1.2085295915603638,
+      "loss": 1.6246,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.4409780502319336,
+      "rewards/margins": 0.5803458094596863,
+      "rewards/rejected": -3.0213241577148438,
+      "step": 385
+    },
+    {
+      "epoch": 0.8335559711461394,
+      "grad_norm": 17.08993723906078,
+      "learning_rate": 4.0332358013644015e-08,
+      "logits/chosen": 2.3219351768493652,
+      "logits/rejected": 2.316385507583618,
+      "logps/chosen": -0.8949235081672668,
+      "logps/rejected": -1.2672293186187744,
+      "loss": 1.6161,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.2373085021972656,
+      "rewards/margins": 0.9307647943496704,
+      "rewards/rejected": -3.1680736541748047,
+      "step": 390
+    },
+    {
+      "epoch": 0.8442425861608336,
+      "grad_norm": 7.486937950213667,
+      "learning_rate": 3.538780159953347e-08,
+      "logits/chosen": 2.171618700027466,
+      "logits/rejected": 2.133547067642212,
+      "logps/chosen": -0.913979172706604,
+      "logps/rejected": -1.011819839477539,
+      "loss": 1.6306,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.284947633743286,
+      "rewards/margins": 0.24460192024707794,
+      "rewards/rejected": -2.5295493602752686,
+      "step": 395
+    },
+    {
+      "epoch": 0.8549292011755276,
+      "grad_norm": 12.731226626352624,
+      "learning_rate": 3.074339975080836e-08,
+      "logits/chosen": 2.207425594329834,
+      "logits/rejected": 2.2549290657043457,
+      "logps/chosen": -1.030874490737915,
+      "logps/rejected": -1.2776422500610352,
+      "loss": 1.6417,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.577186107635498,
+      "rewards/margins": 0.6169193983078003,
+      "rewards/rejected": -3.194105863571167,
+      "step": 400
+    },
+    {
+      "epoch": 0.8549292011755276,
+      "eval_logits/chosen": 2.0145604610443115,
+      "eval_logits/rejected": 2.0190441608428955,
+      "eval_logps/chosen": -0.8955881595611572,
+      "eval_logps/rejected": -1.058284044265747,
+      "eval_loss": 1.6235809326171875,
+      "eval_rewards/accuracies": 0.6209677457809448,
+      "eval_rewards/chosen": -2.2389702796936035,
+      "eval_rewards/margins": 0.40673962235450745,
+      "eval_rewards/rejected": -2.645709753036499,
+      "eval_runtime": 44.336,
+      "eval_samples_per_second": 44.23,
+      "eval_steps_per_second": 1.398,
+      "step": 400
+    },
+    {
+      "epoch": 0.8656158161902218,
+      "grad_norm": 10.867554141726604,
+      "learning_rate": 2.6405648083415833e-08,
+      "logits/chosen": 2.151744842529297,
+      "logits/rejected": 2.200650691986084,
+      "logps/chosen": -0.980237603187561,
+      "logps/rejected": -1.0543644428253174,
+      "loss": 1.7482,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.450594186782837,
+      "rewards/margins": 0.18531720340251923,
+      "rewards/rejected": -2.635911226272583,
+      "step": 405
+    },
+    {
+      "epoch": 0.8763024312049158,
+      "grad_norm": 8.929503879536968,
+      "learning_rate": 2.2380613335296033e-08,
+      "logits/chosen": 2.186063289642334,
+      "logits/rejected": 2.2187790870666504,
+      "logps/chosen": -0.9089058041572571,
+      "logps/rejected": -1.0911632776260376,
+      "loss": 1.5847,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.272264242172241,
+      "rewards/margins": 0.45564380288124084,
+      "rewards/rejected": -2.727908134460449,
+      "step": 410
+    },
+    {
+      "epoch": 0.88698904621961,
+      "grad_norm": 7.217288770833436,
+      "learning_rate": 1.8673924881500823e-08,
+      "logits/chosen": 2.505709409713745,
+      "logits/rejected": 2.470183849334717,
+      "logps/chosen": -0.8387972116470337,
+      "logps/rejected": -1.0221149921417236,
+      "loss": 1.6029,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.0969929695129395,
+      "rewards/margins": 0.45829471945762634,
+      "rewards/rejected": -2.5552875995635986,
+      "step": 415
+    },
+    {
+      "epoch": 0.897675661234304,
+      "grad_norm": 8.941360081202905,
+      "learning_rate": 1.5290766861003475e-08,
+      "logits/chosen": 2.191061019897461,
+      "logits/rejected": 2.173102378845215,
+      "logps/chosen": -0.9252120852470398,
+      "logps/rejected": -1.03108811378479,
+      "loss": 1.6027,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.31303071975708,
+      "rewards/margins": 0.2646896243095398,
+      "rewards/rejected": -2.5777201652526855,
+      "step": 420
+    },
+    {
+      "epoch": 0.9083622762489981,
+      "grad_norm": 14.568857335011407,
+      "learning_rate": 1.2235870926211616e-08,
+      "logits/chosen": 2.0643928050994873,
+      "logits/rejected": 2.0896859169006348,
+      "logps/chosen": -0.9114057421684265,
+      "logps/rejected": -1.0619531869888306,
+      "loss": 1.6274,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.2785143852233887,
+      "rewards/margins": 0.37636831402778625,
+      "rewards/rejected": -2.6548826694488525,
+      "step": 425
+    },
+    {
+      "epoch": 0.9190488912636923,
+      "grad_norm": 12.761707033542287,
+      "learning_rate": 9.513509625323518e-09,
+      "logits/chosen": 2.1255576610565186,
+      "logits/rejected": 2.181107759475708,
+      "logps/chosen": -0.85987389087677,
+      "logps/rejected": -1.0060930252075195,
+      "loss": 1.5736,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.149684429168701,
+      "rewards/margins": 0.3655479848384857,
+      "rewards/rejected": -2.515232563018799,
+      "step": 430
+    },
+    {
+      "epoch": 0.9297355062783863,
+      "grad_norm": 16.68383920608418,
+      "learning_rate": 7.127490426783123e-09,
+      "logits/chosen": 2.2676453590393066,
+      "logits/rejected": 2.224987506866455,
+      "logps/chosen": -0.9531320333480835,
+      "logps/rejected": -1.1150267124176025,
+      "loss": 1.6026,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.3828301429748535,
+      "rewards/margins": 0.4047369956970215,
+      "rewards/rejected": -2.787567138671875,
+      "step": 435
+    },
+    {
+      "epoch": 0.9404221212930804,
+      "grad_norm": 11.429700658399854,
+      "learning_rate": 5.08115039419113e-09,
+      "logits/chosen": 2.2096710205078125,
+      "logits/rejected": 2.2330925464630127,
+      "logps/chosen": -0.8609305620193481,
+      "logps/rejected": -0.9673101305961609,
+      "loss": 1.5978,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.1523263454437256,
+      "rewards/margins": 0.265949010848999,
+      "rewards/rejected": -2.4182753562927246,
+      "step": 440
+    },
+    {
+      "epoch": 0.9511087363077745,
+      "grad_norm": 7.426544303240404,
+      "learning_rate": 3.3773515191196646e-09,
+      "logits/chosen": 2.20963716506958,
+      "logits/rejected": 2.201622724533081,
+      "logps/chosen": -0.8218202590942383,
+      "logps/rejected": -0.8649258613586426,
+      "loss": 1.6414,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.0545506477355957,
+      "rewards/margins": 0.10776408016681671,
+      "rewards/rejected": -2.1623148918151855,
+      "step": 445
+    },
+    {
+      "epoch": 0.9617953513224686,
+      "grad_norm": 12.349821491663924,
+      "learning_rate": 2.0184767183584474e-09,
+      "logits/chosen": 2.325861930847168,
+      "logits/rejected": 2.2664694786071777,
+      "logps/chosen": -0.9952027201652527,
+      "logps/rejected": -1.3025867938995361,
+      "loss": 1.6015,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.488007068634033,
+      "rewards/margins": 0.7684597969055176,
+      "rewards/rejected": -3.2564666271209717,
+      "step": 450
+    },
+    {
+      "epoch": 0.9724819663371627,
+      "grad_norm": 10.41544260252462,
+      "learning_rate": 1.0064265011902328e-09,
+      "logits/chosen": 2.0219030380249023,
+      "logits/rejected": 1.9868360757827759,
+      "logps/chosen": -1.0134050846099854,
+      "logps/rejected": -1.0333858728408813,
+      "loss": 1.6377,
+      "rewards/accuracies": 0.4000000059604645,
+      "rewards/chosen": -2.533512592315674,
+      "rewards/margins": 0.049951858818531036,
+      "rewards/rejected": -2.5834643840789795,
+      "step": 455
+    },
+    {
+      "epoch": 0.9831685813518568,
+      "grad_norm": 9.044209064035641,
+      "learning_rate": 3.4261631135654167e-10,
+      "logits/chosen": 2.2376327514648438,
+      "logits/rejected": 2.1572484970092773,
+      "logps/chosen": -1.0239425897598267,
+      "logps/rejected": -1.3165332078933716,
+      "loss": 1.5875,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.559856414794922,
+      "rewards/margins": 0.7314767241477966,
+      "rewards/rejected": -3.291332960128784,
+      "step": 460
+    },
+    {
+      "epoch": 0.9938551963665508,
+      "grad_norm": 11.3294414057108,
+      "learning_rate": 2.797454743164174e-11,
+      "logits/chosen": 2.186420202255249,
+      "logits/rejected": 2.1705265045166016,
+      "logps/chosen": -0.875818133354187,
+      "logps/rejected": -1.0201952457427979,
+      "loss": 1.5967,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -2.1895456314086914,
+      "rewards/margins": 0.36094242334365845,
+      "rewards/rejected": -2.550488233566284,
+      "step": 465
+    },
+    {
+      "epoch": 0.9981298423724285,
+      "step": 467,
+      "total_flos": 0.0,
+      "train_loss": 1.6566840494470372,
+      "train_runtime": 5417.7619,
+      "train_samples_per_second": 11.052,
+      "train_steps_per_second": 0.086
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 467,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": false,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f2b77a893b9f093dd80df1a7919ac514a78023dfb6581fd6cfc149f1fd2b6d92
+size 6584