Model save

README.md

@@ -0,0 +1,58 @@
+---
+base_model: orpo-explorers/kaist-mistral-orpo-OHP-15k-Stratified-1-beta-0.2-1epoch
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: kaist-mistral-orpo-OHP-15k-Stratified-1-beta-0.2-1epoch-capybara-2epoch
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# kaist-mistral-orpo-OHP-15k-Stratified-1-beta-0.2-1epoch-capybara-2epoch
+
+This model is a fine-tuned version of [orpo-explorers/kaist-mistral-orpo-OHP-15k-Stratified-1-beta-0.2-1epoch](https://huggingface.co/orpo-explorers/kaist-mistral-orpo-OHP-15k-Stratified-1-beta-0.2-1epoch) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 4
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- num_epochs: 2
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.39.3
+- Pytorch 2.1.2.post303
+- Datasets 2.18.0
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.5399103337283587,
+    "train_runtime": 8872.0808,
+    "train_samples": 14154,
+    "train_samples_per_second": 3.191,
+    "train_steps_per_second": 0.05
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:16aed7405b7c177b69ea267ff5b6a17dc0226c9c9d5bd01e22ada31e71121514
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5fcb9c80c995bd7d3ec35b249b8d04d07832bfcb0976a1ff1c741340d3ad43c
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:be5b88a753020982007fe904cf30bc011de9fffec635c15bb8d9d57b6aa613d6
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 2.0,
+    "train_loss": 0.5399103337283587,
+    "train_runtime": 8872.0808,
+    "train_samples": 14154,
+    "train_samples_per_second": 3.191,
+    "train_steps_per_second": 0.05
+}

trainer_state.json

@@ -0,0 +1,822 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.9954853273137698,
+  "eval_steps": 500,
+  "global_step": 442,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05,
+      "grad_norm": 3.34375,
+      "learning_rate": 4.9936877763590664e-06,
+      "log_odds_chosen": 0.2890412509441376,
+      "log_odds_ratio": -0.6332851648330688,
+      "logits/chosen": -3.1468911170959473,
+      "logits/rejected": -3.138887643814087,
+      "logps/chosen": -0.8719667196273804,
+      "logps/rejected": -1.0382100343704224,
+      "loss": 0.6482,
+      "nll_loss": 0.5332359075546265,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17439337074756622,
+      "rewards/margins": 0.033248625695705414,
+      "rewards/rejected": -0.20764198899269104,
+      "step": 10
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 3.046875,
+      "learning_rate": 4.9747829807701e-06,
+      "log_odds_chosen": 0.3164082169532776,
+      "log_odds_ratio": -0.6052478551864624,
+      "logits/chosen": -3.0846054553985596,
+      "logits/rejected": -3.0846097469329834,
+      "logps/chosen": -0.8607593774795532,
+      "logps/rejected": -1.0491193532943726,
+      "loss": 0.6131,
+      "nll_loss": 0.4854271411895752,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.17215189337730408,
+      "rewards/margins": 0.03767203167080879,
+      "rewards/rejected": -0.20982392132282257,
+      "step": 20
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 3.671875,
+      "learning_rate": 4.943381078271214e-06,
+      "log_odds_chosen": 0.3783247768878937,
+      "log_odds_ratio": -0.6129769086837769,
+      "logits/chosen": -3.084336996078491,
+      "logits/rejected": -3.0786914825439453,
+      "logps/chosen": -0.8424398303031921,
+      "logps/rejected": -1.0533756017684937,
+      "loss": 0.602,
+      "nll_loss": 0.4758261740207672,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16848795115947723,
+      "rewards/margins": 0.0421871617436409,
+      "rewards/rejected": -0.21067512035369873,
+      "step": 30
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 3.0,
+      "learning_rate": 4.89964064152747e-06,
+      "log_odds_chosen": 0.3220486640930176,
+      "log_odds_ratio": -0.6127817630767822,
+      "logits/chosen": -3.073997974395752,
+      "logits/rejected": -3.069819688796997,
+      "logps/chosen": -0.8541064262390137,
+      "logps/rejected": -1.0315220355987549,
+      "loss": 0.5993,
+      "nll_loss": 0.47563648223876953,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.17082129418849945,
+      "rewards/margins": 0.03548307344317436,
+      "rewards/rejected": -0.2063043862581253,
+      "step": 40
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 2.90625,
+      "learning_rate": 4.84378255007397e-06,
+      "log_odds_chosen": 0.3029581606388092,
+      "log_odds_ratio": -0.6211820840835571,
+      "logits/chosen": -3.0848793983459473,
+      "logits/rejected": -3.0841445922851562,
+      "logps/chosen": -0.8334375619888306,
+      "logps/rejected": -1.0042452812194824,
+      "loss": 0.5913,
+      "nll_loss": 0.4705049991607666,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1666875034570694,
+      "rewards/margins": 0.03416154906153679,
+      "rewards/rejected": -0.20084905624389648,
+      "step": 50
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 3.015625,
+      "learning_rate": 4.7760888749230414e-06,
+      "log_odds_chosen": 0.4831446707248688,
+      "log_odds_ratio": -0.5493755340576172,
+      "logits/chosen": -3.095670461654663,
+      "logits/rejected": -3.0957283973693848,
+      "logps/chosen": -0.8023236989974976,
+      "logps/rejected": -1.0722805261611938,
+      "loss": 0.5964,
+      "nll_loss": 0.4757974147796631,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -0.16046474874019623,
+      "rewards/margins": 0.05399137735366821,
+      "rewards/rejected": -0.21445612609386444,
+      "step": 60
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 3.09375,
+      "learning_rate": 4.696901454167989e-06,
+      "log_odds_chosen": 0.3349444270133972,
+      "log_odds_ratio": -0.6006864309310913,
+      "logits/chosen": -3.0792784690856934,
+      "logits/rejected": -3.073970079421997,
+      "logps/chosen": -0.8161932229995728,
+      "logps/rejected": -0.9950379133224487,
+      "loss": 0.6022,
+      "nll_loss": 0.48054131865501404,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -0.16323864459991455,
+      "rewards/margins": 0.035768941044807434,
+      "rewards/rejected": -0.19900760054588318,
+      "step": 70
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 3.171875,
+      "learning_rate": 4.6066201667762944e-06,
+      "log_odds_chosen": 0.46497654914855957,
+      "log_odds_ratio": -0.5399398803710938,
+      "logits/chosen": -3.1036829948425293,
+      "logits/rejected": -3.099396228790283,
+      "logps/chosen": -0.8102518916130066,
+      "logps/rejected": -1.0698442459106445,
+      "loss": 0.5611,
+      "nll_loss": 0.47170430421829224,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -0.16205039620399475,
+      "rewards/margins": 0.05191843584179878,
+      "rewards/rejected": -0.21396884322166443,
+      "step": 80
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 3.234375,
+      "learning_rate": 4.505700913289246e-06,
+      "log_odds_chosen": 0.385485976934433,
+      "log_odds_ratio": -0.5975229144096375,
+      "logits/chosen": -3.096364974975586,
+      "logits/rejected": -3.085583209991455,
+      "logps/chosen": -0.8479864001274109,
+      "logps/rejected": -1.076240062713623,
+      "loss": 0.6009,
+      "nll_loss": 0.47817081212997437,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1695972979068756,
+      "rewards/margins": 0.04565075412392616,
+      "rewards/rejected": -0.21524803340435028,
+      "step": 90
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 2.890625,
+      "learning_rate": 4.3946533136249926e-06,
+      "log_odds_chosen": 0.43778282403945923,
+      "log_odds_ratio": -0.5804494619369507,
+      "logits/chosen": -3.0998694896698,
+      "logits/rejected": -3.0987915992736816,
+      "logps/chosen": -0.8241409063339233,
+      "logps/rejected": -1.0574108362197876,
+      "loss": 0.5988,
+      "nll_loss": 0.4965757429599762,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16482816636562347,
+      "rewards/margins": 0.046653974801301956,
+      "rewards/rejected": -0.21148213744163513,
+      "step": 100
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 3.203125,
+      "learning_rate": 4.274038133610629e-06,
+      "log_odds_chosen": 0.46187520027160645,
+      "log_odds_ratio": -0.5494967103004456,
+      "logits/chosen": -3.1161484718322754,
+      "logits/rejected": -3.114893913269043,
+      "logps/chosen": -0.7468121647834778,
+      "logps/rejected": -0.9928558468818665,
+      "loss": 0.5811,
+      "nll_loss": 0.4424869120121002,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -0.14936241507530212,
+      "rewards/margins": 0.04920873045921326,
+      "rewards/rejected": -0.19857117533683777,
+      "step": 110
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 3.375,
+      "learning_rate": 4.1444644532387485e-06,
+      "log_odds_chosen": 0.4036913812160492,
+      "log_odds_ratio": -0.6148982048034668,
+      "logits/chosen": -3.0943081378936768,
+      "logits/rejected": -3.089369297027588,
+      "logps/chosen": -0.8185051083564758,
+      "logps/rejected": -1.0283105373382568,
+      "loss": 0.5743,
+      "nll_loss": 0.4471573233604431,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16370102763175964,
+      "rewards/margins": 0.04196108505129814,
+      "rewards/rejected": -0.2056621015071869,
+      "step": 120
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 3.109375,
+      "learning_rate": 4.006586590948141e-06,
+      "log_odds_chosen": 0.38943806290626526,
+      "log_odds_ratio": -0.5949686169624329,
+      "logits/chosen": -3.105708360671997,
+      "logits/rejected": -3.1006293296813965,
+      "logps/chosen": -0.8150092363357544,
+      "logps/rejected": -1.0240681171417236,
+      "loss": 0.5787,
+      "nll_loss": 0.46157145500183105,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16300183534622192,
+      "rewards/margins": 0.041811779141426086,
+      "rewards/rejected": -0.20481359958648682,
+      "step": 130
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 3.109375,
+      "learning_rate": 3.861100799460336e-06,
+      "log_odds_chosen": 0.5897554159164429,
+      "log_odds_ratio": -0.542380690574646,
+      "logits/chosen": -3.100862503051758,
+      "logits/rejected": -3.0946009159088135,
+      "logps/chosen": -0.7791546583175659,
+      "logps/rejected": -1.0703848600387573,
+      "loss": 0.5747,
+      "nll_loss": 0.5026167631149292,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.15583094954490662,
+      "rewards/margins": 0.05824603885412216,
+      "rewards/rejected": -0.214076966047287,
+      "step": 140
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 3.03125,
+      "learning_rate": 3.7087417498572946e-06,
+      "log_odds_chosen": 0.4274469316005707,
+      "log_odds_ratio": -0.5712376236915588,
+      "logits/chosen": -3.102602005004883,
+      "logits/rejected": -3.1062116622924805,
+      "logps/chosen": -0.8305424451828003,
+      "logps/rejected": -1.077083945274353,
+      "loss": 0.5692,
+      "nll_loss": 0.4742380678653717,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16610850393772125,
+      "rewards/margins": 0.04930829256772995,
+      "rewards/rejected": -0.2154167890548706,
+      "step": 150
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 3.28125,
+      "learning_rate": 3.550278821654866e-06,
+      "log_odds_chosen": 0.34345707297325134,
+      "log_odds_ratio": -0.6129624247550964,
+      "logits/chosen": -3.095494508743286,
+      "logits/rejected": -3.0863471031188965,
+      "logps/chosen": -0.8034600019454956,
+      "logps/rejected": -0.9921743273735046,
+      "loss": 0.587,
+      "nll_loss": 0.4692765176296234,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1606920212507248,
+      "rewards/margins": 0.037742845714092255,
+      "rewards/rejected": -0.19843485951423645,
+      "step": 160
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 4.28125,
+      "learning_rate": 3.386512217606339e-06,
+      "log_odds_chosen": 0.3175373375415802,
+      "log_odds_ratio": -0.6278958320617676,
+      "logits/chosen": -3.1244797706604004,
+      "logits/rejected": -3.1204514503479004,
+      "logps/chosen": -0.7932506799697876,
+      "logps/rejected": -0.9527732133865356,
+      "loss": 0.6102,
+      "nll_loss": 0.472064733505249,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15865013003349304,
+      "rewards/margins": 0.031904518604278564,
+      "rewards/rejected": -0.1905546486377716,
+      "step": 170
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 3.765625,
+      "learning_rate": 3.218268922855452e-06,
+      "log_odds_chosen": 0.42049679160118103,
+      "log_odds_ratio": -0.5808514356613159,
+      "logits/chosen": -3.1028733253479004,
+      "logits/rejected": -3.0980591773986816,
+      "logps/chosen": -0.7747179269790649,
+      "logps/rejected": -0.9938864707946777,
+      "loss": 0.5825,
+      "nll_loss": 0.44331270456314087,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -0.154943585395813,
+      "rewards/margins": 0.04383372142910957,
+      "rewards/rejected": -0.19877730309963226,
+      "step": 180
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 3.328125,
+      "learning_rate": 3.046398528844248e-06,
+      "log_odds_chosen": 0.5076274275779724,
+      "log_odds_ratio": -0.5742210745811462,
+      "logits/chosen": -3.104665517807007,
+      "logits/rejected": -3.095426082611084,
+      "logps/chosen": -0.776441216468811,
+      "logps/rejected": -1.0134727954864502,
+      "loss": 0.5712,
+      "nll_loss": 0.4288255274295807,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.15528824925422668,
+      "rewards/margins": 0.04740630090236664,
+      "rewards/rejected": -0.20269456505775452,
+      "step": 190
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 2.953125,
+      "learning_rate": 2.871768943064129e-06,
+      "log_odds_chosen": 0.3015575408935547,
+      "log_odds_ratio": -0.6296088695526123,
+      "logits/chosen": -3.0799665451049805,
+      "logits/rejected": -3.07787823677063,
+      "logps/chosen": -0.8669988512992859,
+      "logps/rejected": -1.0326287746429443,
+      "loss": 0.5744,
+      "nll_loss": 0.47870683670043945,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -0.17339977622032166,
+      "rewards/margins": 0.033125996589660645,
+      "rewards/rejected": -0.2065257579088211,
+      "step": 200
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 2.859375,
+      "learning_rate": 2.695262006314912e-06,
+      "log_odds_chosen": 0.3043849766254425,
+      "log_odds_ratio": -0.6296250224113464,
+      "logits/chosen": -3.0792198181152344,
+      "logits/rejected": -3.0685715675354004,
+      "logps/chosen": -0.8420785069465637,
+      "logps/rejected": -1.0109227895736694,
+      "loss": 0.5851,
+      "nll_loss": 0.49167829751968384,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.16841569542884827,
+      "rewards/margins": 0.03376886993646622,
+      "rewards/rejected": -0.20218458771705627,
+      "step": 210
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 3.390625,
+      "learning_rate": 2.517769039603744e-06,
+      "log_odds_chosen": 0.47437596321105957,
+      "log_odds_ratio": -0.5693033933639526,
+      "logits/chosen": -3.0923125743865967,
+      "logits/rejected": -3.08634614944458,
+      "logps/chosen": -0.7851861119270325,
+      "logps/rejected": -1.0160605907440186,
+      "loss": 0.6031,
+      "nll_loss": 0.4828670620918274,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.15703722834587097,
+      "rewards/margins": 0.04617489501833916,
+      "rewards/rejected": -0.20321211218833923,
+      "step": 220
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 5.125,
+      "learning_rate": 2.3401863431710864e-06,
+      "log_odds_chosen": 0.8126243352890015,
+      "log_odds_ratio": -0.4562348425388336,
+      "logits/chosen": -3.0906317234039307,
+      "logits/rejected": -3.08894681930542,
+      "logps/chosen": -0.6749258041381836,
+      "logps/rejected": -1.0676677227020264,
+      "loss": 0.5063,
+      "nll_loss": 0.4003027081489563,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.13498517870903015,
+      "rewards/margins": 0.0785483792424202,
+      "rewards/rejected": -0.21353355050086975,
+      "step": 230
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 3.34375,
+      "learning_rate": 2.163410670372652e-06,
+      "log_odds_chosen": 0.8564330339431763,
+      "log_odds_ratio": -0.4268684387207031,
+      "logits/chosen": -3.0698952674865723,
+      "logits/rejected": -3.070732831954956,
+      "logps/chosen": -0.7170256972312927,
+      "logps/rejected": -1.1589481830596924,
+      "loss": 0.4975,
+      "nll_loss": 0.45029813051223755,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.14340513944625854,
+      "rewards/margins": 0.08838452398777008,
+      "rewards/rejected": -0.23178966343402863,
+      "step": 240
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.9883346992732256e-06,
+      "log_odds_chosen": 1.0222368240356445,
+      "log_odds_ratio": -0.36799290776252747,
+      "logits/chosen": -3.055107593536377,
+      "logits/rejected": -3.045011281967163,
+      "logps/chosen": -0.6467097997665405,
+      "logps/rejected": -1.2033334970474243,
+      "loss": 0.4936,
+      "nll_loss": 0.3834058940410614,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": -0.1293419599533081,
+      "rewards/margins": 0.1113247275352478,
+      "rewards/rejected": -0.24066667258739471,
+      "step": 250
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 2.921875,
+      "learning_rate": 1.8158425248197931e-06,
+      "log_odds_chosen": 0.9676607251167297,
+      "log_odds_ratio": -0.3842445909976959,
+      "logits/chosen": -3.078850269317627,
+      "logits/rejected": -3.0772275924682617,
+      "logps/chosen": -0.6771007180213928,
+      "logps/rejected": -1.177585482597351,
+      "loss": 0.5028,
+      "nll_loss": 0.44024211168289185,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.13542014360427856,
+      "rewards/margins": 0.10009696334600449,
+      "rewards/rejected": -0.23551711440086365,
+      "step": 260
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 3.1875,
+      "learning_rate": 1.6468051943575242e-06,
+      "log_odds_chosen": 0.9456725120544434,
+      "log_odds_ratio": -0.39273351430892944,
+      "logits/chosen": -3.085444450378418,
+      "logits/rejected": -3.08333420753479,
+      "logps/chosen": -0.6596750020980835,
+      "logps/rejected": -1.1495213508605957,
+      "loss": 0.4758,
+      "nll_loss": 0.3547888398170471,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.1319349855184555,
+      "rewards/margins": 0.09796925634145737,
+      "rewards/rejected": -0.22990426421165466,
+      "step": 270
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 3.328125,
+      "learning_rate": 1.482076309033254e-06,
+      "log_odds_chosen": 0.8887540102005005,
+      "log_odds_ratio": -0.43248239159584045,
+      "logits/chosen": -3.0775530338287354,
+      "logits/rejected": -3.0756869316101074,
+      "logps/chosen": -0.6567140817642212,
+      "logps/rejected": -1.1050218343734741,
+      "loss": 0.4872,
+      "nll_loss": 0.3975124657154083,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1313427984714508,
+      "rewards/margins": 0.08966155350208282,
+      "rewards/rejected": -0.22100436687469482,
+      "step": 280
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 2.96875,
+      "learning_rate": 1.3224877132984131e-06,
+      "log_odds_chosen": 1.054574728012085,
+      "log_odds_ratio": -0.3588215708732605,
+      "logits/chosen": -3.103698253631592,
+      "logits/rejected": -3.107422351837158,
+      "logps/chosen": -0.6697841882705688,
+      "logps/rejected": -1.2304431200027466,
+      "loss": 0.4864,
+      "nll_loss": 0.4196816384792328,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": -0.13395683467388153,
+      "rewards/margins": 0.11213177442550659,
+      "rewards/rejected": -0.24608862400054932,
+      "step": 290
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 3.109375,
+      "learning_rate": 1.1688452942784592e-06,
+      "log_odds_chosen": 1.018751621246338,
+      "log_odds_ratio": -0.3791121542453766,
+      "logits/chosen": -3.0898470878601074,
+      "logits/rejected": -3.0825257301330566,
+      "logps/chosen": -0.6466778516769409,
+      "logps/rejected": -1.1818552017211914,
+      "loss": 0.4914,
+      "nll_loss": 0.3884134292602539,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.12933556735515594,
+      "rewards/margins": 0.10703550279140472,
+      "rewards/rejected": -0.23637107014656067,
+      "step": 300
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.021924912221062e-06,
+      "log_odds_chosen": 0.8282191157341003,
+      "log_odds_ratio": -0.42912721633911133,
+      "logits/chosen": -3.1064085960388184,
+      "logits/rejected": -3.105886697769165,
+      "logps/chosen": -0.669070839881897,
+      "logps/rejected": -1.1031153202056885,
+      "loss": 0.4969,
+      "nll_loss": 0.4509221017360687,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -0.13381414115428925,
+      "rewards/margins": 0.08680891990661621,
+      "rewards/rejected": -0.22062306106090546,
+      "step": 310
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 3.515625,
+      "learning_rate": 8.824684825733865e-07,
+      "log_odds_chosen": 1.0580097436904907,
+      "log_odds_ratio": -0.38433724641799927,
+      "logits/chosen": -3.081042528152466,
+      "logits/rejected": -3.075192451477051,
+      "logps/chosen": -0.6307107210159302,
+      "logps/rejected": -1.1325790882110596,
+      "loss": 0.493,
+      "nll_loss": 0.4108586311340332,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.12614212930202484,
+      "rewards/margins": 0.10037368535995483,
+      "rewards/rejected": -0.22651581466197968,
+      "step": 320
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 3.046875,
+      "learning_rate": 7.51180229473116e-07,
+      "log_odds_chosen": 0.9154760241508484,
+      "log_odds_ratio": -0.415397584438324,
+      "logits/chosen": -3.0937957763671875,
+      "logits/rejected": -3.0928263664245605,
+      "logps/chosen": -0.6775897741317749,
+      "logps/rejected": -1.1316546201705933,
+      "loss": 0.5044,
+      "nll_loss": 0.4353519380092621,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.13551795482635498,
+      "rewards/margins": 0.09081296622753143,
+      "rewards/rejected": -0.22633090615272522,
+      "step": 330
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 3.15625,
+      "learning_rate": 6.28723129572247e-07,
+      "log_odds_chosen": 1.139094352722168,
+      "log_odds_ratio": -0.351654052734375,
+      "logits/chosen": -3.077972888946533,
+      "logits/rejected": -3.0774223804473877,
+      "logps/chosen": -0.6138228178024292,
+      "logps/rejected": -1.158979058265686,
+      "loss": 0.4728,
+      "nll_loss": 0.3857228457927704,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.12276456505060196,
+      "rewards/margins": 0.10903123766183853,
+      "rewards/rejected": -0.2317957878112793,
+      "step": 340
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 3.234375,
+      "learning_rate": 5.157155641515766e-07,
+      "log_odds_chosen": 1.0156279802322388,
+      "log_odds_ratio": -0.3680209219455719,
+      "logits/chosen": -3.093818426132202,
+      "logits/rejected": -3.095491409301758,
+      "logps/chosen": -0.6614006161689758,
+      "logps/rejected": -1.1964319944381714,
+      "loss": 0.5024,
+      "nll_loss": 0.4554772973060608,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.13228009641170502,
+      "rewards/margins": 0.10700628906488419,
+      "rewards/rejected": -0.2392864227294922,
+      "step": 350
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 3.109375,
+      "learning_rate": 4.127281964319446e-07,
+      "log_odds_chosen": 1.1072419881820679,
+      "log_odds_ratio": -0.365578830242157,
+      "logits/chosen": -3.0632922649383545,
+      "logits/rejected": -3.053469657897949,
+      "logps/chosen": -0.6406615376472473,
+      "logps/rejected": -1.1723295450210571,
+      "loss": 0.4717,
+      "nll_loss": 0.39158958196640015,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.12813231348991394,
+      "rewards/margins": 0.10633359849452972,
+      "rewards/rejected": -0.23446591198444366,
+      "step": 360
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 3.171875,
+      "learning_rate": 3.202810898511424e-07,
+      "log_odds_chosen": 1.0118036270141602,
+      "log_odds_ratio": -0.3721558451652527,
+      "logits/chosen": -3.0760838985443115,
+      "logits/rejected": -3.0786654949188232,
+      "logps/chosen": -0.6295793652534485,
+      "logps/rejected": -1.1554322242736816,
+      "loss": 0.4954,
+      "nll_loss": 0.3821503221988678,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.12591585516929626,
+      "rewards/margins": 0.1051705852150917,
+      "rewards/rejected": -0.23108646273612976,
+      "step": 370
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 3.015625,
+      "learning_rate": 2.388410818585263e-07,
+      "log_odds_chosen": 1.0309429168701172,
+      "log_odds_ratio": -0.3757808804512024,
+      "logits/chosen": -3.0944666862487793,
+      "logits/rejected": -3.102954149246216,
+      "logps/chosen": -0.6560705304145813,
+      "logps/rejected": -1.1918388605117798,
+      "loss": 0.4848,
+      "nll_loss": 0.4180088937282562,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": -0.13121411204338074,
+      "rewards/margins": 0.10715366899967194,
+      "rewards/rejected": -0.23836776614189148,
+      "step": 380
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 3.0625,
+      "learning_rate": 1.6881942648911077e-07,
+      "log_odds_chosen": 1.045115351676941,
+      "log_odds_ratio": -0.3773229420185089,
+      "logits/chosen": -3.099276304244995,
+      "logits/rejected": -3.0971086025238037,
+      "logps/chosen": -0.6689329743385315,
+      "logps/rejected": -1.209533452987671,
+      "loss": 0.4865,
+      "nll_loss": 0.4016016125679016,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -0.13378658890724182,
+      "rewards/margins": 0.10812009871006012,
+      "rewards/rejected": -0.24190667271614075,
+      "step": 390
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 3.109375,
+      "learning_rate": 1.1056971762161584e-07,
+      "log_odds_chosen": 0.8860037922859192,
+      "log_odds_ratio": -0.4059659540653229,
+      "logits/chosen": -3.0970611572265625,
+      "logits/rejected": -3.0827975273132324,
+      "logps/chosen": -0.6797454357147217,
+      "logps/rejected": -1.1361665725708008,
+      "loss": 0.4883,
+      "nll_loss": 0.4087887406349182,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": -0.13594909012317657,
+      "rewards/margins": 0.09128421545028687,
+      "rewards/rejected": -0.22723329067230225,
+      "step": 400
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 3.015625,
+      "learning_rate": 6.438610340755464e-08,
+      "log_odds_chosen": 0.9491767883300781,
+      "log_odds_ratio": -0.3955945372581482,
+      "logits/chosen": -3.095716953277588,
+      "logits/rejected": -3.0901029109954834,
+      "logps/chosen": -0.6622263193130493,
+      "logps/rejected": -1.1627901792526245,
+      "loss": 0.4715,
+      "nll_loss": 0.4081706404685974,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.13244526088237762,
+      "rewards/margins": 0.10011278092861176,
+      "rewards/rejected": -0.23255804181098938,
+      "step": 410
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 3.328125,
+      "learning_rate": 3.050180088809973e-08,
+      "log_odds_chosen": 0.89338219165802,
+      "log_odds_ratio": -0.41199594736099243,
+      "logits/chosen": -3.078094482421875,
+      "logits/rejected": -3.074589252471924,
+      "logps/chosen": -0.6442387700080872,
+      "logps/rejected": -1.103659987449646,
+      "loss": 0.4976,
+      "nll_loss": 0.40062204003334045,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -0.12884774804115295,
+      "rewards/margins": 0.09188426285982132,
+      "rewards/rejected": -0.22073200345039368,
+      "step": 420
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 3.15625,
+      "learning_rate": 9.087918299586772e-09,
+      "log_odds_chosen": 0.9813147783279419,
+      "log_odds_ratio": -0.4028463363647461,
+      "logits/chosen": -3.079704523086548,
+      "logits/rejected": -3.0714404582977295,
+      "logps/chosen": -0.6575345993041992,
+      "logps/rejected": -1.1326944828033447,
+      "loss": 0.4832,
+      "nll_loss": 0.39662784337997437,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -0.13150693476200104,
+      "rewards/margins": 0.0950319841504097,
+      "rewards/rejected": -0.22653889656066895,
+      "step": 430
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 4.03125,
+      "learning_rate": 2.525910147516131e-10,
+      "log_odds_chosen": 0.966279149055481,
+      "log_odds_ratio": -0.4019397795200348,
+      "logits/chosen": -3.0766844749450684,
+      "logits/rejected": -3.0788207054138184,
+      "logps/chosen": -0.6589742302894592,
+      "logps/rejected": -1.1595604419708252,
+      "loss": 0.48,
+      "nll_loss": 0.38374659419059753,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.13179484009742737,
+      "rewards/margins": 0.10011725127696991,
+      "rewards/rejected": -0.23191209137439728,
+      "step": 440
+    },
+    {
+      "epoch": 2.0,
+      "step": 442,
+      "total_flos": 0.0,
+      "train_loss": 0.5399103337283587,
+      "train_runtime": 8872.0808,
+      "train_samples_per_second": 3.191,
+      "train_steps_per_second": 0.05
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 442,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 2,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}