End of training

README.md

@@ -16,7 +16,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.1314
+- Loss: 1.1311
 
 ## Model description
 
@@ -50,11 +50,11 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss |
 |:-------------:|:-----:|:----:|:---------------:|
-| 1.1077        | 0.34  | 50   | 1.1506          |
-| 1.0642        | 0.68  | 100  | 1.1310          |
-| 1.0399        | 1.02  | 150  | 1.1266          |
-| 1.0165        | 1.36  | 200  | 1.1302          |
-| 1.0272        | 1.7   | 250  | 1.1314          |
+| 1.1084        | 0.34  | 50   | 1.1509          |
+| 1.0641        | 0.68  | 100  | 1.1312          |
+| 1.04          | 1.02  | 150  | 1.1267          |
+| 1.016         | 1.36  | 200  | 1.1304          |
+| 1.0272        | 1.7   | 250  | 1.1311          |
 
 
 ### Framework versions

adapter_config.json

@@ -15,17 +15,17 @@
   "megatron_core": "megatron.core",
   "modules_to_save": null,
   "peft_type": "LORA",
-  "r": 16,
+  "r": 64,
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "down_proj",
-    "gate_proj",
-    "v_proj",
     "o_proj",
+    "k_proj",
     "q_proj",
     "up_proj",
-    "k_proj"
+    "down_proj",
+    "gate_proj",
+    "v_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:110c0af297f717ff2cf7806e9ffaee869b95756862078bd045b2a2b00c46f728
-size 83946192
+oid sha256:670b0288c7dfa7cfe688367ff6afc89d6b2d0fc950c82206416a1016080b70a4
+size 335605144

all_results.json

@@ -1,11 +1,11 @@
 {
     "epoch": 2.0,
-    "eval_loss": 1.2288233041763306,
-    "eval_runtime": 22.0177,
-    "eval_samples_per_second": 11.899,
-    "eval_steps_per_second": 1.499,
-    "train_loss": 1.0460854338950851,
-    "train_runtime": 864.371,
-    "train_samples_per_second": 5.451,
-    "train_steps_per_second": 0.34
+    "eval_loss": 1.2889022827148438,
+    "eval_runtime": 26.7524,
+    "eval_samples_per_second": 32.67,
+    "eval_steps_per_second": 2.056,
+    "train_loss": 1.0463855915329083,
+    "train_runtime": 868.1968,
+    "train_samples_per_second": 5.427,
+    "train_steps_per_second": 0.339
 }

eval_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 0.01,
-    "eval_loss": 1.2288233041763306,
-    "eval_runtime": 22.0177,
-    "eval_samples_per_second": 11.899,
-    "eval_steps_per_second": 1.499
+    "epoch": 2.0,
+    "eval_loss": 1.2889022827148438,
+    "eval_runtime": 26.7524,
+    "eval_samples_per_second": 32.67,
+    "eval_steps_per_second": 2.056
 }

metrics.json

@@ -1 +1 @@
-{"run_name": "./output", "train_runtime": 13.1306, "train_samples_per_second": 1.219, "train_steps_per_second": 0.076, "train_loss": 1.3776695728302002, "epoch": 0.01, "eval_loss": 1.2288233041763306, "eval_runtime": 22.0177, "eval_samples_per_second": 11.899, "eval_steps_per_second": 1.499}
+{"run_name": "./output", "train_runtime": 2763.6536, "train_samples_per_second": 12.007, "train_steps_per_second": 0.187, "total_flos": 5.99394721244119e+17, "train_loss": 1.2557983950758533, "epoch": 2.0, "eval_loss": 1.2889022827148438, "eval_runtime": 48.1894, "eval_samples_per_second": 18.137, "eval_steps_per_second": 1.141}

train_results.json

@@ -1,7 +1,7 @@
 {
     "epoch": 2.0,
-    "train_loss": 1.0460854338950851,
-    "train_runtime": 864.371,
-    "train_samples_per_second": 5.451,
-    "train_steps_per_second": 0.34
+    "train_loss": 1.0463855915329083,
+    "train_runtime": 868.1968,
+    "train_samples_per_second": 5.427,
+    "train_steps_per_second": 0.339
 }

trainer_state.json

@@ -11,159 +11,159 @@
     {
       "epoch": 0.07,
       "learning_rate": 3e-05,
-      "loss": 1.2065,
+      "loss": 1.2091,
       "step": 10
     },
     {
       "epoch": 0.14,
       "learning_rate": 3e-05,
-      "loss": 1.1456,
+      "loss": 1.1462,
       "step": 20
     },
     {
       "epoch": 0.2,
       "learning_rate": 3e-05,
-      "loss": 1.1411,
+      "loss": 1.1415,
       "step": 30
     },
     {
       "epoch": 0.27,
       "learning_rate": 3e-05,
-      "loss": 1.1179,
+      "loss": 1.1181,
       "step": 40
     },
     {
       "epoch": 0.34,
       "learning_rate": 3e-05,
-      "loss": 1.1077,
+      "loss": 1.1084,
       "step": 50
     },
     {
       "epoch": 0.34,
-      "eval_loss": 1.1505802869796753,
-      "eval_runtime": 15.5075,
-      "eval_samples_per_second": 16.895,
-      "eval_steps_per_second": 4.256,
+      "eval_loss": 1.1509493589401245,
+      "eval_runtime": 15.7829,
+      "eval_samples_per_second": 16.6,
+      "eval_steps_per_second": 4.182,
       "step": 50
     },
     {
       "epoch": 0.41,
       "learning_rate": 3e-05,
-      "loss": 1.0964,
+      "loss": 1.0965,
       "step": 60
     },
     {
       "epoch": 0.48,
       "learning_rate": 3e-05,
-      "loss": 1.0923,
+      "loss": 1.0929,
       "step": 70
     },
     {
       "epoch": 0.54,
       "learning_rate": 3e-05,
-      "loss": 1.1404,
+      "loss": 1.1405,
       "step": 80
     },
     {
       "epoch": 0.61,
       "learning_rate": 3e-05,
-      "loss": 1.1145,
+      "loss": 1.1153,
       "step": 90
     },
     {
       "epoch": 0.68,
       "learning_rate": 3e-05,
-      "loss": 1.0642,
+      "loss": 1.0641,
       "step": 100
     },
     {
       "epoch": 0.68,
-      "eval_loss": 1.1310365200042725,
-      "eval_runtime": 15.5192,
-      "eval_samples_per_second": 16.882,
-      "eval_steps_per_second": 4.253,
+      "eval_loss": 1.131222128868103,
+      "eval_runtime": 15.7708,
+      "eval_samples_per_second": 16.613,
+      "eval_steps_per_second": 4.185,
       "step": 100
     },
     {
       "epoch": 0.75,
       "learning_rate": 3e-05,
-      "loss": 1.0558,
+      "loss": 1.0562,
       "step": 110
     },
     {
       "epoch": 0.81,
       "learning_rate": 3e-05,
-      "loss": 1.1053,
+      "loss": 1.1051,
       "step": 120
     },
     {
       "epoch": 0.88,
       "learning_rate": 3e-05,
-      "loss": 1.0948,
+      "loss": 1.0951,
       "step": 130
     },
     {
       "epoch": 0.95,
       "learning_rate": 3e-05,
-      "loss": 1.0725,
+      "loss": 1.0712,
       "step": 140
     },
     {
       "epoch": 1.02,
       "learning_rate": 3e-05,
-      "loss": 1.0399,
+      "loss": 1.04,
       "step": 150
     },
     {
       "epoch": 1.02,
-      "eval_loss": 1.1266452074050903,
-      "eval_runtime": 15.5722,
-      "eval_samples_per_second": 16.825,
-      "eval_steps_per_second": 4.238,
+      "eval_loss": 1.1267426013946533,
+      "eval_runtime": 15.8691,
+      "eval_samples_per_second": 16.51,
+      "eval_steps_per_second": 4.159,
       "step": 150
     },
     {
       "epoch": 1.09,
       "learning_rate": 3e-05,
-      "loss": 0.9873,
+      "loss": 0.9894,
       "step": 160
     },
     {
       "epoch": 1.15,
       "learning_rate": 3e-05,
-      "loss": 1.0614,
+      "loss": 1.0622,
       "step": 170
     },
     {
       "epoch": 1.22,
       "learning_rate": 3e-05,
-      "loss": 0.979,
+      "loss": 0.9794,
       "step": 180
     },
     {
       "epoch": 1.29,
       "learning_rate": 3e-05,
-      "loss": 0.9585,
+      "loss": 0.9593,
       "step": 190
     },
     {
       "epoch": 1.36,
       "learning_rate": 3e-05,
-      "loss": 1.0165,
+      "loss": 1.016,
       "step": 200
     },
     {
       "epoch": 1.36,
-      "eval_loss": 1.1301801204681396,
-      "eval_runtime": 15.5114,
-      "eval_samples_per_second": 16.891,
-      "eval_steps_per_second": 4.255,
+      "eval_loss": 1.1303731203079224,
+      "eval_runtime": 15.6796,
+      "eval_samples_per_second": 16.71,
+      "eval_steps_per_second": 4.209,
       "step": 200
     },
     {
       "epoch": 1.43,
       "learning_rate": 3e-05,
-      "loss": 0.987,
+      "loss": 0.9869,
       "step": 210
     },
     {
@@ -175,13 +175,13 @@
     {
       "epoch": 1.56,
       "learning_rate": 3e-05,
-      "loss": 0.9922,
+      "loss": 0.9926,
       "step": 230
     },
     {
       "epoch": 1.63,
       "learning_rate": 3e-05,
-      "loss": 0.9889,
+      "loss": 0.9892,
       "step": 240
     },
     {
@@ -192,44 +192,44 @@
     },
     {
       "epoch": 1.7,
-      "eval_loss": 1.1314338445663452,
-      "eval_runtime": 15.5119,
-      "eval_samples_per_second": 16.89,
-      "eval_steps_per_second": 4.255,
+      "eval_loss": 1.1311020851135254,
+      "eval_runtime": 15.7521,
+      "eval_samples_per_second": 16.633,
+      "eval_steps_per_second": 4.19,
       "step": 250
     },
     {
       "epoch": 1.77,
       "learning_rate": 3e-05,
-      "loss": 0.977,
+      "loss": 0.9765,
       "step": 260
     },
     {
       "epoch": 1.83,
       "learning_rate": 3e-05,
-      "loss": 1.0425,
+      "loss": 1.0433,
       "step": 270
     },
     {
       "epoch": 1.9,
       "learning_rate": 3e-05,
-      "loss": 0.9611,
+      "loss": 0.9614,
       "step": 280
     },
     {
       "epoch": 1.97,
       "learning_rate": 3e-05,
-      "loss": 0.9305,
+      "loss": 0.9299,
       "step": 290
     },
     {
       "epoch": 2.0,
       "step": 294,
-      "total_flos": 6.900509298543821e+16,
-      "train_loss": 1.0460854338950851,
-      "train_runtime": 864.371,
-      "train_samples_per_second": 5.451,
-      "train_steps_per_second": 0.34
+      "total_flos": 7.021903495711949e+16,
+      "train_loss": 1.0463855915329083,
+      "train_runtime": 868.1968,
+      "train_samples_per_second": 5.427,
+      "train_steps_per_second": 0.339
     }
   ],
   "logging_steps": 10,
@@ -237,7 +237,7 @@
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 250,
-  "total_flos": 6.900509298543821e+16,
+  "total_flos": 7.021903495711949e+16,
   "train_batch_size": 4,
   "trial_name": null,
   "trial_params": null

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:670309ed11279de039411d6ea6d7cb68f8f00d95a334c639a77e271f6292bb0f
-size 6648
+oid sha256:575d23392a793f463c05c62aed6a20c9f3f1d14af4f07046c639ce932527d18b
+size 6712