End of training

README.md

@@ -15,8 +15,6 @@ should probably proofread and complete it, then remove this comment. -->
 # Mistral-7B-v0.1_case-briefs
 
 This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on the None dataset.
-It achieves the following results on the evaluation set:
-- Loss: 1.1290
 
 ## Model description
 
@@ -35,26 +33,19 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 3e-05
-- train_batch_size: 4
-- eval_batch_size: 4
+- learning_rate: 0.0002
+- train_batch_size: 1
+- eval_batch_size: 8
 - seed: 42
-- gradient_accumulation_steps: 4
+- gradient_accumulation_steps: 16
 - total_train_batch_size: 16
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: constant
 - lr_scheduler_warmup_ratio: 0.03
-- num_epochs: 2.0
+- training_steps: 1
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss |
-|:-------------:|:-----:|:----:|:---------------:|
-| 1.1008        | 0.34  | 50   | 1.1499          |
-| 1.0663        | 0.68  | 100  | 1.1314          |
-| 1.04          | 1.02  | 150  | 1.1263          |
-| 1.0182        | 1.36  | 200  | 1.1319          |
-| 1.0291        | 1.7   | 250  | 1.1290          |
 
 
 ### Framework versions
@@ -62,5 +53,5 @@ The following hyperparameters were used during training:
 - PEFT 0.7.1
 - Transformers 4.37.2
 - Pytorch 2.1.2+cu121
-- Datasets 2.16.1
+- Datasets 2.17.1
 - Tokenizers 0.15.1

adapter_config.json

@@ -10,7 +10,7 @@
   "layers_to_transform": null,
   "loftq_config": {},
   "lora_alpha": 16,
-  "lora_dropout": 0.1,
+  "lora_dropout": 0.0,
   "megatron_config": null,
   "megatron_core": "megatron.core",
   "modules_to_save": null,
@@ -19,13 +19,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "k_proj",
     "down_proj",
+    "o_proj",
+    "gate_proj",
     "q_proj",
-    "up_proj",
     "v_proj",
-    "gate_proj",
-    "o_proj"
+    "up_proj",
+    "k_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fbd8e69323bbf994e6f8255a1b0e552b1ffe36a2838fc0d7ddc47be401beb79e
+oid sha256:1c65d4d58e8101f2ce059a1c109f69e013107064461a40e5c947d8e82529537f
 size 335605144

all_results.json

@@ -1,11 +1,7 @@
 {
-    "epoch": 2.0,
-    "eval_loss": 1.182191252708435,
-    "eval_runtime": 106.7178,
-    "eval_samples_per_second": 2.455,
-    "eval_steps_per_second": 0.618,
-    "train_loss": 1.0449795049874961,
-    "train_runtime": 831.3165,
-    "train_samples_per_second": 5.668,
-    "train_steps_per_second": 0.354
+    "epoch": 0.01,
+    "train_loss": 1.3776695728302002,
+    "train_runtime": 13.1306,
+    "train_samples_per_second": 1.219,
+    "train_steps_per_second": 0.076
 }

train_results.json

@@ -1,7 +1,7 @@
 {
-    "epoch": 2.0,
-    "train_loss": 1.0449795049874961,
-    "train_runtime": 831.3165,
-    "train_samples_per_second": 5.668,
-    "train_steps_per_second": 0.354
+    "epoch": 0.01,
+    "train_loss": 1.3776695728302002,
+    "train_runtime": 13.1306,
+    "train_samples_per_second": 1.219,
+    "train_steps_per_second": 0.076
 }

trainer_state.json

@@ -1,244 +1,30 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.99660441426146,
-  "eval_steps": 50,
-  "global_step": 294,
+  "epoch": 0.006791171477079796,
+  "eval_steps": 500,
+  "global_step": 1,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.07,
-      "learning_rate": 3e-05,
-      "loss": 1.2025,
-      "step": 10
-    },
-    {
-      "epoch": 0.14,
-      "learning_rate": 3e-05,
-      "loss": 1.1367,
-      "step": 20
-    },
-    {
-      "epoch": 0.2,
-      "learning_rate": 3e-05,
-      "loss": 1.1325,
-      "step": 30
-    },
-    {
-      "epoch": 0.27,
-      "learning_rate": 3e-05,
-      "loss": 1.1111,
-      "step": 40
-    },
-    {
-      "epoch": 0.34,
-      "learning_rate": 3e-05,
-      "loss": 1.1008,
-      "step": 50
-    },
-    {
-      "epoch": 0.34,
-      "eval_loss": 1.1499062776565552,
-      "eval_runtime": 15.0698,
-      "eval_samples_per_second": 17.386,
-      "eval_steps_per_second": 4.38,
-      "step": 50
-    },
-    {
-      "epoch": 0.41,
-      "learning_rate": 3e-05,
-      "loss": 1.0935,
-      "step": 60
-    },
-    {
-      "epoch": 0.48,
-      "learning_rate": 3e-05,
-      "loss": 1.0925,
-      "step": 70
-    },
-    {
-      "epoch": 0.54,
-      "learning_rate": 3e-05,
-      "loss": 1.1389,
-      "step": 80
-    },
-    {
-      "epoch": 0.61,
-      "learning_rate": 3e-05,
-      "loss": 1.1186,
-      "step": 90
-    },
-    {
-      "epoch": 0.68,
-      "learning_rate": 3e-05,
-      "loss": 1.0663,
-      "step": 100
-    },
-    {
-      "epoch": 0.68,
-      "eval_loss": 1.1313854455947876,
-      "eval_runtime": 15.0724,
-      "eval_samples_per_second": 17.383,
-      "eval_steps_per_second": 4.379,
-      "step": 100
-    },
-    {
-      "epoch": 0.75,
-      "learning_rate": 3e-05,
-      "loss": 1.0572,
-      "step": 110
-    },
-    {
-      "epoch": 0.81,
-      "learning_rate": 3e-05,
-      "loss": 1.1099,
-      "step": 120
-    },
-    {
-      "epoch": 0.88,
-      "learning_rate": 3e-05,
-      "loss": 1.094,
-      "step": 130
-    },
-    {
-      "epoch": 0.95,
-      "learning_rate": 3e-05,
-      "loss": 1.074,
-      "step": 140
-    },
-    {
-      "epoch": 1.02,
-      "learning_rate": 3e-05,
-      "loss": 1.04,
-      "step": 150
-    },
-    {
-      "epoch": 1.02,
-      "eval_loss": 1.1262978315353394,
-      "eval_runtime": 15.1147,
-      "eval_samples_per_second": 17.334,
-      "eval_steps_per_second": 4.367,
-      "step": 150
-    },
-    {
-      "epoch": 1.09,
-      "learning_rate": 3e-05,
-      "loss": 0.9901,
-      "step": 160
-    },
-    {
-      "epoch": 1.15,
-      "learning_rate": 3e-05,
-      "loss": 1.0634,
-      "step": 170
-    },
-    {
-      "epoch": 1.22,
-      "learning_rate": 3e-05,
-      "loss": 0.9784,
-      "step": 180
-    },
-    {
-      "epoch": 1.29,
-      "learning_rate": 3e-05,
-      "loss": 0.9506,
-      "step": 190
-    },
-    {
-      "epoch": 1.36,
-      "learning_rate": 3e-05,
-      "loss": 1.0182,
-      "step": 200
-    },
-    {
-      "epoch": 1.36,
-      "eval_loss": 1.1319481134414673,
-      "eval_runtime": 15.0625,
-      "eval_samples_per_second": 17.394,
-      "eval_steps_per_second": 4.382,
-      "step": 200
-    },
-    {
-      "epoch": 1.43,
-      "learning_rate": 3e-05,
-      "loss": 0.9876,
-      "step": 210
-    },
-    {
-      "epoch": 1.49,
-      "learning_rate": 3e-05,
-      "loss": 0.9059,
-      "step": 220
-    },
-    {
-      "epoch": 1.56,
-      "learning_rate": 3e-05,
-      "loss": 0.997,
-      "step": 230
-    },
-    {
-      "epoch": 1.63,
-      "learning_rate": 3e-05,
-      "loss": 0.9893,
-      "step": 240
-    },
-    {
-      "epoch": 1.7,
-      "learning_rate": 3e-05,
-      "loss": 1.0291,
-      "step": 250
-    },
-    {
-      "epoch": 1.7,
-      "eval_loss": 1.1290254592895508,
-      "eval_runtime": 15.0498,
-      "eval_samples_per_second": 17.409,
-      "eval_steps_per_second": 4.385,
-      "step": 250
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 3e-05,
-      "loss": 0.966,
-      "step": 260
-    },
-    {
-      "epoch": 1.83,
-      "learning_rate": 3e-05,
-      "loss": 1.0419,
-      "step": 270
-    },
-    {
-      "epoch": 1.9,
-      "learning_rate": 3e-05,
-      "loss": 0.9625,
-      "step": 280
-    },
-    {
-      "epoch": 1.97,
-      "learning_rate": 3e-05,
-      "loss": 0.9313,
-      "step": 290
-    },
-    {
-      "epoch": 2.0,
-      "step": 294,
-      "total_flos": 7.119009482145792e+16,
-      "train_loss": 1.0449795049874961,
-      "train_runtime": 831.3165,
-      "train_samples_per_second": 5.668,
-      "train_steps_per_second": 0.354
+      "epoch": 0.01,
+      "step": 1,
+      "total_flos": 477581879672832.0,
+      "train_loss": 1.3776695728302002,
+      "train_runtime": 13.1306,
+      "train_samples_per_second": 1.219,
+      "train_steps_per_second": 0.076
     }
   ],
   "logging_steps": 10,
-  "max_steps": 294,
+  "max_steps": 1,
   "num_input_tokens_seen": 0,
-  "num_train_epochs": 2,
+  "num_train_epochs": 1,
   "save_steps": 250,
-  "total_flos": 7.119009482145792e+16,
-  "train_batch_size": 4,
+  "total_flos": 477581879672832.0,
+  "train_batch_size": 1,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9515e50a85ffdadfcbe8222eb903c1e1790911eadb7489ccbbcd0e8696cafb3
+oid sha256:b820acf2b0d501ce1b35e31a5661398a52cf98d95dbd0b403f87e2dc33c54bb1
 size 6648