Train on more data

README.md

@@ -47,7 +47,7 @@ The following hyperparameters were used during training:
 - optimizer: Adam with betas=(0.9, 0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_steps: 10
-- training_steps: 1800
+- num_epochs: 1
 
 ### Training Results
 
@@ -56,7 +56,6 @@ The following hyperparameters were used during training:
 | Groups      | Version | Filter    | n-shot | Metric      | Value  |     | Stderr |
 |-------------|---------|-----------|--------|-------------|--------|-----|--------|
 | kmmlu       | N/A     | none      | 0      | acc         | 0.3004 | ±   | 0.0528 |
-|             |         | none      | 0      | acc_norm    | 0.3004 | ±   | 0.0528 |
 | gsm8k       | Yaml    | get-answer| 5      | exact_match | 0.5625 | ±   | 0.0137 |
 | hellaswag   | Yaml    | none      | 0      | acc         | 0.6393 | ±   | 0.0048 |
 | mmlu        | N/A     | none      | 0      | acc         | 0.6305 | ±   | 0.1452 |
@@ -67,8 +66,7 @@ The following hyperparameters were used during training:
 
 | Groups      | Version | Filter    | n-shot | Metric      | Value  |     | Stderr |
 |-------------|---------|-----------|--------|-------------|--------|-----|--------|
-| kmmlu       | N/A     | none      | 0      | acc         | 0.2946 | ±   | 0.0496 |
-|             |         | none      | 0      | acc_norm    | 0.2946 | ±   | 0.0496 |
+| kmmlu       | N/A     | none      | 0      | acc         | 0.2948 | ±   | 0.0537 |
 | gsm8k       | Yaml    | get-answer| 5      | exact_match | 0.5527 | ±   | 0.0137 |
 | hellaswag   | Yaml    | none      | 0      | acc         | 0.6392 | ±   | 0.0048 |
 | mmlu        | N/A     | none      | 0      | acc         | 0.6303 | ±   | 0.1411 |
@@ -80,4 +78,4 @@ The following hyperparameters were used during training:
 - Transformers 4.37.0.dev0
 - Pytorch 2.1.2+cu121
 - Datasets 2.16.0
-- Tokenizers 0.15.0
+- Tokenizers 0.15.0

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "upstage/SOLAR-10.7B-v1.0",
+  "_name_or_path": "seungduk/KoSOLAR-10.7B-v0.1",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -21,7 +21,7 @@
   "rope_scaling": null,
   "rope_theta": 10000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float16",
   "transformers_version": "4.37.0.dev0",
   "use_cache": false,
   "vocab_size": 48000

model-00001-of-00005.safetensors → model-00001-of-00012.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d510e825becf9c173099932f9c0547789483ddc207839fc2d8aa4fd9fa125124
-size 4956793712
+oid sha256:f0ae8ac5865ae8ca26224c249b7c83d671f1f9b5c4f3c0478861e67ea63bad48
+size 1903218384

model-00002-of-00005.safetensors → model-00002-of-00012.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:17b803b1cdf74d084c58d00b9bafc7a144fbb6959b1e6b35923b450dccfa58e6
-size 4915916176
+oid sha256:53000e1bfd1396a50cac2f27d8a162484e3f7f5f27fb7ce95ad00f42075294d1
+size 1979797984

model-00003-of-00005.safetensors → model-00003-of-00012.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3d6de738bcfbb7540234185024060d183ff7c9998b328337791a6e7e065d692a
-size 4999819336
+oid sha256:55f9865fdb5aaa4faf5d49b449a89d3a935e8179f2dc9b498fe1a3701a3ee63c
+size 1946227312

model-00004-of-00005.safetensors → model-00004-of-00012.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21d58de47a649d1a263c2649a14ccad3841cecf942f0e00e94ee3581ec4b8fd0
-size 4915916184
+oid sha256:bc076d4133a17ff2d3531a8e6667c0537febfe5488b989b7af1abdcfa830ae63
+size 1979798024

model-00005-of-00005.safetensors

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:a646fd121dcc5933a4a77468b814a4a8adad35093f6c9c7f81fb4d29b77b1ce9
-size 1936797416

model-00005-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd23fd11114e0e6033d45cd47c6371f7e08e82b9545adb2375af081480380e5
+size 1946227328

model-00006-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4a34c297dc5cb8c69d9edaf974804fb7bea5a66fa5e9be140ca30335a4168b38
+size 1979798024

model-00007-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:23b8a30ce5ba7ae3ff7529298ef929e9caebdde74bde80cfb087b9482a3e1039
+size 1946227328

model-00008-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d38c8fe99ecfeda8887ccdc4fdfc212d67cb3f23c787d7232aa0401680e3f11c
+size 1979798024

model-00009-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e0e142d0db2a112fe87930cb1de2fb74c205079e437984f6731ff6543224939
+size 1946227328

model-00010-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:236aca1fbe91160048003d21334ed9cb4aa3aeec83531dd28bd24b2efb09fc47
+size 1979798024

model-00011-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:872fe8cf4ee3289af57c88406f63d9c0e121836f1db7f2099f752fb8d7c69b80
+size 1744908432

model-00012-of-00012.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c03284af99d00cf767081e67f2fd54d8b52ddd23245f99569e97e67e98df39d
+size 393216128

model.safetensors.index.json

@@ -3,440 +3,440 @@
     "total_size": 21725192192
   },
   "weight_map": {
-    "lm_head.weight": "model-00005-of-00005.safetensors",
-    "model.embed_tokens.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.10.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.10.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.11.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.2.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.20.input_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.21.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.21.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00005.safetensors",
-    "model.layers.22.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.22.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.3.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.30.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.input_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.mlp.down_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.mlp.gate_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.mlp.up_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.post_attention_layernorm.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.32.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.33.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.33.self_attn.k_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.33.self_attn.o_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.33.self_attn.q_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.33.self_attn.v_proj.weight": "model-00003-of-00005.safetensors",
-    "model.layers.34.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.34.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.35.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.36.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.37.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.38.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.39.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.4.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.40.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.40.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.41.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.42.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.input_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.mlp.down_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.mlp.up_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.post_attention_layernorm.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.43.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.44.input_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.44.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.44.mlp.gate_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.44.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.44.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.44.self_attn.k_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.44.self_attn.o_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.44.self_attn.q_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.44.self_attn.v_proj.weight": "model-00004-of-00005.safetensors",
-    "model.layers.45.input_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.45.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.input_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.46.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.input_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.mlp.down_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.mlp.gate_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.mlp.up_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.post_attention_layernorm.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.self_attn.k_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.self_attn.o_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.self_attn.q_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.47.self_attn.v_proj.weight": "model-00005-of-00005.safetensors",
-    "model.layers.5.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.input_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00005.safetensors",
-    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00005.safetensors",
-    "model.norm.weight": "model-00005-of-00005.safetensors"
+    "lm_head.weight": "model-00012-of-00012.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00004-of-00012.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00005-of-00012.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00006-of-00012.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00012.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00007-of-00012.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.input_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.mlp.down_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.mlp.gate_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.mlp.up_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.post_attention_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.self_attn.k_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.self_attn.o_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.self_attn.q_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.32.self_attn.v_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.input_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.mlp.down_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.mlp.gate_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.mlp.up_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.post_attention_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.self_attn.k_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.self_attn.o_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.self_attn.q_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.33.self_attn.v_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.input_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.mlp.down_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.mlp.gate_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.mlp.up_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.post_attention_layernorm.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.self_attn.k_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.self_attn.o_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.self_attn.q_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.34.self_attn.v_proj.weight": "model-00008-of-00012.safetensors",
+    "model.layers.35.input_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.mlp.down_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.mlp.gate_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.mlp.up_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.post_attention_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.self_attn.k_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.self_attn.o_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.self_attn.q_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.35.self_attn.v_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.input_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.mlp.down_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.mlp.gate_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.mlp.up_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.post_attention_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.self_attn.k_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.self_attn.o_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.self_attn.q_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.36.self_attn.v_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.input_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.mlp.down_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.mlp.gate_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.mlp.up_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.post_attention_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.self_attn.k_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.self_attn.o_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.self_attn.q_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.37.self_attn.v_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.input_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.mlp.down_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.mlp.gate_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.mlp.up_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.post_attention_layernorm.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.self_attn.k_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.self_attn.o_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.self_attn.q_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.38.self_attn.v_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.39.input_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.39.mlp.down_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.39.mlp.gate_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.39.mlp.up_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.39.post_attention_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.39.self_attn.k_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.39.self_attn.o_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.39.self_attn.q_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.39.self_attn.v_proj.weight": "model-00009-of-00012.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.40.input_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.mlp.down_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.mlp.gate_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.mlp.up_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.post_attention_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.self_attn.k_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.self_attn.o_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.self_attn.q_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.40.self_attn.v_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.input_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.mlp.down_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.mlp.gate_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.mlp.up_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.post_attention_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.self_attn.k_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.self_attn.o_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.self_attn.q_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.41.self_attn.v_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.input_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.mlp.down_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.mlp.gate_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.mlp.up_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.post_attention_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.self_attn.k_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.self_attn.o_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.self_attn.q_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.42.self_attn.v_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.input_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.mlp.down_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.mlp.gate_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.mlp.up_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.post_attention_layernorm.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.self_attn.k_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.self_attn.o_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.self_attn.q_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.43.self_attn.v_proj.weight": "model-00010-of-00012.safetensors",
+    "model.layers.44.input_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.mlp.down_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.mlp.gate_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.mlp.up_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.post_attention_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.self_attn.k_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.self_attn.o_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.self_attn.q_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.44.self_attn.v_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.input_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.mlp.down_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.mlp.gate_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.mlp.up_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.post_attention_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.self_attn.k_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.self_attn.o_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.self_attn.q_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.45.self_attn.v_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.input_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.mlp.down_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.mlp.gate_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.mlp.up_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.post_attention_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.self_attn.k_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.self_attn.o_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.self_attn.q_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.46.self_attn.v_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.input_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.mlp.down_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.mlp.gate_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.mlp.up_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.post_attention_layernorm.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.self_attn.k_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.self_attn.o_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.self_attn.q_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.47.self_attn.v_proj.weight": "model-00011-of-00012.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00002-of-00012.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00003-of-00012.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00003-of-00012.safetensors",
+    "model.norm.weight": "model-00011-of-00012.safetensors"
   }
 }

pytorch_model-00001-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:823f2d8ab36dd35153cb2a707820ef849b8dc3d1003b085d3c0755b47aadb20c
+oid sha256:858849843f69e1bb65ec13b80a4c3a8a514085467bc7fd5eb2f5b0d5cdea86db
 size 4956816866

pytorch_model-00005-of-00005.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6333b34f4e23f15c51f362628891a39ac6ccba64aaec6846bdc3efe23501345
+oid sha256:d379580f8b63c860e8b958539e94bff62f16b7aa7d6e88aa837595c1b90ec20f
 size 1936805500

tokenizer_config.json

@@ -32,7 +32,7 @@
       "normalized": false,
       "rstrip": false,
       "single_word": false,
-      "special": false
+      "special": true
     },
     "32001": {
       "content": "<|im_start|>",