galqiwi commited on
Commit
898350e
1 Parent(s): 866577c

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +178 -11
config.json CHANGED
@@ -1,4 +1,5 @@
1
  {
 
2
  "architectures": [
3
  "LlamaForCausalLM"
4
  ],
@@ -10,25 +11,191 @@
10
  "hidden_size": 8192,
11
  "initializer_range": 0.02,
12
  "intermediate_size": 28672,
13
- "max_position_embeddings": 131072,
14
- "mlp_bias": false,
15
  "model_type": "llama",
16
  "num_attention_heads": 64,
17
  "num_hidden_layers": 80,
18
  "num_key_value_heads": 8,
19
  "pretraining_tp": 1,
20
- "rms_norm_eps": 1e-05,
21
- "rope_scaling": {
22
- "factor": 8.0,
23
- "low_freq_factor": 1.0,
24
- "high_freq_factor": 4.0,
25
- "original_max_position_embeddings": 8192,
26
- "rope_type": "llama3"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  },
 
 
28
  "rope_theta": 500000.0,
29
  "tie_word_embeddings": false,
30
- "torch_dtype": "bfloat16",
31
- "transformers_version": "4.43.0.dev0",
32
  "use_cache": true,
33
  "vocab_size": 128256
34
  }
 
 
1
  {
2
+ "_name_or_path": "/slot/sandbox/j/_tmp/data3p0913nd",
3
  "architectures": [
4
  "LlamaForCausalLM"
5
  ],
 
11
  "hidden_size": 8192,
12
  "initializer_range": 0.02,
13
  "intermediate_size": 28672,
14
+ "max_position_embeddings": 8192,
 
15
  "model_type": "llama",
16
  "num_attention_heads": 64,
17
  "num_hidden_layers": 80,
18
  "num_key_value_heads": 8,
19
  "pretraining_tp": 1,
20
+ "quantization_config": {
21
+ "in_group_size": 8,
22
+ "linear_weights_not_to_quantize": [
23
+ "model.layers.0.input_layernorm.weight",
24
+ "model.layers.0.post_attention_layernorm.weight",
25
+ "model.layers.1.input_layernorm.weight",
26
+ "model.layers.1.post_attention_layernorm.weight",
27
+ "model.layers.2.input_layernorm.weight",
28
+ "model.layers.2.post_attention_layernorm.weight",
29
+ "model.layers.3.input_layernorm.weight",
30
+ "model.layers.3.post_attention_layernorm.weight",
31
+ "model.layers.4.input_layernorm.weight",
32
+ "model.layers.4.post_attention_layernorm.weight",
33
+ "model.layers.5.input_layernorm.weight",
34
+ "model.layers.5.post_attention_layernorm.weight",
35
+ "model.layers.6.input_layernorm.weight",
36
+ "model.layers.6.post_attention_layernorm.weight",
37
+ "model.layers.7.input_layernorm.weight",
38
+ "model.layers.7.post_attention_layernorm.weight",
39
+ "model.layers.8.input_layernorm.weight",
40
+ "model.layers.8.post_attention_layernorm.weight",
41
+ "model.layers.9.input_layernorm.weight",
42
+ "model.layers.9.post_attention_layernorm.weight",
43
+ "model.layers.10.input_layernorm.weight",
44
+ "model.layers.10.post_attention_layernorm.weight",
45
+ "model.layers.11.input_layernorm.weight",
46
+ "model.layers.11.post_attention_layernorm.weight",
47
+ "model.layers.12.input_layernorm.weight",
48
+ "model.layers.12.post_attention_layernorm.weight",
49
+ "model.layers.13.input_layernorm.weight",
50
+ "model.layers.13.post_attention_layernorm.weight",
51
+ "model.layers.14.input_layernorm.weight",
52
+ "model.layers.14.post_attention_layernorm.weight",
53
+ "model.layers.15.input_layernorm.weight",
54
+ "model.layers.15.post_attention_layernorm.weight",
55
+ "model.layers.16.input_layernorm.weight",
56
+ "model.layers.16.post_attention_layernorm.weight",
57
+ "model.layers.17.input_layernorm.weight",
58
+ "model.layers.17.post_attention_layernorm.weight",
59
+ "model.layers.18.input_layernorm.weight",
60
+ "model.layers.18.post_attention_layernorm.weight",
61
+ "model.layers.19.input_layernorm.weight",
62
+ "model.layers.19.post_attention_layernorm.weight",
63
+ "model.layers.20.input_layernorm.weight",
64
+ "model.layers.20.post_attention_layernorm.weight",
65
+ "model.layers.21.input_layernorm.weight",
66
+ "model.layers.21.post_attention_layernorm.weight",
67
+ "model.layers.22.input_layernorm.weight",
68
+ "model.layers.22.post_attention_layernorm.weight",
69
+ "model.layers.23.input_layernorm.weight",
70
+ "model.layers.23.post_attention_layernorm.weight",
71
+ "model.layers.24.input_layernorm.weight",
72
+ "model.layers.24.post_attention_layernorm.weight",
73
+ "model.layers.25.input_layernorm.weight",
74
+ "model.layers.25.post_attention_layernorm.weight",
75
+ "model.layers.26.input_layernorm.weight",
76
+ "model.layers.26.post_attention_layernorm.weight",
77
+ "model.layers.27.input_layernorm.weight",
78
+ "model.layers.27.post_attention_layernorm.weight",
79
+ "model.layers.28.input_layernorm.weight",
80
+ "model.layers.28.post_attention_layernorm.weight",
81
+ "model.layers.29.input_layernorm.weight",
82
+ "model.layers.29.post_attention_layernorm.weight",
83
+ "model.layers.30.input_layernorm.weight",
84
+ "model.layers.30.post_attention_layernorm.weight",
85
+ "model.layers.31.input_layernorm.weight",
86
+ "model.layers.31.post_attention_layernorm.weight",
87
+ "model.layers.32.input_layernorm.weight",
88
+ "model.layers.32.post_attention_layernorm.weight",
89
+ "model.layers.33.input_layernorm.weight",
90
+ "model.layers.33.post_attention_layernorm.weight",
91
+ "model.layers.34.input_layernorm.weight",
92
+ "model.layers.34.post_attention_layernorm.weight",
93
+ "model.layers.35.input_layernorm.weight",
94
+ "model.layers.35.post_attention_layernorm.weight",
95
+ "model.layers.36.input_layernorm.weight",
96
+ "model.layers.36.post_attention_layernorm.weight",
97
+ "model.layers.37.input_layernorm.weight",
98
+ "model.layers.37.post_attention_layernorm.weight",
99
+ "model.layers.38.input_layernorm.weight",
100
+ "model.layers.38.post_attention_layernorm.weight",
101
+ "model.layers.39.input_layernorm.weight",
102
+ "model.layers.39.post_attention_layernorm.weight",
103
+ "model.layers.40.input_layernorm.weight",
104
+ "model.layers.40.post_attention_layernorm.weight",
105
+ "model.layers.41.input_layernorm.weight",
106
+ "model.layers.41.post_attention_layernorm.weight",
107
+ "model.layers.42.input_layernorm.weight",
108
+ "model.layers.42.post_attention_layernorm.weight",
109
+ "model.layers.43.input_layernorm.weight",
110
+ "model.layers.43.post_attention_layernorm.weight",
111
+ "model.layers.44.input_layernorm.weight",
112
+ "model.layers.44.post_attention_layernorm.weight",
113
+ "model.layers.45.input_layernorm.weight",
114
+ "model.layers.45.post_attention_layernorm.weight",
115
+ "model.layers.46.input_layernorm.weight",
116
+ "model.layers.46.post_attention_layernorm.weight",
117
+ "model.layers.47.input_layernorm.weight",
118
+ "model.layers.47.post_attention_layernorm.weight",
119
+ "model.layers.48.input_layernorm.weight",
120
+ "model.layers.48.post_attention_layernorm.weight",
121
+ "model.layers.49.input_layernorm.weight",
122
+ "model.layers.49.post_attention_layernorm.weight",
123
+ "model.layers.50.input_layernorm.weight",
124
+ "model.layers.50.post_attention_layernorm.weight",
125
+ "model.layers.51.input_layernorm.weight",
126
+ "model.layers.51.post_attention_layernorm.weight",
127
+ "model.layers.52.input_layernorm.weight",
128
+ "model.layers.52.post_attention_layernorm.weight",
129
+ "model.layers.53.input_layernorm.weight",
130
+ "model.layers.53.post_attention_layernorm.weight",
131
+ "model.layers.54.input_layernorm.weight",
132
+ "model.layers.54.post_attention_layernorm.weight",
133
+ "model.layers.55.input_layernorm.weight",
134
+ "model.layers.55.post_attention_layernorm.weight",
135
+ "model.layers.56.input_layernorm.weight",
136
+ "model.layers.56.post_attention_layernorm.weight",
137
+ "model.layers.57.input_layernorm.weight",
138
+ "model.layers.57.post_attention_layernorm.weight",
139
+ "model.layers.58.input_layernorm.weight",
140
+ "model.layers.58.post_attention_layernorm.weight",
141
+ "model.layers.59.input_layernorm.weight",
142
+ "model.layers.59.post_attention_layernorm.weight",
143
+ "model.layers.60.input_layernorm.weight",
144
+ "model.layers.60.post_attention_layernorm.weight",
145
+ "model.layers.61.input_layernorm.weight",
146
+ "model.layers.61.post_attention_layernorm.weight",
147
+ "model.layers.62.input_layernorm.weight",
148
+ "model.layers.62.post_attention_layernorm.weight",
149
+ "model.layers.63.input_layernorm.weight",
150
+ "model.layers.63.post_attention_layernorm.weight",
151
+ "model.layers.64.input_layernorm.weight",
152
+ "model.layers.64.post_attention_layernorm.weight",
153
+ "model.layers.65.input_layernorm.weight",
154
+ "model.layers.65.post_attention_layernorm.weight",
155
+ "model.layers.66.input_layernorm.weight",
156
+ "model.layers.66.post_attention_layernorm.weight",
157
+ "model.layers.67.input_layernorm.weight",
158
+ "model.layers.67.post_attention_layernorm.weight",
159
+ "model.layers.68.input_layernorm.weight",
160
+ "model.layers.68.post_attention_layernorm.weight",
161
+ "model.layers.69.input_layernorm.weight",
162
+ "model.layers.69.post_attention_layernorm.weight",
163
+ "model.layers.70.input_layernorm.weight",
164
+ "model.layers.70.post_attention_layernorm.weight",
165
+ "model.layers.71.input_layernorm.weight",
166
+ "model.layers.71.post_attention_layernorm.weight",
167
+ "model.layers.72.input_layernorm.weight",
168
+ "model.layers.72.post_attention_layernorm.weight",
169
+ "model.layers.73.input_layernorm.weight",
170
+ "model.layers.73.post_attention_layernorm.weight",
171
+ "model.layers.74.input_layernorm.weight",
172
+ "model.layers.74.post_attention_layernorm.weight",
173
+ "model.layers.75.input_layernorm.weight",
174
+ "model.layers.75.post_attention_layernorm.weight",
175
+ "model.layers.76.input_layernorm.weight",
176
+ "model.layers.76.post_attention_layernorm.weight",
177
+ "model.layers.77.input_layernorm.weight",
178
+ "model.layers.77.post_attention_layernorm.weight",
179
+ "model.layers.78.input_layernorm.weight",
180
+ "model.layers.78.post_attention_layernorm.weight",
181
+ "model.layers.79.input_layernorm.weight",
182
+ "model.layers.79.post_attention_layernorm.weight",
183
+ "model.embed_tokens.weight",
184
+ "model.norm.weight",
185
+ "lm_head.weight"
186
+ ],
187
+ "nbits_per_codebook": 16,
188
+ "num_codebooks": 1,
189
+ "out_group_size": 1,
190
+ "quant_method": "aqlm"
191
  },
192
+ "rms_norm_eps": 1e-05,
193
+ "rope_scaling": null,
194
  "rope_theta": 500000.0,
195
  "tie_word_embeddings": false,
196
+ "torch_dtype": "float16",
197
+ "transformers_version": "4.40.0.dev0",
198
  "use_cache": true,
199
  "vocab_size": 128256
200
  }
201
+