const_tag: &MODEL1 maywell/Llama-3-Synatra-11B-v1 | |
const_tag: &ZERO_LINEAR_FACTOR 0.0 | |
block_expansion_env: | |
parameters: | |
scale: | |
- filter: o_proj | |
value: | |
- filter: down_proj | |
value: | |
- value: 1.0 | |
slices: | |
- sources: | |
- model: | |
layer_range: [0, 4] | |
- sources: | |
- model: | |
layer_range: [3, 4] | |
<<: | |
- sources: | |
- model: | |
layer_range: [4, 8] | |
- sources: | |
- model: | |
layer_range: [7, 8] | |
<<: | |
- sources: | |
- model: | |
layer_range: [8, 12] | |
- sources: | |
- model: | |
layer_range: [11, 12] | |
<<: | |
- sources: | |
- model: | |
layer_range: [12, 16] | |
- sources: | |
- model: | |
layer_range: [15, 16] | |
<<: | |
- sources: | |
- model: | |
layer_range: [16, 20] | |
- sources: | |
- model: | |
layer_range: [19, 20] | |
<<: | |
- sources: | |
- model: | |
layer_range: [20, 24] | |
- sources: | |
- model: | |
layer_range: [23, 24] | |
<<: | |
- sources: | |
- model: | |
layer_range: [24, 28] | |
- sources: | |
- model: | |
layer_range: [27, 28] | |
<<: | |
- sources: | |
- model: | |
layer_range: [28, 32] | |
- sources: | |
- model: | |
layer_range: [31, 32] | |
<<: | |
merge_method: passthrough | |
dtype: bfloat16 |