File size: 2,782 Bytes
4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e 4286500 9d8458e |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 |
base_model: bamec66557/VICIOUS_MESH-12B-OMEGA
dtype: bfloat16
merge_method: slerp
tokenizer_source: base
# Slices Configuration
slices:
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [0, 10]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [0, 10]
parameters:
t:
- name: self_attn
value: [0.5, 0.55, 0.6, 0.65, 0.7]
- name: mlp
value: [1.0, 1.05, 1.1, 1.15, 1.2]
- name: layer_norm
value: [0.9, 0.95, 1.0, 1.05, 1.1]
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [10, 20]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [10, 20]
parameters:
t:
- name: self_attn
value: [0.4, 0.45, 0.5, 0.55, 0.6]
- name: mlp
value: [1.1, 1.15, 1.2, 1.25, 1.3]
- name: layer_norm
value: [1.0, 1.05, 1.1, 1.15, 1.2]
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [20, 30]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [20, 30]
parameters:
t:
- name: self_attn
value: [0.6, 0.65, 0.7, 0.75, 0.8]
- name: mlp
value: [0.9, 0.95, 1.0, 1.05, 1.1]
- name: layer_norm
value: [0.85, 0.9, 0.95, 1.0, 1.05]
- sources:
- model: bamec66557/VICIOUS_MESH-12B-OMEGA
layer_range: [30, 40]
- model: bamec66557/VICIOUS_MESH-12B-BETA
layer_range: [30, 40]
parameters:
t:
- name: self_attn
value: [0.7, 0.75, 0.8, 0.85, 0.9]
- name: mlp
value: [0.8, 0.85, 0.9, 0.95, 1.0]
- name: layer_norm
value: [0.8, 0.85, 0.9, 0.95, 1.0]
# Regularization
regularization:
- method: gradient_penalty
scale: 0.05 # Increased influence for gradient control
- method: weight_clipping
clip_range: [-0.2, 0.2] # Broader clipping range for flexibility
- method: random_noise
scale: 0.01 # Stronger noise injection
- method: attention_dropout
scale: 0.1 # Higher dropout to reduce attention fixation
# Postprocessing
postprocessing:
- operation: entropy_regularization
scale: 0.05 # Stronger encouragement for diverse outputs
- operation: non_linear_scaling
parameters:
function: tanh
- operation: sharpening
intensity: 0.5 # Enhanced sharpening for precise outputs
- operation: gaussian_smoothing
sigma: 1.5 # Increased smoothing for stable outputs
- operation: normalize
- operation: dynamic_scaling
scale_range: [0.8, 1.2] # Expanded dynamic range for scaling
- operation: smoothing
parameters:
adaptive: true
range: [0.85, 1.15] # Wider adaptive smoothing range
kernel_size: 5
|