125 lines
1.6 KiB
JSON
125 lines
1.6 KiB
JSON
|
{
|
||
|
"activation_fn_name": "swish",
|
||
|
"architectures": [
|
||
|
"OpenELMForCausalLM"
|
||
|
],
|
||
|
"auto_map": {
|
||
|
"AutoConfig": "configuration_openelm.OpenELMConfig",
|
||
|
"AutoModelForCausalLM": "modeling_openelm.OpenELMForCausalLM"
|
||
|
},
|
||
|
"bos_token_id": 1,
|
||
|
"eos_token_id": 2,
|
||
|
"ffn_dim_divisor": 256,
|
||
|
"ffn_multipliers": [
|
||
|
0.5,
|
||
|
0.63,
|
||
|
0.76,
|
||
|
0.89,
|
||
|
1.02,
|
||
|
1.15,
|
||
|
1.28,
|
||
|
1.41,
|
||
|
1.54,
|
||
|
1.67,
|
||
|
1.8,
|
||
|
1.93,
|
||
|
2.06,
|
||
|
2.19,
|
||
|
2.31,
|
||
|
2.44,
|
||
|
2.57,
|
||
|
2.7,
|
||
|
2.83,
|
||
|
2.96,
|
||
|
3.09,
|
||
|
3.22,
|
||
|
3.35,
|
||
|
3.48,
|
||
|
3.61,
|
||
|
3.74,
|
||
|
3.87,
|
||
|
4.0
|
||
|
],
|
||
|
"ffn_with_glu": true,
|
||
|
"head_dim": 64,
|
||
|
"initializer_range": 0.02,
|
||
|
"max_context_length": 2048,
|
||
|
"model_dim": 2048,
|
||
|
"model_type": "openelm",
|
||
|
"normalization_layer_name": "rms_norm",
|
||
|
"normalize_qk_projections": true,
|
||
|
"num_gqa_groups": 4,
|
||
|
"num_kv_heads": [
|
||
|
4,
|
||
|
4,
|
||
|
4,
|
||
|
5,
|
||
|
5,
|
||
|
5,
|
||
|
5,
|
||
|
5,
|
||
|
5,
|
||
|
5,
|
||
|
6,
|
||
|
6,
|
||
|
6,
|
||
|
6,
|
||
|
6,
|
||
|
6,
|
||
|
6,
|
||
|
6,
|
||
|
7,
|
||
|
7,
|
||
|
7,
|
||
|
7,
|
||
|
7,
|
||
|
7,
|
||
|
8,
|
||
|
8,
|
||
|
8,
|
||
|
8
|
||
|
],
|
||
|
"num_query_heads": [
|
||
|
16,
|
||
|
16,
|
||
|
16,
|
||
|
20,
|
||
|
20,
|
||
|
20,
|
||
|
20,
|
||
|
20,
|
||
|
20,
|
||
|
20,
|
||
|
24,
|
||
|
24,
|
||
|
24,
|
||
|
24,
|
||
|
24,
|
||
|
24,
|
||
|
24,
|
||
|
24,
|
||
|
28,
|
||
|
28,
|
||
|
28,
|
||
|
28,
|
||
|
28,
|
||
|
28,
|
||
|
32,
|
||
|
32,
|
||
|
32,
|
||
|
32
|
||
|
],
|
||
|
"num_transformer_layers": 28,
|
||
|
"qkv_multipliers": [
|
||
|
0.5,
|
||
|
1.0
|
||
|
],
|
||
|
"rope_freq_constant": 10000,
|
||
|
"rope_max_length": 4096,
|
||
|
"share_input_output_layers": true,
|
||
|
"torch_dtype": "bfloat16",
|
||
|
"transformers_version": "4.39.3",
|
||
|
"use_cache": true,
|
||
|
"vocab_size": 32000
|
||
|
}
|