172 lines
3.7 KiB
JSON
172 lines
3.7 KiB
JSON
{
|
|
"add_bos_token": true,
|
|
"add_dummy_prefix_space": false,
|
|
"add_eos_token": true,
|
|
"add_prefix_space": false,
|
|
"added_tokens_decoder": {
|
|
"0": {
|
|
"content": "<unk>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"1": {
|
|
"content": "<s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"3": {
|
|
"content": "<pad>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"4": {
|
|
"content": "<sep>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"5": {
|
|
"content": "<mask>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"6": {
|
|
"content": "<cls>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": true
|
|
},
|
|
"7": {
|
|
"content": "<|system|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"8": {
|
|
"content": "<|assistant|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"9": {
|
|
"content": "<|user|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"10": {
|
|
"content": "<|available_tools|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"11": {
|
|
"content": "<|tool_calls|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"12": {
|
|
"content": "<|tool_results|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"13": {
|
|
"content": "<|code|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"14": {
|
|
"content": "<|file|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"102397": {
|
|
"content": "<|prefix|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"102398": {
|
|
"content": "<|suffix|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
},
|
|
"102399": {
|
|
"content": "<|middle|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
}
|
|
},
|
|
"bos_token": "<s>",
|
|
"clean_up_tokenization_spaces": false,
|
|
"cls_token": "<cls>",
|
|
"do_lower_case": false,
|
|
"eos_token": "</s>",
|
|
"extra_ids": 0,
|
|
"extra_special_tokens": {},
|
|
"keep_accents": true,
|
|
"legacy": false,
|
|
"mask_token": "<mask>",
|
|
"model_max_length": 1000000000000000019884624838656,
|
|
"pad_token": "<pad>",
|
|
"padding_side": "right",
|
|
"sep_token": "<sep>",
|
|
"sp_model_kwargs": {},
|
|
"spaces_between_special_tokens": false,
|
|
"tokenizer_class": "LlamaTokenizer",
|
|
"unk_token": "<unk>",
|
|
"use_default_system_prompt": false
|
|
}
|