858 lines
18 KiB
JSON
858 lines
18 KiB
JSON
{
|
|
"added_tokens_decoder": {
|
|
"1": {
|
|
"content": "<s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": true
|
|
},
|
|
"2": {
|
|
"content": "</s>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": true
|
|
},
|
|
"3": {
|
|
"content": "<pad>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": true
|
|
},
|
|
"4": {
|
|
"content": "<|im_start|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"5": {
|
|
"content": "<|im_end|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"6": {
|
|
"content": "<|im_sp_00|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"7": {
|
|
"content": "<|im_sp_01|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"8": {
|
|
"content": "<|im_sp_02|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"9": {
|
|
"content": "<|im_sp_03|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"10": {
|
|
"content": "<|im_sp_04|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"11": {
|
|
"content": "<|im_sp_05|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"12": {
|
|
"content": "<|im_sp_06|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"13": {
|
|
"content": "<|im_sp_07|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"14": {
|
|
"content": "<|im_sp_08|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"15": {
|
|
"content": "<|im_sp_09|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"16": {
|
|
"content": "<|im_sp_10|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"17": {
|
|
"content": "<|im_sp_11|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"18": {
|
|
"content": "<|im_sp_12|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"19": {
|
|
"content": "<|im_sp_13|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"20": {
|
|
"content": "<|im_sp_14|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"21": {
|
|
"content": "<|im_sp_15|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"22": {
|
|
"content": "<|im_sp_16|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"23": {
|
|
"content": "<|im_sp_17|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"24": {
|
|
"content": "<|im_sp_18|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"25": {
|
|
"content": "<|im_sp_19|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"26": {
|
|
"content": "<|im_sp_20|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"27": {
|
|
"content": "<|im_sp_21|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"28": {
|
|
"content": "<|im_sp_22|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"29": {
|
|
"content": "<|im_sp_23|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"30": {
|
|
"content": "<|im_sp_24|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"31": {
|
|
"content": "<|im_sp_25|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"32": {
|
|
"content": "<|im_sp_26|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"33": {
|
|
"content": "<|im_sp_27|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"34": {
|
|
"content": "<|im_sp_28|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"35": {
|
|
"content": "<|im_sp_29|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"36": {
|
|
"content": "<|im_sp_30|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"37": {
|
|
"content": "<|im_sp_31|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"38": {
|
|
"content": "<|im_sp_32|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"39": {
|
|
"content": "<|im_sp_33|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"40": {
|
|
"content": "<|im_sp_34|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"41": {
|
|
"content": "<|im_sp_35|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"42": {
|
|
"content": "<|im_sp_36|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"43": {
|
|
"content": "<|im_sp_37|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"44": {
|
|
"content": "<|im_sp_38|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"45": {
|
|
"content": "<|im_sp_39|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"46": {
|
|
"content": "<|im_sp_40|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"47": {
|
|
"content": "<|im_sp_41|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"48": {
|
|
"content": "<|im_sp_42|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"49": {
|
|
"content": "<|im_sp_43|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"50": {
|
|
"content": "<|im_sp_44|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"51": {
|
|
"content": "<|im_sp_45|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"52": {
|
|
"content": "<|im_sp_46|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"53": {
|
|
"content": "<|im_sp_47|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"54": {
|
|
"content": "<|im_sp_48|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"55": {
|
|
"content": "<|im_sp_49|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"56": {
|
|
"content": "<|im_sp_50|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"57": {
|
|
"content": "<|im_sp_51|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"58": {
|
|
"content": "<|im_sp_52|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"59": {
|
|
"content": "<|im_sp_53|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"60": {
|
|
"content": "<|im_sp_54|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"61": {
|
|
"content": "<|im_sp_55|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"62": {
|
|
"content": "<|im_sp_56|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"63": {
|
|
"content": "<|im_sp_57|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"64": {
|
|
"content": "<|im_sp_58|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"65": {
|
|
"content": "<|im_sp_59|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"66": {
|
|
"content": "<|im_sp_60|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"67": {
|
|
"content": "<|im_sp_61|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"68": {
|
|
"content": "<|im_sp_62|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"69": {
|
|
"content": "<|im_sp_63|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"70": {
|
|
"content": "<|im_sp_64|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"71": {
|
|
"content": "<|im_sp_65|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"72": {
|
|
"content": "<|im_sp_66|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"73": {
|
|
"content": "<|im_sp_67|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"74": {
|
|
"content": "<|im_sp_68|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"75": {
|
|
"content": "<|im_sp_69|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"76": {
|
|
"content": "<|im_sp_70|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"77": {
|
|
"content": "<|im_sp_71|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"78": {
|
|
"content": "<|im_sp_72|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"79": {
|
|
"content": "<|im_sp_73|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"80": {
|
|
"content": "<|im_sp_74|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"81": {
|
|
"content": "<|im_sp_75|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"82": {
|
|
"content": "<|im_sp_76|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"83": {
|
|
"content": "<|im_sp_77|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"84": {
|
|
"content": "<|im_sp_78|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"85": {
|
|
"content": "<|im_sp_79|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"86": {
|
|
"content": "<|im_sp_80|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"87": {
|
|
"content": "<|im_sp_81|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"88": {
|
|
"content": "<|im_sp_82|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"89": {
|
|
"content": "<|im_sp_83|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"90": {
|
|
"content": "<|im_sp_84|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"91": {
|
|
"content": "<|im_sp_85|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"92": {
|
|
"content": "<|im_sp_86|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"93": {
|
|
"content": "<|im_sp_87|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"94": {
|
|
"content": "<|im_sp_88|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"95": {
|
|
"content": "<|im_sp_89|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"96": {
|
|
"content": "<|im_sp_90|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"97": {
|
|
"content": "<|im_sp_91|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"98": {
|
|
"content": "<|im_sp_92|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"99": {
|
|
"content": "<|im_sp_93|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"100": {
|
|
"content": "<|im_sp_94|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"101": {
|
|
"content": "<|im_sp_95|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"102": {
|
|
"content": "<|im_sp_96|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"103": {
|
|
"content": "<|im_sp_97|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"104": {
|
|
"content": "<|im_sp_98|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"105": {
|
|
"content": "<|im_sp_99|>",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": true,
|
|
"special": false
|
|
},
|
|
"116": {
|
|
"content": "\n",
|
|
"lstrip": false,
|
|
"normalized": false,
|
|
"rstrip": false,
|
|
"single_word": false,
|
|
"special": false
|
|
}
|
|
},
|
|
"clean_up_tokenization_spaces": false,
|
|
"extra_special_tokens": {},
|
|
"model_max_length": 1000000000000000019884624838656,
|
|
"tokenizer_class": "PreTrainedTokenizerFast",
|
|
"model_input_names": ["input_ids", "attention_mask"]
|
|
}
|