From 00d123555352f63c389bb57dec15a76171f6e94c Mon Sep 17 00:00:00 2001
From: wsgtest <4918b15d@leinao.ai>
Date: Fri, 5 Jul 2024 17:36:46 +0800
Subject: [PATCH] Add tokenizer_config.json

Signed-off-by: wsgtest <4918b15d@leinao.ai>
---
 tokenizer_config.json | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 tokenizer_config.json

diff --git a/tokenizer_config.json b/tokenizer_config.json
new file mode 100644
index 0000000..7592fbd
--- /dev/null
+++ b/tokenizer_config.json
@@ -0,0 +1 @@
+{"unk_token": "<|endoftext|>", "bos_token": "<|endoftext|>", "eos_token": "<|endoftext|>", "add_prefix_space": false, "model_max_length": 1024, "special_tokens_map_file": null, "name_or_path": "./models/", "tokenizer_class": "GPT2Tokenizer"}
\ No newline at end of file