first commit
This commit is contained in:
xxl
parent
5c17ed1168
commit
459becf365
309
README.md
309
README.md
|
|
@ -1,3 +1,308 @@
|
||||||
# granite-3.0-1b-a400m-base
|
---
|
||||||
|
pipeline_tag: text-generation
|
||||||
|
inference: false
|
||||||
|
license: apache-2.0
|
||||||
|
library_name: transformers
|
||||||
|
tags:
|
||||||
|
- language
|
||||||
|
- granite-3.0
|
||||||
|
model-index:
|
||||||
|
- name: granite-3.0-1b-a400m-base
|
||||||
|
results:
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: human-exams
|
||||||
|
name: MMLU
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 25.69
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: human-exams
|
||||||
|
name: MMLU-Pro
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 11.38
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: human-exams
|
||||||
|
name: AGI-Eval
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 19.96
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: commonsense
|
||||||
|
name: WinoGrande
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 62.43
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: commonsense
|
||||||
|
name: OBQA
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 39
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: commonsense
|
||||||
|
name: SIQA
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 35.76
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: commonsense
|
||||||
|
name: PIQA
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 75.35
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: commonsense
|
||||||
|
name: Hellaswag
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 64.92
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: commonsense
|
||||||
|
name: TruthfulQA
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 39.49
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: reading-comprehension
|
||||||
|
name: BoolQ
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 65.44
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: reading-comprehension
|
||||||
|
name: SQuAD 2.0
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 17.78
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: reasoning
|
||||||
|
name: ARC-C
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 38.14
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: reasoning
|
||||||
|
name: GPQA
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 24.41
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: reasoning
|
||||||
|
name: BBH
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 29.84
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: reasoning
|
||||||
|
name: MUSR
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 33.99
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: code
|
||||||
|
name: HumanEval
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 21.95
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: code
|
||||||
|
name: MBPP
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 23.2
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: math
|
||||||
|
name: GSM8K
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 19.26
|
||||||
|
veriefied: false
|
||||||
|
- task:
|
||||||
|
type: text-generation
|
||||||
|
dataset:
|
||||||
|
type: math
|
||||||
|
name: MATH
|
||||||
|
metrics:
|
||||||
|
- name: pass@1
|
||||||
|
type: pass@1
|
||||||
|
value: 8.96
|
||||||
|
veriefied: false
|
||||||
|
new_version: ibm-granite/granite-3.1-1b-a400m-base
|
||||||
|
---
|
||||||
|
|
||||||
granite-3.0-1b-a400m-base
|
<!--  -->
|
||||||
|
<!--  -->
|
||||||
|
|
||||||
|
# Granite-3.0-1B-A400M-Base
|
||||||
|
|
||||||
|
**Model Summary:**
|
||||||
|
Granite-3.0-1B-A400M-Base is a decoder-only language model to support a variety of text-to-text generation tasks. It is trained from scratch following a two-stage training strategy. In the first stage, it is trained on 8 trillion tokens sourced from diverse domains. During the second stage, it is further trained on 2 trillion tokens using a carefully curated mix of high-quality data, aiming to enhance its performance on specific tasks.
|
||||||
|
|
||||||
|
- **Developers:** Granite Team, IBM
|
||||||
|
- **GitHub Repository:** [ibm-granite/granite-3.0-language-models](https://github.com/ibm-granite/granite-3.0-language-models)
|
||||||
|
- **Website**: [Granite Docs](https://www.ibm.com/granite/docs/)
|
||||||
|
- **Paper:** [Granite 3.0 Language Models](https://github.com/ibm-granite/granite-3.0-language-models/blob/main/paper.pdf)
|
||||||
|
- **Release Date**: October 21st, 2024
|
||||||
|
- **License:** [Apache 2.0](https://www.apache.org/licenses/LICENSE-2.0)
|
||||||
|
|
||||||
|
**Supported Languages:**
|
||||||
|
English, German, Spanish, French, Japanese, Portuguese, Arabic, Czech, Italian, Korean, Dutch, and Chinese. Users may finetune Granite 3.0 models for languages beyond these 12 languages.
|
||||||
|
|
||||||
|
**Intended use:**
|
||||||
|
Prominent use cases of LLMs in text-to-text generation include summarization, text classification, extraction, question-answering, and more. All Granite Base models are able to handle these tasks as they were trained on a large amount of data from various domains. Moreover, they can serve as baseline to create specialized models for specific application scenarios.
|
||||||
|
|
||||||
|
**Generation:**
|
||||||
|
This is a simple example of how to use Granite-3.0-1B-A400M-Base model.
|
||||||
|
|
||||||
|
Install the following libraries:
|
||||||
|
|
||||||
|
```shell
|
||||||
|
pip install torch torchvision torchaudio
|
||||||
|
pip install accelerate
|
||||||
|
pip install transformers
|
||||||
|
```
|
||||||
|
Then, copy the code snippet below to run the example.
|
||||||
|
|
||||||
|
```python
|
||||||
|
from transformers import AutoModelForCausalLM, AutoTokenizer
|
||||||
|
device = "auto"
|
||||||
|
model_path = "ibm-granite/granite-3.0-1b-a400m-base"
|
||||||
|
tokenizer = AutoTokenizer.from_pretrained(model_path)
|
||||||
|
# drop device_map if running on CPU
|
||||||
|
model = AutoModelForCausalLM.from_pretrained(model_path, device_map=device)
|
||||||
|
model.eval()
|
||||||
|
# change input text as desired
|
||||||
|
input_text = "Where is the Thomas J. Watson Research Center located?"
|
||||||
|
# tokenize the text
|
||||||
|
input_tokens = tokenizer(input_text, return_tensors="pt").to(device)
|
||||||
|
# generate output tokens
|
||||||
|
output = model.generate(**input_tokens,
|
||||||
|
max_length=4000)
|
||||||
|
# decode output tokens into text
|
||||||
|
output = tokenizer.batch_decode(output)
|
||||||
|
# print output
|
||||||
|
print(output)
|
||||||
|
```
|
||||||
|
|
||||||
|
**Model Architecture:**
|
||||||
|
Granite-3.0-1B-A400M-Base is based on a decoder-only sparse Mixture of Experts (MoE) transformer architecture. Core components of this architecture are: Fine-grained Experts, Dropless Token Routing, and Load Balancing Loss.
|
||||||
|
|
||||||
|
| Model | 2B Dense | 8B Dense | 1B MoE | 3B MoE |
|
||||||
|
| :-------- | :--------| :--------| :-------- | :--------|
|
||||||
|
| Embedding size | 2048 | 4096 | **1024** | 1536 |
|
||||||
|
| Number of layers | 40 | 40 | **24** | 32 |
|
||||||
|
| Attention head size | 64 | 128 | **64** | 64 |
|
||||||
|
| Number of attention heads | 32 | 32 | **16** | 24 |
|
||||||
|
| Number of KV heads | 8 | 8 | **8** | 8 |
|
||||||
|
| MLP hidden size | 8192 | 12800 | **512** | 512 |
|
||||||
|
| MLP activation | SwiGLU | SwiGLU | **SwiGLU** | SwiGLU |
|
||||||
|
| Number of Experts | — | — | **32** | 40 |
|
||||||
|
| MoE TopK | — | — | **8** | 8 |
|
||||||
|
| Initialization std | 0.1 | 0.1 | **0.1** | 0.1 |
|
||||||
|
| Sequence Length | 4096 | 4096 | **4096** | 4096 |
|
||||||
|
| Position Embedding | RoPE | RoPE | **RoPE** | RoPE |
|
||||||
|
| # Parameters | 2.5B | 8.1B | **1.3B** | 3.3B |
|
||||||
|
| # Active Parameters | 2.5B | 8.1B | **400M** | 800M |
|
||||||
|
| # Training tokens | 12T | 12T | **10T** | 10T |
|
||||||
|
|
||||||
|
**Training Data:**
|
||||||
|
This model is trained on a mix of open source and proprietary data following a two-stage training strategy.
|
||||||
|
* Stage 1 data: The data for stage 1 is sourced from diverse domains, such as: web, code, academic sources, books, and math data.
|
||||||
|
* Stage 2 data: The data for stage 2 comprises a curated mix of high-quality data from the same domains, plus multilingual and instruction data. The goal of this second training phase is to enhance the model’s performance on specific tasks.
|
||||||
|
|
||||||
|
A detailed attribution of datasets can be found in the [Granite Technical Report](https://github.com/ibm-granite/granite-3.0-language-models/blob/main/paper.pdf) and [Accompanying Author List](https://github.com/ibm-granite/granite-3.0-language-models/blob/main/author-ack.pdf).
|
||||||
|
|
||||||
|
**Infrastructure:**
|
||||||
|
We train Granite 3.0 Language Models using IBM's super computing cluster, Blue Vela, which is outfitted with NVIDIA H100 GPUs. This cluster provides a scalable and efficient infrastructure for training our models over thousands of GPUs while minimizing environmental impact by utilizing 100% renewable energy sources.
|
||||||
|
|
||||||
|
**Ethical Considerations and Limitations:**
|
||||||
|
The use of Large Language Models involves risks and ethical considerations people must be aware of, including but not limited to: bias and fairness, misinformation, and autonomous decision-making. Granite-3.0-1B-A400M-Base model is not the exception in this regard. Even though this model is suited for multiple generative AI tasks, it has not undergone any safety alignment, there it may produce problematic outputs. Additionally, it remains uncertain whether smaller models might exhibit increased susceptibility to hallucination in generation scenarios by copying text verbatim from the training dataset due to their reduced sizes and memorization capacities. This aspect is currently an active area of research, and we anticipate more rigorous exploration, comprehension, and mitigations in this domain. Regarding ethics, a latent risk associated with all Large Language Models is their malicious utilization. We urge the community to use Granite-3.0-1B-A400M-Base model with ethical intentions and in a responsible way.
|
||||||
|
|
||||||
|
**Resources**
|
||||||
|
- ⭐️ Learn about the latest updates with Granite: https://www.ibm.com/granite
|
||||||
|
- 📄 Get started with tutorials, best practices, and prompt engineering advice: https://www.ibm.com/granite/docs/
|
||||||
|
- 💡 Learn about the latest Granite learning resources: https://ibm.biz/granite-learning-resources
|
||||||
|
|
||||||
|
<!-- ## Citation
|
||||||
|
```
|
||||||
|
@misc{granite-models,
|
||||||
|
author = {author 1, author2, ...},
|
||||||
|
title = {},
|
||||||
|
journal = {},
|
||||||
|
volume = {},
|
||||||
|
year = {2024},
|
||||||
|
url = {https://arxiv.org/abs/0000.00000},
|
||||||
|
}
|
||||||
|
``` -->
|
||||||
|
|
@ -0,0 +1,34 @@
|
||||||
|
{
|
||||||
|
"architectures": [
|
||||||
|
"GraniteMoeForCausalLM"
|
||||||
|
],
|
||||||
|
"attention_bias": false,
|
||||||
|
"attention_dropout": 0.0,
|
||||||
|
"attention_multiplier": 0.015625,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"embedding_multiplier": 12.0,
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"hidden_act": "silu",
|
||||||
|
"hidden_size": 1024,
|
||||||
|
"initializer_range": 0.02,
|
||||||
|
"intermediate_size": 512,
|
||||||
|
"logits_scaling": 6.0,
|
||||||
|
"max_position_embeddings": 4096,
|
||||||
|
"model_type": "granitemoe",
|
||||||
|
"num_attention_heads": 16,
|
||||||
|
"num_experts_per_tok": 8,
|
||||||
|
"num_hidden_layers": 24,
|
||||||
|
"num_key_value_heads": 8,
|
||||||
|
"num_local_experts": 32,
|
||||||
|
"output_router_logits": false,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"residual_multiplier": 0.22,
|
||||||
|
"rms_norm_eps": 1e-06,
|
||||||
|
"rope_scaling": null,
|
||||||
|
"rope_theta": 10000,
|
||||||
|
"router_aux_loss_coef": 0.001,
|
||||||
|
"tie_word_embeddings": true,
|
||||||
|
"transformers_version": "4.45.0.dev0",
|
||||||
|
"use_cache": true,
|
||||||
|
"vocab_size": 49152
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,7 @@
|
||||||
|
{
|
||||||
|
"_from_model_config": true,
|
||||||
|
"bos_token_id": 0,
|
||||||
|
"eos_token_id": 0,
|
||||||
|
"pad_token_id": 0,
|
||||||
|
"transformers_version": "4.45.0.dev0"
|
||||||
|
}
|
||||||
Binary file not shown.
Binary file not shown.
|
|
@ -0,0 +1,226 @@
|
||||||
|
{
|
||||||
|
"metadata": {
|
||||||
|
"total_size": 5539827712
|
||||||
|
},
|
||||||
|
"weight_map": {
|
||||||
|
"model.embed_tokens.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.norm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"lm_head.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.block_sparse_moe.input_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.block_sparse_moe.output_linear.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.21.block_sparse_moe.router.layer.weight": "model-00001-of-00002.safetensors",
|
||||||
|
"model.layers.21.block_sparse_moe.input_linear.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.21.block_sparse_moe.output_linear.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.21.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.21.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.21.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.21.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.block_sparse_moe.router.layer.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.block_sparse_moe.input_linear.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.block_sparse_moe.output_linear.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.22.self_attn.o_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.input_layernorm.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.post_attention_layernorm.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.block_sparse_moe.router.layer.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.block_sparse_moe.input_linear.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.block_sparse_moe.output_linear.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.self_attn.q_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.self_attn.k_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.self_attn.v_proj.weight": "model-00002-of-00002.safetensors",
|
||||||
|
"model.layers.23.self_attn.o_proj.weight": "model-00002-of-00002.safetensors"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,51 @@
|
||||||
|
{
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|endoftext|>",
|
||||||
|
"<fim_prefix>",
|
||||||
|
"<fim_middle>",
|
||||||
|
"<fim_suffix>",
|
||||||
|
"<fim_pad>",
|
||||||
|
"<filename>",
|
||||||
|
"<gh_stars>",
|
||||||
|
"<issue_start>",
|
||||||
|
"<issue_comment>",
|
||||||
|
"<issue_closed>",
|
||||||
|
"<jupyter_start>",
|
||||||
|
"<jupyter_text>",
|
||||||
|
"<jupyter_code>",
|
||||||
|
"<jupyter_output>",
|
||||||
|
"<empty_output>",
|
||||||
|
"<commit_before>",
|
||||||
|
"<commit_msg>",
|
||||||
|
"<commit_after>",
|
||||||
|
"<reponame>"
|
||||||
|
],
|
||||||
|
"bos_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"eos_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"pad_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
},
|
||||||
|
"unk_token": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
|
|
@ -0,0 +1,187 @@
|
||||||
|
{
|
||||||
|
"add_prefix_space": false,
|
||||||
|
"added_tokens_decoder": {
|
||||||
|
"0": {
|
||||||
|
"content": "<|endoftext|>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"1": {
|
||||||
|
"content": "<fim_prefix>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"2": {
|
||||||
|
"content": "<fim_middle>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"3": {
|
||||||
|
"content": "<fim_suffix>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"4": {
|
||||||
|
"content": "<fim_pad>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"5": {
|
||||||
|
"content": "<filename>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"6": {
|
||||||
|
"content": "<gh_stars>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"7": {
|
||||||
|
"content": "<issue_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"8": {
|
||||||
|
"content": "<issue_comment>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"9": {
|
||||||
|
"content": "<issue_closed>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"10": {
|
||||||
|
"content": "<jupyter_start>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"11": {
|
||||||
|
"content": "<jupyter_text>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"12": {
|
||||||
|
"content": "<jupyter_code>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"13": {
|
||||||
|
"content": "<jupyter_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"14": {
|
||||||
|
"content": "<empty_output>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"15": {
|
||||||
|
"content": "<commit_before>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"16": {
|
||||||
|
"content": "<commit_msg>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"17": {
|
||||||
|
"content": "<commit_after>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
},
|
||||||
|
"18": {
|
||||||
|
"content": "<reponame>",
|
||||||
|
"lstrip": false,
|
||||||
|
"normalized": false,
|
||||||
|
"rstrip": false,
|
||||||
|
"single_word": false,
|
||||||
|
"special": true
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"additional_special_tokens": [
|
||||||
|
"<|endoftext|>",
|
||||||
|
"<fim_prefix>",
|
||||||
|
"<fim_middle>",
|
||||||
|
"<fim_suffix>",
|
||||||
|
"<fim_pad>",
|
||||||
|
"<filename>",
|
||||||
|
"<gh_stars>",
|
||||||
|
"<issue_start>",
|
||||||
|
"<issue_comment>",
|
||||||
|
"<issue_closed>",
|
||||||
|
"<jupyter_start>",
|
||||||
|
"<jupyter_text>",
|
||||||
|
"<jupyter_code>",
|
||||||
|
"<jupyter_output>",
|
||||||
|
"<empty_output>",
|
||||||
|
"<commit_before>",
|
||||||
|
"<commit_msg>",
|
||||||
|
"<commit_after>",
|
||||||
|
"<reponame>"
|
||||||
|
],
|
||||||
|
"bos_token": "<|endoftext|>",
|
||||||
|
"clean_up_tokenization_spaces": true,
|
||||||
|
"eos_token": "<|endoftext|>",
|
||||||
|
"model_max_length": 9223372036854775807,
|
||||||
|
"pad_token": "<|endoftext|>",
|
||||||
|
"padding_side": "left",
|
||||||
|
"tokenizer_class": "GPT2Tokenizer",
|
||||||
|
"unk_token": "<|endoftext|>",
|
||||||
|
"vocab_size": 49152
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue