pice35408784b54431987c4d13c457b9cd
/
uform-gen2-qwen-500m_a14046212370067456393158
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

first commit

This commit is contained in:
YYJ-aaaa 2024-12-25 16:06:43 +08:00
parent 2a7ef30729
commit 649c41598c
11 changed files with 1590 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
cat.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 139 KiB

28
config.json Normal file
View File

@ -0,0 +1,28 @@
{
"_name_or_path": "../weights/vlm-qwen-big-uform",
"architectures": [
"VLMForCausalLM"
],
"auto_map": {
"AutoConfig": "configuration_uform_gen.VLMConfig",
"AutoModel": "modeling_uform_gen.VLMForCausalLM",
"AutoProcessor": "processing_uform_gen.VLMProcessor"
},
"image_encoder_hidden_size": 1280,
"image_encoder_name_or_path": "unum-cloud/uform-vl-english-big",
"image_encoder_num_heads": 16,
"image_encoder_num_layers": 32,
"image_encoder_patch_size": 14,
"image_encoder_pooling": "cls",
"image_pooler_intermediate_size": 3200,
"image_pooler_num_attn_heads": 16,
"image_size": 336,
"image_token_id": 151646,
"initializer_range": 0.02,
"model_type": "vlm",
"num_image_latents": 256,
"text_decoder_name_or_path": "Qwen/Qwen1.5-0.5B-Chat",
"torch_dtype": "float32",
"transformers_version": "4.37.2",
"use_cache": true
}

43
configuration_uform_gen.py Normal file
View File

@ -0,0 +1,43 @@
from transformers.configuration_utils import PretrainedConfig
from typing import List
class VLMConfig(PretrainedConfig):
model_type = "vlm"
def __init__(
self,
text_decoder_name_or_path: str = "",
image_encoder_name_or_path: str = "",
image_size: int = 336,
image_pooler_num_attn_heads: int = 16,
image_pooler_intermediate_size: int = 3200,
image_token_id: int = 151646,
image_encoder_hidden_size: int = 1280,
image_encoder_patch_size: int = 14,
image_encoder_num_layers: int = 32,
image_encoder_num_heads: int = 16,
image_encoder_pooling: str = "cls",
num_image_latents: int = 256,
initializer_range: float = 0.02,
use_cache: bool = True,
**kwargs,
):
self.text_decoder_name_or_path = text_decoder_name_or_path
self.image_encoder_name_or_path = image_encoder_name_or_path
self.image_pooler_num_attn_heads = image_pooler_num_attn_heads
self.image_pooler_intermediate_size = image_pooler_intermediate_size
self.image_token_id = image_token_id
self.image_size = image_size
self.image_encoder_hidden_size = image_encoder_hidden_size
self.image_encoder_patch_size = image_encoder_patch_size
self.image_encoder_num_layers = image_encoder_num_layers
self.image_encoder_num_heads = image_encoder_num_heads
self.image_encoder_pooling = image_encoder_pooling
self.num_image_latents = num_image_latents
self.initializer_range = initializer_range
self.use_cache = use_cache
super().__init__(**kwargs)

4
generation_config.json Normal file
View File

@ -0,0 +1,4 @@
{
"_from_model_config": true,
"transformers_version": "4.37.2"
}

BIN
interior.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 93 KiB

BIN
model-00001-of-00002.safetensors (Stored with Git LFS) Normal file
View File

Binary file not shown.

BIN
model-00002-of-00002.safetensors (Stored with Git LFS) Normal file
View File

Binary file not shown.

901
model.safetensors.index.json Normal file
View File

@ -0,0 +1,901 @@
{
"metadata": {
"total_size": 5093506560
},
"weight_map": {
"image_encoder.blocks.0.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.0.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.1.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.10.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.11.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.12.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.13.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.14.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.15.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.16.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.17.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.18.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.19.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.2.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.20.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.21.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.22.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.23.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.24.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.25.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.26.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.27.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.28.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.29.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.3.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.30.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.ls2.weight": "model-00002-of-00002.safetensors",
"image_encoder.blocks.31.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.mlp.output_layer.bias": "model-00002-of-00002.safetensors",
"image_encoder.blocks.31.mlp.output_layer.weight": "model-00002-of-00002.safetensors",
"image_encoder.blocks.31.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.31.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.4.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.5.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.6.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.7.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.8.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.key.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.key.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.out.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.out.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.query.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.query.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.value.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.attn.value.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.ls1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.ls2.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.mlp.hidden_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.mlp.hidden_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.mlp.output_layer.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.mlp.output_layer.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.norm1.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.norm1.weight": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.norm2.bias": "model-00001-of-00002.safetensors",
"image_encoder.blocks.9.norm2.weight": "model-00001-of-00002.safetensors",
"image_encoder.cls_token": "model-00001-of-00002.safetensors",
"image_encoder.norm.bias": "model-00002-of-00002.safetensors",
"image_encoder.norm.weight": "model-00002-of-00002.safetensors",
"image_encoder.patch_embed.bias": "model-00001-of-00002.safetensors",
"image_encoder.patch_embed.weight": "model-00001-of-00002.safetensors",
"image_encoder.pos_embed": "model-00001-of-00002.safetensors",
"image_pooler.image_latents": "model-00002-of-00002.safetensors",
"image_pooler.pooler.linear1.bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.linear1.weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.linear2.bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.linear2.weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.multihead_attn.in_proj_bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.multihead_attn.in_proj_weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.multihead_attn.out_proj.bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.multihead_attn.out_proj.weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.norm1.bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.norm1.weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.norm2.bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.norm2.weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.norm3.bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.norm3.weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.self_attn.in_proj_bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.self_attn.in_proj_weight": "model-00002-of-00002.safetensors",
"image_pooler.pooler.self_attn.out_proj.bias": "model-00002-of-00002.safetensors",
"image_pooler.pooler.self_attn.out_proj.weight": "model-00002-of-00002.safetensors",
"image_pooler.projection.bias": "model-00002-of-00002.safetensors",
"image_pooler.projection.weight": "model-00002-of-00002.safetensors",
"text_decoder.lm_head.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.embed_tokens.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.0.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.1.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.10.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.11.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.12.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.13.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.14.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.15.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.16.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.17.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.18.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.19.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.2.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.20.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.21.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.22.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.23.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.3.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.4.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.5.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.6.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.7.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.8.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.input_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.post_attention_layernorm.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.self_attn.k_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.self_attn.k_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.self_attn.o_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.self_attn.q_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.self_attn.q_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.self_attn.v_proj.bias": "model-00001-of-00002.safetensors",
"text_decoder.model.layers.9.self_attn.v_proj.weight": "model-00001-of-00002.safetensors",
"text_decoder.model.norm.weight": "model-00001-of-00002.safetensors"
}
}

245
modeling_uform_gen.py Normal file
View File

@ -0,0 +1,245 @@
from typing import List, Optional, Tuple, Union
from .configuration_uform_gen import VLMConfig
import torch
import torch.nn.functional as F
from torch.utils.checkpoint import checkpoint
from torch import nn
from transformers.modeling_outputs import CausalLMOutputWithPast
from transformers.modeling_utils import PreTrainedModel
from transformers.models.auto.modeling_auto import AutoModelForCausalLM, AutoModel
from transformers import AutoConfig
from transformers.utils import logging
from .vision_encoder import VisionEncoder
class ImageFeaturesPooler(nn.Module):
def __init__(self, config, text_config):
super().__init__()
self.pooler = nn.TransformerDecoderLayer(
config.image_encoder_hidden_size,
config.image_pooler_num_attn_heads,
config.image_pooler_intermediate_size,
activation=nn.functional.silu,
batch_first=True,
norm_first=True,
)
self.image_latents = nn.Parameter(
torch.randn(1, config.num_image_latents, config.image_encoder_hidden_size)
* config.initializer_range**0.5
)
self.projection = nn.Linear(config.image_encoder_hidden_size, text_config.hidden_size)
def forward(self, features):
features = self.pooler(
self.image_latents.expand(features.size(0), -1, -1), features
)
return self.projection(features)
class VLMPreTrainedModel(PreTrainedModel):
config_class = VLMConfig
base_model_prefix = "vlm"
supports_gradient_checkpointing = True
_no_split_modules = []
_skip_keys_device_placement = "past_key_values"
def _init_weights(self, module):
pass
def _initialize_weights(self, module):
pass
class VLMForCausalLM(VLMPreTrainedModel):
def __init__(self, config: VLMConfig):
super().__init__(config)
self.config = config
self.text_config = AutoConfig.from_pretrained(
config.text_decoder_name_or_path,
trust_remote_code=True
)
self.text_decoder = AutoModelForCausalLM.from_config(
self.text_config,
trust_remote_code=True
)
self.image_encoder = VisionEncoder(
config.image_encoder_hidden_size,
config.image_encoder_patch_size,
config.image_encoder_num_layers,
config.image_encoder_num_heads,
)
self.image_pooler = ImageFeaturesPooler(config, self.text_config)
def get_input_embeddings(self):
return self.text_decoder.get_input_embeddings()
def set_input_embeddings(self, value):
self.text_decoder.set_input_embeddings(value)
def get_images_embeddings(self, images):
features = self.image_encoder(images)
return self.image_pooler(features)
def gather_continuous_embeddings(
self,
input_ids: torch.Tensor,
word_embeddings: torch.Tensor,
image_embeddings: torch.Tensor
) -> torch.Tensor:
start_indices = (input_ids == self.config.image_token_id).nonzero()[:, 1]
embeddings = []
for sample_idx, start_idx in enumerate(start_indices.tolist()):
embeddings.append(
torch.cat(
(
word_embeddings[sample_idx, :start_idx],
image_embeddings[sample_idx],
word_embeddings[sample_idx, start_idx + 1 :],
),
dim=0,
)
)
return torch.stack(embeddings, dim=0)
def forward(
self,
input_ids: torch.LongTensor = None,
images: torch.Tensor = None,
attention_mask: Optional[torch.Tensor] = None,
position_ids: Optional[torch.LongTensor] = None,
past_key_values: Optional[List[torch.FloatTensor]] = None,
inputs_embeds: Optional[torch.FloatTensor] = None,
use_cache: Optional[bool] = None,
labels: Optional[torch.Tensor] = None,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
return_dict: Optional[bool] = None
) -> Union[dict, Tuple, CausalLMOutputWithPast]:
output_attentions = (
output_attentions
if output_attentions is not None
else self.config.output_attentions
)
output_hidden_states = (
output_hidden_states
if output_hidden_states is not None
else self.config.output_hidden_states
)
use_cache = use_cache if use_cache is not None else self.config.use_cache
return_dict = (
return_dict if return_dict is not None else self.config.use_return_dict
)
if input_ids is not None and inputs_embeds is not None:
raise ValueError(
"You cannot specify both input_ids and inputs_embeds at the same time"
)
elif input_ids is None and inputs_embeds is None:
raise ValueError("You have to specify either input_is or inputs_embeds")
if inputs_embeds is None and past_key_values is None:
inputs_embeds = self.get_input_embeddings()(input_ids)
if images is not None:
image_embeds = self.get_images_embeddings(images)
inputs_embeds = self.gather_continuous_embeddings(
input_ids,
inputs_embeds,
image_embeds
)
if position_ids is None:
seq_length = (
inputs_embeds.shape[1]
if inputs_embeds is not None
else input_ids.shape[1]
)
past_key_values_length = 0
if past_key_values is not None:
past_key_values_length = past_key_values[0][0].shape[2]
device = input_ids.device if input_ids is not None else inputs_embeds.device
position_ids = torch.arange(
past_key_values_length,
seq_length + past_key_values_length,
dtype=torch.long,
device=device,
)
position_ids = position_ids.unsqueeze(0)
outputs = self.text_decoder(
inputs_embeds=inputs_embeds,
input_ids=input_ids if past_key_values is not None else None,
attention_mask=attention_mask,
position_ids=position_ids,
past_key_values=past_key_values,
output_attentions=output_attentions,
output_hidden_states=output_hidden_states,
use_cache=use_cache,
return_dict=return_dict,
)
return outputs
def prepare_inputs_for_generation(
self,
input_ids,
images=None,
past_key_values=None,
attention_mask=None,
inputs_embeds=None,
**kwargs,
):
if past_key_values:
input_ids = input_ids[:, -1:]
position_ids = kwargs.get("position_ids", None)
if attention_mask is not None and position_ids is None:
# create position_ids on the fly for batch generation
position_ids = attention_mask.long().cumsum(-1) - 1
position_ids.masked_fill_(attention_mask == 0, 1)
if past_key_values:
position_ids = position_ids[:, -1].unsqueeze(-1)
# if `inputs_embeds` are passed, we only want to use them in the 1st generation step
if inputs_embeds is not None and past_key_values is None:
model_inputs = {"inputs_embeds": inputs_embeds}
n_samples = inputs_embeds.shape[0]
else:
model_inputs = {"input_ids": input_ids}
n_samples = input_ids.shape[0]
if images is not None:
model_inputs["images"] = images
model_inputs.update(
{
"position_ids": position_ids,
"past_key_values": past_key_values,
"use_cache": kwargs.get("use_cache"),
"attention_mask": attention_mask,
"images": images if past_key_values is None else None,
}
)
return model_inputs
@classmethod
def from_config(cls, config, **kwargs):
return cls._from_config(config, **kwargs)
VLMConfig.register_for_auto_class()
VLMForCausalLM.register_for_auto_class("AutoModel")

181
processing_uform_gen.py Normal file
View File

@ -0,0 +1,181 @@
from functools import partial
import torch
import torch.nn.functional as F
from transformers.processing_utils import ProcessorMixin
from transformers.image_processing_utils import BaseImageProcessor
from transformers import AutoTokenizer, AutoConfig
from transformers import BatchFeature
from PIL import Image
from torchvision.transforms import (
Compose,
Normalize,
Resize,
ToTensor
)
IMAGENET_MEAN = (0.48145466, 0.4578275, 0.40821073)
IMAGENET_STD = (0.26862954, 0.26130258, 0.27577711)
def convert_to_rgb(x):
return x.convert("RGB")
def expand2square(image, background_color):
width, height = image.size
if width == height:
return image
elif width > height:
result = Image.new(image.mode, (width, width), background_color)
result.paste(image, (0, (width - height) // 2))
return result
else:
result = Image.new(image.mode, (height, height), background_color)
result.paste(image, ((height - width) // 2, 0))
return result
class ImageProcessor(BaseImageProcessor):
def __init__(
self,
image_size: int,
**kwargs
):
super().__init__(**kwargs)
self.transform = Compose(
[
convert_to_rgb,
partial(
expand2square,
background_color=tuple(int(255 * v) for v in IMAGENET_MEAN)
),
Resize(image_size),
ToTensor(),
Normalize(
mean=IMAGENET_MEAN,
std=IMAGENET_STD,
),
]
)
def preprocess(
self,
image: Image
):
return self.transform(image)
def __repr__(self):
return repr(self.transform)
class VLMProcessor(ProcessorMixin):
def __init__(self, config):
self.config = config
self.image_size = config.image_size
self.feature_extractor = ImageProcessor(self.image_size)
self.tokenizer = AutoTokenizer.from_pretrained(
config.text_decoder_name_or_path, additional_special_tokens=["<image>"]
)
self.tokenizer.chat_template = "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}"
self.num_image_latents = config.num_image_latents
# super().__init__(self.image_processor, self.tokenizer)
def __call__(
self, text=None, images=None, **kwargs
):
if text is not None:
if isinstance(text, str):
text = [text]
tokenized_texts = []
for t in text:
messages = [
{"role": "system", "content": "You are a helpful assistant."},
{"role": "user", "content": f" <image> {t}"},
]
tokenized_prompt = self.tokenizer.apply_chat_template(
messages, add_generation_prompt=True, return_tensors="pt"
)
tokenized_texts.append(tokenized_prompt)
max_len = max(len(t[0]) for t in tokenized_texts)
input_ids = torch.full(
(len(tokenized_texts), max_len),
fill_value=self.tokenizer.pad_token_id,
dtype=torch.int64,
)
attention_mask = torch.full(
(len(tokenized_texts), max_len), fill_value=0, dtype=torch.int64
)
for i, tokens in enumerate(tokenized_texts):
input_ids[i, -len(tokens[0]) :] = tokens[0]
attention_mask[i, -len(tokens[0]) :] = 1
attention_mask = F.pad(
attention_mask, pad=(0, self.num_image_latents - 1), value=1
)
encoding = BatchFeature(
data={"input_ids": input_ids, "attention_mask": attention_mask}
)
if images is not None:
if isinstance(images, (list, tuple)):
image_features = torch.empty(
(len(images), 3, self.image_size , self.image_size),
dtype=torch.float32,
)
for i, image in enumerate(images):
image_features[i] = self.feature_extractor(image)
else:
image_features = self.feature_extractor(images).unsqueeze(0)
if text is not None and images is not None:
encoding["images"] = image_features
return encoding
elif text is not None:
return encoding
else:
return BatchFeature(
data={
"images": image_features,
},
tensor_type=return_tensors,
)
def batch_decode(self, *args, **kwargs):
"""
This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
refer to the docstring of this method for more information.
"""
return self.tokenizer.batch_decode(*args, **kwargs)
def decode(self, *args, **kwargs):
"""
This method forwards all its arguments to CLIPTokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
the docstring of this method for more information.
"""
return self.tokenizer.decode(*args, **kwargs)
@classmethod
def from_pretrained(
cls,
pretrained_model_name_or_path,
trust_remote_code=False,
**kwargs
):
config = AutoConfig.from_pretrained(
pretrained_model_name_or_path,
trust_remote_code=trust_remote_code
)
return cls(config)

182
vision_encoder.py Normal file
View File

@ -0,0 +1,182 @@
import torch.nn as nn
import torch.nn.functional as F
import torch
from torch import Tensor
from typing import Optional
class Attention(nn.Module):
def __init__(
self,
dim: int,
num_heads: int,
dropout_prob: float = 0
):
super().__init__()
self.use_sdp = int(torch.__version__[0]) > 1
self.query = nn.Linear(dim, dim)
self.key = nn.Linear(dim, dim)
self.value = nn.Linear(dim, dim)
self.out = nn.Linear(dim, dim)
self.dropout_prob = dropout_prob
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.scale = self.head_dim**-0.5
def forward(
self,
x: Tensor,
attn_mask: Optional[Tensor] = None,
context: Optional[Tensor] = None,
is_causal: bool = False,
) -> Tensor:
query = self.reshape(self.query(x))
key = self.reshape(self.key(x if context is None else context))
value = self.reshape(self.value(x if context is None else context))
if self.use_sdp:
x = F.scaled_dot_product_attention(
query,
key,
value,
attn_mask,
dropout_p=self.dropout_prob if self.training else 0,
is_causal=is_causal,
)
else:
attn = query @ key.transpose(-2, -1) * self.scale
if attn_mask is not None:
attn += attn_mask
attn = attn.softmax(dim=-1)
x = attn @ value
return self.out(x.transpose(2, 1).flatten(2))
def reshape(self, x: Tensor) -> Tensor:
batch_size, seq_len, _ = x.shape
x = x.view(batch_size, seq_len, self.num_heads, self.head_dim)
return x.transpose(2, 1)
class MLP(nn.Module):
def __init__(
self,
dim: int,
dim_expand_factor: int = 4
):
super().__init__()
self.hidden_layer = nn.Linear(dim, dim * dim_expand_factor)
self.output_layer = nn.Linear(dim * dim_expand_factor, dim)
def forward(self, x: Tensor) -> Tensor:
x = F.gelu(self.hidden_layer(x))
return self.output_layer(x)
class LayerScale(nn.Module):
def __init__(
self,
dim: int,
init_values: float = 1e-5,
inplace: bool = False
):
super().__init__()
self.weight = nn.Parameter(init_values * torch.ones(dim))
self.inplace = inplace
def forward(self, x: Tensor) -> Tensor:
return x.mul_(self.weight) if self.inplace else x * self.weight
class VisionEncoderBlock(nn.Module):
def __init__(
self,
dim: int,
num_heads: int
):
super().__init__()
self.norm1 = nn.LayerNorm(dim, eps=1e-6)
self.attn = Attention(dim, num_heads)
self.ls1 = LayerScale(dim)
self.norm2 = nn.LayerNorm(dim, eps=1e-6)
self.mlp = MLP(dim)
self.ls2 = LayerScale(dim)
def forward(self, x: Tensor) -> Tensor:
x = x + self.ls1(self.attn(self.norm1(x)))
x = x + self.ls2(self.mlp(self.norm2(x)))
return x
class VisionEncoder(nn.Module):
def __init__(
self,
dim: int,
patch_size: int,
num_layers: int,
num_heads: int,
):
super().__init__()
self.n_patch = 224 // patch_size
self.seq_len = self.n_patch ** 2
self.patch_size = patch_size
self.patch_embed = nn.Conv2d(3, dim, patch_size, patch_size)
self.pos_embed = nn.Parameter(torch.randn(1, self.seq_len, dim) * 0.02)
self.cls_token = nn.Parameter(torch.zeros(1, 1, dim))
self.interpolate_offset = 0.1
self.interpolate_antialias = False
self.blocks = nn.Sequential(
*[
VisionEncoderBlock(dim, num_heads)
for _ in range(num_layers)
]
)
self.norm = nn.LayerNorm(dim, eps=1e-6)
def interpolate_pos_encoding(self, x, h, w):
previous_dtype = x.dtype
if x.shape[1] == self.seq_len and w == h:
return self.pos_embed
pos_embed = self.pos_embed.float()
dim = x.shape[-1]
w0 = w // self.patch_size
h0 = h // self.patch_size
# we add a small number to avoid floating point error in the interpolation
# see discussion at https://github.com/facebookresearch/dino/issues/8
w0, h0 = w0 + self.interpolate_offset, h0 + self.interpolate_offset
sx, sy = float(w0) / self.n_patch, float(h0) / self.n_patch
pos_embed = nn.functional.interpolate(
pos_embed.reshape(1, self.n_patch, self.n_patch, dim).permute(0, 3, 1, 2),
scale_factor=(sy, sx),
mode="bicubic",
antialias=self.interpolate_antialias,
)
return pos_embed.to(previous_dtype).flatten(start_dim=2).transpose(2, 1)
def forward(self, x: Tensor) -> Tensor:
h, w = x.shape[2:]
x = self.patch_embed(x).flatten(start_dim=2).transpose(2, 1)
x = x + self.interpolate_pos_encoding(x, h, w)
x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)
x = self.blocks(x)
return self.norm(x)