113 lines
4.0 KiB
Python
113 lines
4.0 KiB
Python
# coding=utf-8
|
|
# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
|
|
#
|
|
# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
|
|
# and OPT implementations in this library. It has been modified from its
|
|
# original forms to accommodate minor architectural differences compared
|
|
# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
""" MiniCPM model configuration"""
|
|
import os
|
|
from typing import Union
|
|
|
|
from transformers.utils import logging
|
|
from transformers import LlamaConfig, PretrainedConfig
|
|
from transformers.models.idefics2.modeling_idefics2 import Idefics2VisionConfig
|
|
|
|
logger = logging.get_logger(__name__)
|
|
|
|
|
|
class MiniCPMVSliceConfig(PretrainedConfig):
|
|
model_type = "minicpmv"
|
|
|
|
def __init__(
|
|
self,
|
|
patch_size=14,
|
|
max_slice_nums=9,
|
|
scale_resolution=448,
|
|
**kwargs,
|
|
):
|
|
super().__init__(**kwargs)
|
|
self.patch_size = patch_size
|
|
self.max_slice_nums = max_slice_nums
|
|
self.scale_resolution = scale_resolution
|
|
|
|
@classmethod
|
|
def from_pretrained(cls, pretrained_model_name_or_path: Union[str, os.PathLike], **kwargs) -> "PretrainedConfig":
|
|
cls._set_token_in_kwargs(kwargs)
|
|
|
|
config_dict, kwargs = cls.get_config_dict(pretrained_model_name_or_path, **kwargs)
|
|
|
|
if config_dict.get("model_type") == "minicpmv":
|
|
config_dict = config_dict["slice_config"]
|
|
|
|
if "model_type" in config_dict and hasattr(cls, "model_type") and config_dict["model_type"] != cls.model_type:
|
|
logger.warning(
|
|
f"You are using a model of type {config_dict['model_type']} to instantiate a model of type "
|
|
f"{cls.model_type}. This is not supported for all configurations of models and can yield errors."
|
|
)
|
|
|
|
return cls.from_dict(config_dict, **kwargs)
|
|
|
|
|
|
|
|
class MiniCPMVConfig(LlamaConfig):
|
|
model_type = "minicpmv"
|
|
keys_to_ignore_at_inference = ["past_key_values"]
|
|
|
|
default_vision_config = {
|
|
"hidden_size": 1152,
|
|
"image_size": 980,
|
|
"intermediate_size": 4304,
|
|
"model_type": "idefics2",
|
|
"num_attention_heads": 16,
|
|
"num_hidden_layers": 27,
|
|
"patch_size": 14,
|
|
}
|
|
|
|
def __init__(
|
|
self,
|
|
use_cache=True,
|
|
query_num=64,
|
|
image_size=448,
|
|
drop_vision_last_layer=True,
|
|
batch_vision_input=True,
|
|
slice_config=None,
|
|
vision_config=None,
|
|
**kwargs,
|
|
):
|
|
self.use_cache = use_cache
|
|
self.query_num = query_num
|
|
self.image_size = image_size
|
|
self.drop_vision_last_layer = drop_vision_last_layer
|
|
self.batch_vision_input = batch_vision_input
|
|
|
|
if slice_config is None:
|
|
self.slice_config = MiniCPMVSliceConfig(max_slice_nums=1)
|
|
else:
|
|
self.slice_config = MiniCPMVSliceConfig(**slice_config)
|
|
self.slice_mode = True
|
|
|
|
# same as HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit
|
|
if vision_config is None:
|
|
self.vision_config = Idefics2VisionConfig(**self.default_vision_config)
|
|
logger.info("vision_config is None, using default vision config")
|
|
elif isinstance(vision_config, dict):
|
|
self.vision_config = Idefics2VisionConfig(**vision_config)
|
|
elif isinstance(vision_config, Idefics2VisionConfig):
|
|
self.vision_config = vision_config
|
|
|
|
self.patch_size = self.vision_config.patch_size
|
|
|
|
super().__init__(**kwargs) |