|
from transformers import PretrainedConfig |
|
from typing import List |
|
|
|
from transformers import Qwen2Config, CLIPVisionConfig |
|
|
|
class InfMLLMUnifiedHDChatConfig(PretrainedConfig): |
|
def __init__( |
|
self, |
|
vison_config=None, |
|
lm_config=None, |
|
lm_model="", |
|
lm_tokenizer="", |
|
lora_modules="", |
|
lora_llm=False, |
|
lora_r=128, |
|
lora_alpha=256, |
|
lora_dropout=0, |
|
|
|
encoder_img="", |
|
image_size_img=336, |
|
lora_encoder_img=False, |
|
hd_num=9, |
|
|
|
encoder_video="", |
|
|
|
max_txt_len=4096, |
|
conv_style='qwen-7b-chat', |
|
precision="bf16", |
|
**kwargs |
|
): |
|
self.lm_model = lm_model |
|
self.lm_tokenizer = lm_tokenizer |
|
self.lora_modules = lora_modules |
|
self.lora_llm = lora_llm |
|
self.lora_r = lora_r |
|
self.lora_alpha = lora_alpha |
|
self.lora_dropout = lora_dropout |
|
|
|
self.encoder_img = encoder_img |
|
self.image_size_img = image_size_img |
|
self.lora_encoder_img = lora_encoder_img |
|
self.hd_num = hd_num |
|
|
|
self.encoder_video = encoder_video |
|
|
|
self.max_txt_len = max_txt_len |
|
self.conv_style = conv_style |
|
|
|
self.precision = precision |
|
|
|
if type(vison_config) == dict: |
|
self.vision_config = CLIPVisionConfig(**vison_config) |
|
else: |
|
self.vision_config = vison_config |
|
|
|
if type(lm_config) == dict: |
|
self.lm_config = Qwen2Config(**lm_config) |
|
else: |
|
self.lm_config = lm_config |
|
super().__init__(**kwargs) |
|
|
