import argparse from safetensors import safe_open from transformers import AutoTokenizer from pathlib import Path import torch import sys import os if 'NO_LOCAL_GGUF' not in os.environ: print(str(Path(__file__).parent / 'gguf-py')) sys.path.insert(1, str(Path(__file__).parent / 'gguf-py')) from gguf import * def k(raw_key: str, arch: str) -> str: return raw_key.format(arch=arch) parser = argparse.ArgumentParser() parser.add_argument("--model", type=str, default="../nanoLLaVA") parser.add_argument("--tokenizer", type=str, default="nanoLLaVA") args = parser.parse_args() tensors = safe_open(f'{args.model}/model.safetensors', framework="pt", device="cpu") ### Vision encoder ftype = 1 # fp16 fname_middle = "mmproj-" has_text_encoder = False has_llava_projector = True fname_out = f"{args.model}/nanollava-mmproj-f16.gguf" fout = GGUFWriter(fname_out, arch="clip") fout.add_bool("clip.has_text_encoder", False) fout.add_bool("clip.has_vision_encoder", True) fout.add_bool("clip.has_llava_projector", True) fout.add_file_type(ftype) # fp16 model_name = "qnguyen3/nanoLLaVA" fout.add_name(model_name) fout.add_description("image encoder for " + model_name) fout.add_string("clip.projector_type", "mlp") # vision model hparams VISION = "clip.vision" fout.add_uint32("clip.vision.image_size", 378) fout.add_uint32("clip.vision.patch_size", 14) fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), 1152) fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), 4304) fout.add_uint32("clip.vision.projection_dim", 2048) fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), 16) fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6) fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), 27 + 1) fout.add_array("clip.vision.image_mean", [0.5, 0.5, 0.5]) fout.add_array("clip.vision.image_std", [0.5, 0.5, 0.5]) fout.add_bool("clip.use_gelu", True) # using regular GELU instead of quick # vision projection fout.add_tensor( "mm.0.weight", tensors.get_tensor("model.mm_projector.0.weight").to( torch.float16 ).numpy().copy() ) fout.add_tensor( "mm.0.bias", tensors.get_tensor("model.mm_projector.0.bias").to(torch.float32).numpy().copy(), ) fout.add_tensor( "mm.2.weight", tensors.get_tensor("model.mm_projector.2.weight").to( torch.float16 ).numpy().copy(), ) fout.add_tensor( "mm.2.bias", tensors.get_tensor("model.mm_projector.2.bias").to(torch.float32).numpy().copy(), ) # encoder (siglip) fout.add_tensor( "v.position_embd.weight", tensors.get_tensor("model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight").to( torch.float16 ).numpy().copy(), ) fout.add_tensor( "v.patch_embd.weight", tensors.get_tensor( "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight" ) .reshape(1152, 3, 14, 14) .to(torch.float16).numpy().copy(), ) fout.add_tensor( "v.patch_embd.bias", tensors.get_tensor( "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.bias" ).to(torch.float32).numpy().copy(), ) fout.add_tensor( "v.post_ln.weight", tensors.get_tensor("model.vision_tower.vision_tower.vision_model.post_layernorm.weight").to( torch.float32 ).numpy().copy(), ) fout.add_tensor( "v.post_ln.bias", tensors.get_tensor("model.vision_tower.vision_tower.vision_model.post_layernorm.bias").to( torch.float32 ).numpy().copy(), ) def blk_tensor(i, name): return tensors.get_tensor( rf"model.vision_tower.vision_tower.vision_model.encoder.layers.{i}.{name}" ) def add_tensor(blk_id, gguf_id=None): if gguf_id is None: gguf_id = blk_id fout.add_tensor(f"v.blk.{gguf_id}.attn_q.weight", blk_tensor(blk_id, "self_attn.q_proj.weight").to(torch.float16).numpy().copy()) fout.add_tensor(f"v.blk.{gguf_id}.attn_q.bias", blk_tensor(blk_id, "self_attn.q_proj.bias").to(torch.float32).numpy().copy()) fout.add_tensor(f"v.blk.{gguf_id}.attn_k.weight", blk_tensor(blk_id, "self_attn.k_proj.weight").to(torch.float16).numpy().copy()) fout.add_tensor(f"v.blk.{gguf_id}.attn_k.bias", blk_tensor(blk_id, "self_attn.k_proj.bias").to(torch.float32).numpy().copy()) fout.add_tensor(f"v.blk.{gguf_id}.attn_v.weight", blk_tensor(blk_id, "self_attn.v_proj.weight").to(torch.float16).numpy().copy()) fout.add_tensor(f"v.blk.{gguf_id}.attn_v.bias", blk_tensor(blk_id, "self_attn.v_proj.bias").to(torch.float32).numpy().copy()) fout.add_tensor( f"v.blk.{gguf_id}.attn_out.weight", blk_tensor(blk_id, "self_attn.out_proj.weight").to(torch.float16).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.attn_out.bias", blk_tensor(blk_id, "self_attn.out_proj.bias").to(torch.float32).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ln1.weight", blk_tensor(blk_id, "layer_norm1.weight").to(torch.float32).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ln1.bias", blk_tensor(blk_id, "layer_norm1.bias").to(torch.float32).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ffn_down.weight", blk_tensor(blk_id, "mlp.fc1.weight").to(torch.float16).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ffn_down.bias", blk_tensor(blk_id, "mlp.fc1.bias").to(torch.float32).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ffn_up.weight", blk_tensor(blk_id, "mlp.fc2.weight").to(torch.float16).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ffn_up.bias", blk_tensor(blk_id, "mlp.fc2.bias").to(torch.float32).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ln2.weight", blk_tensor(blk_id, "layer_norm2.weight").to(torch.float32).numpy().copy(), ) fout.add_tensor( f"v.blk.{gguf_id}.ln2.bias", blk_tensor(blk_id, "layer_norm2.bias").to(torch.float32).numpy().copy(), ) for i in range(27): add_tensor(i) # Duplicate the last block (llava-cli skips over this) add_tensor(26, 27) fout.write_header_to_file() fout.write_kv_data_to_file() fout.write_tensors_to_file() fout.close() print(f'successfully exported to {fname_out}')