Upload gguf convert script

nanollava_mmproj_convert.py

@@ -0,0 +1,190 @@
+import argparse
+from safetensors import safe_open
+from transformers import AutoTokenizer
+from pathlib import Path
+import torch
+import sys
+import os
+
+if 'NO_LOCAL_GGUF' not in os.environ:
+    print(str(Path(__file__).parent / 'gguf-py'))
+    sys.path.insert(1, str(Path(__file__).parent / 'gguf-py'))
+
+from gguf import *
+
+
+def k(raw_key: str, arch: str) -> str:
+    return raw_key.format(arch=arch)
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("--model", type=str, default="../nanoLLaVA")
+parser.add_argument("--tokenizer", type=str, default="nanoLLaVA")
+args = parser.parse_args()
+
+tensors = safe_open(f'{args.model}/model.safetensors', framework="pt", device="cpu")
+
+### Vision encoder
+
+ftype = 1  # fp16
+
+fname_middle = "mmproj-"
+has_text_encoder = False
+has_llava_projector = True
+
+fname_out = f"{args.model}/nanollava-mmproj-f16.gguf"
+fout = GGUFWriter(fname_out, arch="clip")
+
+fout.add_bool("clip.has_text_encoder", False)
+fout.add_bool("clip.has_vision_encoder", True)
+fout.add_bool("clip.has_llava_projector", True)
+fout.add_file_type(ftype)  # fp16
+
+model_name = "qnguyen3/nanoLLaVA"
+fout.add_name(model_name)
+fout.add_description("image encoder for " + model_name)
+fout.add_string("clip.projector_type", "mlp")
+
+# vision model hparams
+VISION = "clip.vision"
+fout.add_uint32("clip.vision.image_size", 378)
+fout.add_uint32("clip.vision.patch_size", 14)
+fout.add_uint32(k(KEY_EMBEDDING_LENGTH, VISION), 1152)
+fout.add_uint32(k(KEY_FEED_FORWARD_LENGTH, VISION), 4304)
+fout.add_uint32("clip.vision.projection_dim", 2048)
+fout.add_uint32(k(KEY_ATTENTION_HEAD_COUNT, VISION), 16)
+fout.add_float32(k(KEY_ATTENTION_LAYERNORM_EPS, VISION), 1e-6)
+fout.add_uint32(k(KEY_BLOCK_COUNT, VISION), 27 + 1)
+
+fout.add_array("clip.vision.image_mean", [0.5, 0.5, 0.5])
+fout.add_array("clip.vision.image_std", [0.5, 0.5, 0.5])
+fout.add_bool("clip.use_gelu", True)  # using regular GELU instead of quick
+
+# vision projection
+fout.add_tensor(
+    "mm.0.weight",
+    tensors.get_tensor("model.mm_projector.0.weight").to(
+        torch.float16
+    ).numpy().copy()
+)
+fout.add_tensor(
+    "mm.0.bias",
+    tensors.get_tensor("model.mm_projector.0.bias").to(torch.float32).numpy().copy(),
+)
+fout.add_tensor(
+    "mm.2.weight",
+    tensors.get_tensor("model.mm_projector.2.weight").to(
+        torch.float16
+    ).numpy().copy(),
+)
+fout.add_tensor(
+    "mm.2.bias",
+    tensors.get_tensor("model.mm_projector.2.bias").to(torch.float32).numpy().copy(),
+)
+
+# encoder (siglip)
+fout.add_tensor(
+    "v.position_embd.weight",
+    tensors.get_tensor("model.vision_tower.vision_tower.vision_model.embeddings.position_embedding.weight").to(
+        torch.float16
+    ).numpy().copy(),
+)
+fout.add_tensor(
+    "v.patch_embd.weight",
+    tensors.get_tensor(
+        "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.weight"
+    )
+    .reshape(1152, 3, 14, 14)
+    .to(torch.float16).numpy().copy(),
+)
+fout.add_tensor(
+    "v.patch_embd.bias",
+    tensors.get_tensor(
+        "model.vision_tower.vision_tower.vision_model.embeddings.patch_embedding.bias"
+    ).to(torch.float32).numpy().copy(),
+)
+
+fout.add_tensor(
+    "v.post_ln.weight",
+    tensors.get_tensor("model.vision_tower.vision_tower.vision_model.post_layernorm.weight").to(
+        torch.float32
+    ).numpy().copy(),
+)
+fout.add_tensor(
+    "v.post_ln.bias",
+    tensors.get_tensor("model.vision_tower.vision_tower.vision_model.post_layernorm.bias").to(
+        torch.float32
+    ).numpy().copy(),
+)
+
+def blk_tensor(i, name):
+    return tensors.get_tensor(
+        rf"model.vision_tower.vision_tower.vision_model.encoder.layers.{i}.{name}"
+    )
+
+def add_tensor(blk_id, gguf_id=None):
+    if gguf_id is None:
+        gguf_id = blk_id
+
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_q.weight", blk_tensor(blk_id, "self_attn.q_proj.weight").to(torch.float16).numpy().copy())
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_q.bias", blk_tensor(blk_id, "self_attn.q_proj.bias").to(torch.float32).numpy().copy())
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_k.weight", blk_tensor(blk_id, "self_attn.k_proj.weight").to(torch.float16).numpy().copy())
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_k.bias", blk_tensor(blk_id, "self_attn.k_proj.bias").to(torch.float32).numpy().copy())
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_v.weight", blk_tensor(blk_id, "self_attn.v_proj.weight").to(torch.float16).numpy().copy())
+    fout.add_tensor(f"v.blk.{gguf_id}.attn_v.bias", blk_tensor(blk_id, "self_attn.v_proj.bias").to(torch.float32).numpy().copy())
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.attn_out.weight",
+        blk_tensor(blk_id, "self_attn.out_proj.weight").to(torch.float16).numpy().copy(),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.attn_out.bias",
+        blk_tensor(blk_id, "self_attn.out_proj.bias").to(torch.float32).numpy().copy(),
+    )
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln1.weight",
+        blk_tensor(blk_id, "layer_norm1.weight").to(torch.float32).numpy().copy(),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln1.bias",
+        blk_tensor(blk_id, "layer_norm1.bias").to(torch.float32).numpy().copy(),
+    )
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_down.weight",
+        blk_tensor(blk_id, "mlp.fc1.weight").to(torch.float16).numpy().copy(),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_down.bias",
+        blk_tensor(blk_id, "mlp.fc1.bias").to(torch.float32).numpy().copy(),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_up.weight",
+        blk_tensor(blk_id, "mlp.fc2.weight").to(torch.float16).numpy().copy(),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ffn_up.bias",
+        blk_tensor(blk_id, "mlp.fc2.bias").to(torch.float32).numpy().copy(),
+    )
+
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln2.weight",
+        blk_tensor(blk_id, "layer_norm2.weight").to(torch.float32).numpy().copy(),
+    )
+    fout.add_tensor(
+        f"v.blk.{gguf_id}.ln2.bias",
+        blk_tensor(blk_id, "layer_norm2.bias").to(torch.float32).numpy().copy(),
+    )
+
+for i in range(27):
+    add_tensor(i)
+
+# Duplicate the last block (llava-cli skips over this)
+add_tensor(26, 27)
+
+fout.write_header_to_file()
+fout.write_kv_data_to_file()
+fout.write_tensors_to_file()
+fout.close()
+print(f'successfully exported to {fname_out}')