Upload 6 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+baklava.png filter=lfs diff=lfs merge=lfs -text
+bee.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,14 @@
----
-title: CosmosLLaVA
-emoji: 📉
-colorFrom: indigo
-colorTo: purple
-sdk: gradio
-sdk_version: 4.41.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: Try CosmosLLaVA
+emoji: 🔥
+colorFrom: yellow
+colorTo: green
+sdk: gradio
+sdk_version: 4.28.3
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: The best open source Turkish vision model
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -1,141 +1,144 @@
-import spaces
-
-import time
-from threading import Thread
-
-import gradio as gr
-import torch
-from PIL import Image
-from transformers import AutoProcessor
-from llava.constants import (
-    IMAGE_TOKEN_INDEX,
-    DEFAULT_IMAGE_TOKEN,
-    DEFAULT_IM_START_TOKEN,
-    DEFAULT_IM_END_TOKEN,
-    IMAGE_PLACEHOLDER,
-)
-from llava.model.builder import load_pretrained_model
-from llava.utils import disable_torch_init
-from llava.mm_utils import (
-    process_images,
-    tokenizer_image_token,
-    get_model_name_from_path,
-)
-from io import BytesIO
-import requests
-import os
-from conversation import Conversation, SeparatorStyle
-
-model_id = "ytu-ce-cosmos/Turkish-LLaVA-v0.1"
-
-disable_torch_init()
-model_name = get_model_name_from_path(model_id)
-tokenizer, model, image_processor, context_len = load_pretrained_model(
-    model_id, None, model_name
-)
-
-def load_image(image_file):
-    if image_file.startswith("http") or image_file.startswith("https"):
-        response = requests.get(image_file)
-        image = Image.open(BytesIO(response.content)).convert("RGB")
-    elif os.path.exists(image_file):
-        image = Image.open(image_file).convert("RGB")
-    else:
-        raise FileNotFoundError(f"Image file {image_file} not found.")
-    return image
-
-def infer_single_image(model_id, image_file, prompt):
-    image_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN
-    if IMAGE_PLACEHOLDER in prompt:
-        if model.config.mm_use_im_start_end:
-            prompt = re.sub(IMAGE_PLACEHOLDER, image_token_se, prompt)
-        else:
-            prompt = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, prompt)
-    else:
-        if model.config.mm_use_im_start_end:
-            prompt = image_token_se + "\n" + prompt
-        else:
-            prompt = DEFAULT_IMAGE_TOKEN + "\n" + prompt
-
-    conv = Conversation(
-        system="""<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nSen bir yapay zeka asistanısın. Kullanıcı sana bir görev verecek. Amacın görevi olabildiğince sadık bir şekilde tamamlamak. Görevi yerine getirirken adım adım düşün ve adımlarını gerekçelendir.""",
-        roles=("<|start_header_id|>user<|end_header_id|>\n\n", "<|start_header_id|>assistant<|end_header_id|>\n\n"),
-        version="llama3",
-        messages=[],
-        offset=0,
-        sep_style=SeparatorStyle.MPT,
-        sep="<|eot_id|>",
-    )
-    conv.append_message(conv.roles[0], prompt)
-    conv.append_message(conv.roles[1], None)
-    full_prompt = conv.get_prompt()
-
-    print("full prompt: ", full_prompt)
-
-    image = load_image(image_file)
-    image_tensor = process_images(
-        [image],
-        image_processor,
-        model.config
-    ).to(model.device, dtype=torch.float16)
-
-    input_ids = (
-        tokenizer_image_token(full_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
-        .unsqueeze(0)
-        .cuda()
-    )
-
-    with torch.inference_mode():
-        output_ids = model.generate(
-            input_ids,
-            images=image_tensor,
-            image_sizes=[image.size],
-            do_sample=False,
-            max_new_tokens=512,
-            use_cache=True,
-        )
-
-    output = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
-    return output
-
-@spaces.GPU
-def bot_streaming(message, history):
-    print(message)
-    if message["files"]:
-        if type(message["files"][-1]) == dict:
-            image = message["files"][-1]["path"]
-        else:
-            image = message["files"][-1]
-    else:
-        for hist in history:
-            if type(hist[0]) == tuple:
-                image = hist[0][0]
-    try:
-        if image is None:
-            gr.Error("You need to upload an image for LLaVA to work.")
-    except NameError:
-        gr.Error("You need to upload an image for LLaVA to work.")
-
-    prompt = message['text']
-
-    result = infer_single_image(model_id, image, prompt)
-    yield result
-
-chatbot = gr.Chatbot(scale=1)
-chat_input = gr.MultimodalTextbox(interactive=True, file_types=["image"], placeholder="Enter message or upload file...", show_label=False)
-
-with gr.Blocks(fill_height=True) as demo:
-    gr.ChatInterface(
-        fn=bot_streaming,
-        title="LLaVA Llama-3-8B",
-        examples=[{"text": "Çiçeğin üzerinde ne var?", "files": ["./bee.jpg"]},
-                  {"text": "Bu tatlı nasıl yapılır?", "files": ["./baklava.png"]}],
-        description="",
-        stop_btn="Stop Generation",
-        multimodal=True,
-        textbox=chat_input,
-        chatbot=chatbot,
-    )
-
-demo.queue(api_open=False)
-demo.launch(show_api=False, share=False)
+import spaces
+
+import time
+from threading import Thread
+
+import gradio as gr
+import torch
+from PIL import Image
+from transformers import AutoProcessor
+from llava.constants import (
+    IMAGE_TOKEN_INDEX,
+    DEFAULT_IMAGE_TOKEN,
+    DEFAULT_IM_START_TOKEN,
+    DEFAULT_IM_END_TOKEN,
+    IMAGE_PLACEHOLDER,
+)
+from llava.model.builder import load_pretrained_model
+from llava.utils import disable_torch_init
+from llava.mm_utils import (
+    process_images,
+    tokenizer_image_token,
+    get_model_name_from_path,
+)
+from io import BytesIO
+import requests
+import os
+from conversation import Conversation, SeparatorStyle
+
+model_id = "ytu-ce-cosmos/Turkish-LLaVA-v0.1"
+
+disable_torch_init()
+model_name = get_model_name_from_path(model_id)
+tokenizer, model, image_processor, context_len = load_pretrained_model(
+    model_id, None, model_name
+)
+
+def load_image(image_file):
+    if image_file.startswith("http") or image_file.startswith("https"):
+        response = requests.get(image_file)
+        image = Image.open(BytesIO(response.content)).convert("RGB")
+    elif os.path.exists(image_file):
+        image = Image.open(image_file).convert("RGB")
+    else:
+        raise FileNotFoundError(f"Görüntü dosyası {image_file} bulunamadı.")
+    return image
+
+def infer_single_image(model_id, image_file, prompt):
+    image_token_se = DEFAULT_IM_START_TOKEN + DEFAULT_IMAGE_TOKEN + DEFAULT_IM_END_TOKEN
+    if IMAGE_PLACEHOLDER in prompt:
+        if model.config.mm_use_im_start_end:
+            prompt = re.sub(IMAGE_PLACEHOLDER, image_token_se, prompt)
+        else:
+            prompt = re.sub(IMAGE_PLACEHOLDER, DEFAULT_IMAGE_TOKEN, prompt)
+    else:
+        if model.config.mm_use_im_start_end:
+            prompt = image_token_se + "\n" + prompt
+        else:
+            prompt = DEFAULT_IMAGE_TOKEN + "\n" + prompt
+
+    conv = Conversation(
+        system="""<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nSen bir yapay zeka asistanısın. Kullanıcı sana bir görev verecek. Amacın görevi olabildiğince sadık bir şekilde tamamlamak. Görevi yerine getirirken adım adım düşün ve adımlarını gerekçelendir.""",
+        roles=("<|start_header_id|>user<|end_header_id|>\n\n", "<|start_header_id|>assistant<|end_header_id|>\n\n"),
+        version="llama3",
+        messages=[],
+        offset=0,
+        sep_style=SeparatorStyle.MPT,
+        sep="<|eot_id|>",
+    )
+    conv.append_message(conv.roles[0], prompt)
+    conv.append_message(conv.roles[1], None)
+    full_prompt = conv.get_prompt()
+
+    print("full prompt: ", full_prompt)
+
+    image = load_image(image_file)
+    image_tensor = process_images(
+        [image],
+        image_processor,
+        model.config
+    ).to(model.device, dtype=torch.float16)
+
+    input_ids = (
+        tokenizer_image_token(full_prompt, tokenizer, IMAGE_TOKEN_INDEX, return_tensors="pt")
+        .unsqueeze(0)
+        .cuda()
+    )
+
+    with torch.inference_mode():
+        output_ids = model.generate(
+            input_ids,
+            images=image_tensor,
+            image_sizes=[image.size],
+            do_sample=False,
+            max_new_tokens=512,
+            use_cache=True,
+        )
+
+    output = tokenizer.batch_decode(output_ids, skip_special_tokens=True)[0].strip()
+    return output
+
+@spaces.GPU
+def bot_streaming(message, history):
+    print(message)
+    if message["files"]:
+        if type(message["files"][-1]) == dict:
+            image = message["files"][-1]["path"]
+        else:
+            image = message["files"][-1]
+    else:
+        for hist in history:
+            if type(hist[0]) == tuple:
+                image = hist[0][0]
+    try:
+        if image is None:
+            gr.Error("LLaVA'nın çalışması için bir resim yüklemeniz gerekir.")
+    except NameError:
+        gr.Error("LLaVA'nın çalışması için bir resim yüklemeniz gerekir.")
+
+    prompt = message['text']
+
+    result = infer_single_image(model_id, image, prompt)
+    
+    print(result)
+    
+    yield result
+
+chatbot = gr.Chatbot(scale=1)
+chat_input = gr.MultimodalTextbox(interactive=True, file_types=["image"], placeholder="Mesaj girin veya dosya yükleyin...", show_label=False)
+
+with gr.Blocks(fill_height=True) as demo:
+    gr.ChatInterface(
+        fn=bot_streaming,
+        title="LLaVA Llama-3-8B",
+        examples=[{"text": "Çiçeğin üzerinde ne var?", "files": ["./bee.jpg"]},
+                  {"text": "Bu tatlı nasıl yapılır?", "files": ["./baklava.png"]}],
+        description="",
+        stop_btn="Stop Generation",
+        multimodal=True,
+        textbox=chat_input,
+        chatbot=chatbot,
+    )
+
+demo.queue(api_open=False)
+demo.launch(show_api=False, share=False)

conversation.py

@@ -0,0 +1,209 @@
+import dataclasses
+from enum import auto, Enum
+from typing import List, Tuple
+import base64
+from io import BytesIO
+from PIL import Image
+
+
+class SeparatorStyle(Enum):
+    """Different separator style."""
+    SINGLE = auto()
+    TWO = auto()
+    MPT = auto()
+    PLAIN = auto()
+    LLAMA_2 = auto()
+
+
+@dataclasses.dataclass
+class Conversation:
+    """A class that keeps all conversation history."""
+    system: str
+    roles: List[str]
+    messages: List[List[str]]
+    offset: int
+    sep_style: SeparatorStyle = SeparatorStyle.SINGLE
+    sep: str = "###"
+    sep2: str = None
+    version: str = "Unknown"
+
+    skip_next: bool = False
+
+    def get_prompt(self):
+        messages = self.messages
+        if len(messages) > 0 and type(messages[0][1]) is tuple:
+            messages = self.messages.copy()
+            init_role, init_msg = messages[0].copy()
+            init_msg = init_msg[0].replace("<image>", "").strip()
+            if 'mmtag' in self.version:
+                messages[0] = (init_role, init_msg)
+                messages.insert(0, (self.roles[0], "<Image><image></Image>"))
+                messages.insert(1, (self.roles[1], "Received."))
+            else:
+                messages[0] = (init_role, "<image>\n" + init_msg)
+
+        if self.sep_style == SeparatorStyle.SINGLE:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + self.sep
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.TWO:
+            seps = [self.sep, self.sep2]
+            ret = self.system + seps[0]
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + ": " + message + seps[i % 2]
+                else:
+                    ret += role + ":"
+        elif self.sep_style == SeparatorStyle.MPT:
+            ret = self.system + self.sep
+            for role, message in messages:
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += role + message + self.sep
+                else:
+                    ret += role
+        elif self.sep_style == SeparatorStyle.LLAMA_2:
+            wrap_sys = lambda msg: f"<<SYS>>\n{msg}\n<</SYS>>\n\n" if len(msg) > 0 else msg
+            wrap_inst = lambda msg: f"[INST] {msg} [/INST]"
+            ret = ""
+
+            for i, (role, message) in enumerate(messages):
+                if i == 0:
+                    assert message, "first message should not be none"
+                    assert role == self.roles[0], "first message should come from user"
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    if i == 0: message = wrap_sys(self.system) + message
+                    if i % 2 == 0:
+                        message = wrap_inst(message)
+                        ret += self.sep + message
+                    else:
+                        ret += " " + message + " " + self.sep2
+                else:
+                    ret += ""
+            ret = ret.lstrip(self.sep)
+        elif self.sep_style == SeparatorStyle.PLAIN:
+            seps = [self.sep, self.sep2]
+            ret = self.system
+            for i, (role, message) in enumerate(messages):
+                if message:
+                    if type(message) is tuple:
+                        message, _, _ = message
+                    ret += message + seps[i % 2]
+                else:
+                    ret += ""
+        else:
+            raise ValueError(f"Invalid style: {self.sep_style}")
+
+        return ret
+
+    def append_message(self, role, message):
+        self.messages.append([role, message])
+
+    def process_image(self, image, image_process_mode, return_pil=False, image_format='PNG', max_len=1344, min_len=672):
+        if image_process_mode == "Pad":
+            def expand2square(pil_img, background_color=(122, 116, 104)):
+                width, height = pil_img.size
+                if width == height:
+                    return pil_img
+                elif width > height:
+                    result = Image.new(pil_img.mode, (width, width), background_color)
+                    result.paste(pil_img, (0, (width - height) // 2))
+                    return result
+                else:
+                    result = Image.new(pil_img.mode, (height, height), background_color)
+                    result.paste(pil_img, ((height - width) // 2, 0))
+                    return result
+
+            image = expand2square(image)
+        elif image_process_mode in ["Default", "Crop"]:
+            pass
+        elif image_process_mode == "Resize":
+            image = image.resize((336, 336))
+        else:
+            raise ValueError(f"Invalid image_process_mode: {image_process_mode}")
+        if max(image.size) > max_len:
+            max_hw, min_hw = max(image.size), min(image.size)
+            aspect_ratio = max_hw / min_hw
+            shortest_edge = int(min(max_len / aspect_ratio, min_len, min_hw))
+            longest_edge = int(shortest_edge * aspect_ratio)
+            W, H = image.size
+            if H > W:
+                H, W = longest_edge, shortest_edge
+            else:
+                H, W = shortest_edge, longest_edge
+            image = image.resize((W, H))
+        if return_pil:
+            return image
+        else:
+            buffered = BytesIO()
+            image.save(buffered, format=image_format)
+            img_b64_str = base64.b64encode(buffered.getvalue()).decode()
+            return img_b64_str
+
+    def get_images(self, return_pil=False):
+        images = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    msg, image, image_process_mode = msg
+                    image = self.process_image(image, image_process_mode, return_pil=return_pil)
+                    images.append(image)
+        return images
+
+    def to_gradio_chatbot(self):
+        ret = []
+        for i, (role, msg) in enumerate(self.messages[self.offset:]):
+            if i % 2 == 0:
+                if type(msg) is tuple:
+                    msg, image, image_process_mode = msg
+                    img_b64_str = self.process_image(
+                        image, "Default", return_pil=False,
+                        image_format='JPEG')
+                    img_str = f'<img src="data:image/jpeg;base64,{img_b64_str}" alt="user upload image" />'
+                    msg = img_str + msg.replace('<image>', '').strip()
+                    ret.append([msg, None])
+                else:
+                    ret.append([msg, None])
+            else:
+                ret[-1][-1] = msg
+        return ret
+
+    def copy(self):
+        return Conversation(
+            system=self.system,
+            roles=self.roles,
+            messages=[[x, y] for x, y in self.messages],
+            offset=self.offset,
+            sep_style=self.sep_style,
+            sep=self.sep,
+            sep2=self.sep2,
+            version=self.version)
+
+    def dict(self):
+        if len(self.get_images()) > 0:
+            return {
+                "system": self.system,
+                "roles": self.roles,
+                "messages": [[x, y[0] if type(y) is tuple else y] for x, y in self.messages],
+                "offset": self.offset,
+                "sep": self.sep,
+                "sep2": self.sep2,
+            }
+        return {
+            "system": self.system,
+            "roles": self.roles,
+            "messages": self.messages,
+            "offset": self.offset,
+            "sep": self.sep,
+            "sep2": self.sep2,
+        }

requirements.txt

@@ -0,0 +1,2 @@
+llava-torch
+spaces