Spaces:
Runtime error
Runtime error
| import os | |
| import sys | |
| from pathlib import Path | |
| os.system("python -m pip install --upgrade pip") | |
| os.system("cd multimodal && pip install .") | |
| os.system("cd multimodal/YOLOX && pip install .") | |
| import numpy as np | |
| import torch | |
| from PIL import Image | |
| import tempfile | |
| import string | |
| import cv2 | |
| import gradio as gr | |
| import torch | |
| from PIL import Image | |
| from huggingface_hub import hf_hub_download, login | |
| from open_flamingo.src.factory import create_model_and_transforms | |
| from open_flamingo.chat.conversation import ChatBOT, CONV_VISION | |
| sys.path.append(str(Path(__file__).parent.parent.parent)) | |
| TEMP_FILE_DIR = Path(__file__).parent / 'temp' | |
| TEMP_FILE_DIR.mkdir(parents=True, exist_ok=True) | |
| SHARED_UI_WARNING = f'''### [NOTE] It is possible that you are waiting in a lengthy queue. | |
| You can duplicate and use it with a paid private GPU. | |
| <a class="duplicate-button" style="display:inline-block" target="_blank" href="https://huggingface.co/spaces/Vision-CAIR/minigpt4?duplicate=true"><img style="margin-top:0;margin-bottom:0" src="https://huggingface.co/datasets/huggingface/badges/raw/main/duplicate-this-space-xl-dark.svg" alt="Duplicate Space"></a> | |
| Alternatively, you can also use the demo on our [project page](https://compositionalvlm.github.io/). | |
| ''' | |
| flamingo, image_processor, tokenizer, vis_embed_size = create_model_and_transforms( | |
| "ViT-L-14", | |
| "datacomp_xl_s13b_b90k", | |
| "EleutherAI/pythia-1.4b", | |
| "EleutherAI/pythia-1.4b", | |
| location_token_num=1000, | |
| lora=False, | |
| lora_r=16, | |
| use_sam=None, | |
| add_visual_token=True, | |
| use_format_v2=True, | |
| add_box=True, | |
| add_pe=False, | |
| add_relation=False, | |
| enhance_data=False, | |
| ) | |
| model_name = "pythiaS" | |
| checkpoint_path = hf_hub_download("chendl/compositional_test", "pythiaS.pt") | |
| checkpoint = torch.load(checkpoint_path, map_location="cpu")["model_state_dict"] | |
| model_state_dict = {} | |
| for key in checkpoint.keys(): | |
| model_state_dict[key.replace("module.", "")] = checkpoint[key] | |
| if "vision_encoder.logit_scale" in model_state_dict: | |
| # previous checkpoint has some unnecessary weights | |
| del model_state_dict["vision_encoder.logit_scale"] | |
| del model_state_dict["vision_encoder.visual.proj"] | |
| del model_state_dict["vision_encoder.visual.ln_post.weight"] | |
| del model_state_dict["vision_encoder.visual.ln_post.bias"] | |
| flamingo.load_state_dict(model_state_dict, strict=True) | |
| chat = ChatBOT(flamingo, image_processor, tokenizer, vis_embed_size,model_name) | |
| def get_outputs( | |
| model, | |
| batch_images, | |
| attention_mask, | |
| max_generation_length, | |
| min_generation_length, | |
| num_beams, | |
| length_penalty, | |
| input_ids, | |
| image_start_index_list=None, | |
| image_nums=None, | |
| bad_words_ids=None, | |
| ): | |
| # and torch.cuda.amp.autocast(dtype=torch.float16) | |
| with torch.inference_mode(): | |
| outputs = model( | |
| vision_x=batch_images, | |
| lang_x=input_ids, | |
| attention_mask=attention_mask, | |
| labels=None, | |
| image_nums=image_nums, | |
| image_start_index_list=image_start_index_list, | |
| added_bbox_list=None, | |
| add_box=False, | |
| ) | |
| # outputs = model.generate( | |
| # batch_images, | |
| # input_ids, | |
| # attention_mask=attention_mask, | |
| # max_new_tokens=max_generation_length, | |
| # min_length=min_generation_length, | |
| # num_beams=num_beams, | |
| # length_penalty=length_penalty, | |
| # image_start_index_list=image_start_index_list, | |
| # image_nums=image_nums, | |
| # bad_words_ids=bad_words_ids, | |
| # ) | |
| return outputs | |
| def generate( | |
| idx, | |
| image, | |
| text, | |
| vis_embed_size=256, | |
| rank=0, | |
| world_size=1, | |
| ): | |
| if image is None: | |
| raise gr.Error("Please upload an image.") | |
| flamingo.eval() | |
| loc_token_ids = [] | |
| for i in range(1000): | |
| loc_token_ids.append(int(tokenizer(f"<loc_{i}>", add_special_tokens=False)["input_ids"][-1])) | |
| media_token_id = tokenizer("<|#image#|>", add_special_tokens=False)["input_ids"][-1] | |
| endofmedia_token_id = tokenizer("<|#endofimage#|>", add_special_tokens=False)["input_ids"][-1] | |
| pad_token_id = tokenizer(tokenizer.pad_token, add_special_tokens=False)["input_ids"][-1] | |
| bos_token_id = tokenizer(tokenizer.bos_token, add_special_tokens=False)["input_ids"][-1] | |
| prebox_token_id = tokenizer("<|#prebox#|>", add_special_tokens=False)["input_ids"][-1] | |
| image_ori = image | |
| image = image.convert("RGB") | |
| width = image.width | |
| height = image.height | |
| image = image.resize((224, 224)) | |
| batch_images = image_processor(image).unsqueeze(0).unsqueeze(1).unsqueeze(0) | |
| if idx == 1: | |
| prompt = [ | |
| f"{tokenizer.bos_token}<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|><|#object#|> {text.rstrip('.').strip()}<|#endofobject#|><|#visual#|>"] | |
| bad_words_ids = None | |
| max_generation_length = 5 | |
| else: | |
| prompt = [f"<|#image#|>{tokenizer.pad_token * vis_embed_size}<|#endofimage#|>{text.rstrip('.')}"] | |
| bad_words_ids = loc_word_ids | |
| max_generation_length = 30 | |
| encodings = tokenizer( | |
| prompt, | |
| padding="longest", | |
| truncation=True, | |
| return_tensors="pt", | |
| max_length=2000, | |
| ) | |
| input_ids = encodings["input_ids"] | |
| attention_mask = encodings["attention_mask"] | |
| image_start_index_list = ((input_ids == media_token_id).nonzero(as_tuple=True)[-1] + 1).tolist() | |
| image_start_index_list = [[x] for x in image_start_index_list] | |
| image_nums = [1] * len(input_ids) | |
| outputs = get_outputs( | |
| model=flamingo, | |
| batch_images=batch_images, | |
| attention_mask=attention_mask, | |
| max_generation_length=max_generation_length, | |
| min_generation_length=4, | |
| num_beams=1, | |
| length_penalty=1.0, | |
| input_ids=input_ids, | |
| bad_words_ids=bad_words_ids, | |
| image_start_index_list=image_start_index_list, | |
| image_nums=image_nums, | |
| ) | |
| boxes = outputs["boxes"] | |
| scores = outputs["scores"] | |
| if len(scores) > 0: | |
| box = boxes[scores.argmax()] / 224 | |
| print(f"{box}") | |
| if idx == 1: | |
| open_cv_image = np.array(image_ori) | |
| # Convert RGB to BGR | |
| open_cv_image = open_cv_image[:, :, ::-1].copy() | |
| box = box * [width, height, width, height] | |
| # for box in boxes: | |
| open_cv_image = cv2.rectangle(open_cv_image, box[:2].astype(int), box[2:].astype(int), (255, 0, 0), 2) | |
| out_image = Image.fromarray(cv2.cvtColor(open_cv_image, cv2.COLOR_BGR2RGB)) | |
| return f"Output:{box}", out_image | |
| elif idx == 2: | |
| gen_text = tokenizer.batch_decode(outputs) | |
| return (f"Question: {text.strip()} Answer: {gen_text}") | |
| else: | |
| gen_text = tokenizer.batch_decode(outputs) | |
| return (f"Output:{gen_text}") | |
| title = """<h1 align="center">Demo of Compositional-VLM</h1>""" | |
| description = """<h3>This is the demo of Compositional-VLM. Upload your images and start chatting!</h3>""" | |
| article = """<div style='display:flex; gap: 0.25rem; '><a href='https://vis-www.cs.umass.edu/CoVLM/'><img src='https://img.shields.io/badge/Project-Page-Green'></a><a href='https://github.com/UMass-Foundation-Model/CoVLM'><img src='https://img.shields.io/badge/Github-Code-blue'></a><a href='https://arxiv.org/abs/2311.03354'><img src='https://img.shields.io/badge/Paper-PDF-red'></a></div> | |
| """ | |
| # TODO show examples below | |
| # ======================================== | |
| # Gradio Setting | |
| # ======================================== | |
| def gradio_reset(chat_state, img_list): | |
| if chat_state is not None: | |
| chat_state = [] | |
| if img_list is not None: | |
| img_list = [] | |
| return None, gr.update(value=None, interactive=True), gr.update(placeholder='Please upload your image first', | |
| interactive=False), gr.update( | |
| value="Upload & Start Chat", interactive=True), chat_state, img_list | |
| def build_image(image): | |
| if image is None: | |
| return None | |
| # res = draw_bounding_boxes(image=image, boxes=boxes_to_draw, colors=color_to_draw, width=8) | |
| from torchvision.transforms import ToPILImage | |
| # res = ToPILImage()(res) | |
| _, path = tempfile.mkstemp(suffix='.jpg', dir=TEMP_FILE_DIR) | |
| image.save(path) | |
| return path | |
| def upload_img(gr_img, text_input, chat_state, chatbot): | |
| if gr_img is None: | |
| return None, None, gr.update(interactive=True), chat_state, None | |
| chat_state = [] | |
| img_list = [] | |
| path = build_image(gr_img) | |
| chatbot = chatbot + [[(path,), None]] | |
| llm_message = chat.upload_img(gr_img, chat_state, img_list) | |
| return gr.update(interactive=False), gr.Textbox(placeholder='Type and press Enter', interactive=True), gr.update( | |
| value="Start Chatting", interactive=False), chat_state, img_list, chatbot | |
| def gradio_ask(user_message, chatbot, chat_state, radio): | |
| # if len(user_message) == 0: | |
| # return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state | |
| chat.ask(user_message, chat_state, radio) | |
| chatbot = chatbot + [[user_message, None]] | |
| return chatbot, chat_state | |
| def generate_ans(user_message, chatbot, chat_state, img_list, radio, text, num_beams, temperature): | |
| # if len(user_message) == 0: | |
| # return gr.update(interactive=True, placeholder='Input should not be empty!'), chatbot, chat_state | |
| chat.ask(user_message, chat_state, radio) | |
| chatbot = chatbot + [[user_message, None]] | |
| # return chatbot, chat_state | |
| image = None | |
| llm_message, image = \ | |
| chat.answer(conv=chat_state, img_list=img_list, max_new_tokens=300, num_beams=1, temperature=temperature, | |
| max_length=2000, radio=radio, text_input=text) | |
| chatbot[-1][1] = llm_message | |
| if chat_state[-1]["from"] == "gpt": | |
| chat_state[-1]["value"] = llm_message | |
| if image == None: | |
| return "", chatbot, chat_state, img_list | |
| else: | |
| path = build_image(image) | |
| chatbot = chatbot + [[None, (path,)]] | |
| return "", chatbot, chat_state, img_list | |
| def gradio_answer(chatbot, chat_state, img_list, radio, text, num_beams, temperature): | |
| image = None | |
| llm_message, image = \ | |
| chat.answer(conv=chat_state, img_list=img_list, max_new_tokens=300, num_beams=1, temperature=temperature, | |
| max_length=2000, radio=radio, text_input=text) | |
| chatbot[-1][1] = llm_message | |
| if chat_state[-1]["from"] == "gpt": | |
| chat_state[-1]["value"] = llm_message | |
| if image == None: | |
| return "", chatbot, chat_state, img_list | |
| else: | |
| path = build_image(image) | |
| chatbot = chatbot + [[None, (path,)]] | |
| return "", chatbot, chat_state, img_list | |
| task_template = { | |
| "Cap": "Summarize the content of the photo <image>.", | |
| "VQA": "For this image <image>, I want a simple and direct answer to my question: <question>", | |
| "REC": "Can you point out <expr> in the image <image> and provide the coordinates of its location?", | |
| "GC": "Can you give me a description of the region <boxes> in image <image>?", | |
| "Advanced": "<question>", | |
| } | |
| with gr.Blocks() as demo: | |
| gr.Markdown(title) | |
| gr.Markdown(SHARED_UI_WARNING) | |
| gr.Markdown(description) | |
| gr.Markdown(article) | |
| with gr.Row(): | |
| with gr.Column(scale=0.5): | |
| image = gr.Image(type="pil") | |
| upload_button = gr.Button(value="Upload & Start Chat", interactive=True, variant="primary") | |
| clear = gr.Button("Restart") | |
| radio = gr.Radio( | |
| ["Cap", "VQA", "REC", "Advanced"], label="Task Template", value='Cap', | |
| ) | |
| num_beams = gr.Slider( | |
| minimum=1, | |
| maximum=5, | |
| value=1, | |
| step=1, | |
| interactive=True, | |
| label="beam search numbers)", | |
| ) | |
| temperature = gr.Slider( | |
| minimum=0.1, | |
| maximum=2.0, | |
| value=1.0, | |
| step=0.1, | |
| interactive=True, | |
| label="Temperature", | |
| ) | |
| with gr.Column(): | |
| chat_state = gr.State() | |
| img_list = gr.State() | |
| chatbot = gr.Chatbot(label='Compositional-VLM') | |
| # template = gr.Textbox(label='Template', show_label=True, lines=1, interactive=False, | |
| # value='Provide a comprehensive description of the image <image> and specify the positions of any mentioned objects in square brackets.') | |
| # text_input = gr.Textbox(label='<question>', show_label=True, placeholder="Please upload your image first， then input...", lines=3, | |
| # value=None, visible=False, interactive=False) | |
| # with gr.Row(): | |
| text_input = gr.Textbox(label='User', placeholder='Please upload your image first, then input...', | |
| interactive=False) | |
| # submit_button = gr.Button(value="Submit", interactive=True, variant="primary") | |
| upload_button.click(upload_img, [image, text_input, chat_state, chatbot], | |
| [image, text_input, upload_button, chat_state, img_list, chatbot]) | |
| # submit_button.click(gradio_ask, [text_input, chatbot, chat_state,radio], [chatbot, chat_state]).then( | |
| # gradio_answer, [chatbot, chat_state, img_list, radio, text_input,num_beams, temperature], [text_input,chatbot, chat_state, img_list] | |
| # ) | |
| text_input.submit(generate_ans, | |
| [text_input, chatbot, chat_state, img_list, radio, text_input, num_beams, temperature], | |
| [text_input, chatbot, chat_state, img_list]) | |
| # text_input.submit(gradio_ask, [text_input, chatbot, chat_state, radio], [chatbot, chat_state]).then( | |
| # gradio_answer, [chatbot, chat_state, img_list, radio, text_input, num_beams, temperature], | |
| # [text_input, chatbot, chat_state, img_list] | |
| # ) | |
| clear.click(gradio_reset, [chat_state, img_list], [chatbot, image, text_input, upload_button, chat_state, img_list], | |
| queue=False) | |
| demo.launch(share=True) | |
| # | |
| # with gr.Blocks() as demo: | |
| # gr.Markdown( | |
| # """ | |
| # 🍜 Object Centric Pretraining Demo | |
| # In this demo we showcase the in-context learning and grounding capabilities of the Object-Centric Pretrained model, a large multimodal model. Note that we add two additional demonstrations to the ones presented to improve the demo experience. | |
| # The model is trained on an interleaved mixture of text, images and bounding box and is able to generate text conditioned on sequences of images/text. | |
| # """ | |
| # ) | |
| # | |
| # with gr.Accordion("See terms and conditions"): | |
| # gr.Markdown( | |
| # """**Please read the following information carefully before proceeding.**This demo does NOT store any personal information on its users, and it does NOT store user queries.""") | |
| # | |
| # with gr.Tab("📷 Image Captioning"): | |
| # with gr.Row(): | |
| # | |
| # | |
| # query_image = gr.Image(type="pil") | |
| # with gr.Row(): | |
| # chat_input = gr.Textbox(lines=1, label="Chat Input") | |
| # text_output = gr.Textbox(value="Output:", label="Model output") | |
| # | |
| # run_btn = gr.Button("Run model") | |
| # | |
| # | |
| # | |
| # def on_click_fn(img,text): return generate(0, img, text) | |
| # | |
| # run_btn.click(on_click_fn, inputs=[query_image,chat_input], outputs=[text_output]) | |
| # | |
| # with gr.Tab("🦓 Grounding"): | |
| # with gr.Row(): | |
| # with gr.Column(scale=1): | |
| # query_image = gr.Image(type="pil") | |
| # with gr.Column(scale=1): | |
| # out_image = gr.Image(type="pil") | |
| # with gr.Row(): | |
| # chat_input = gr.Textbox(lines=1, label="Chat Input") | |
| # text_output = gr.Textbox(value="Output:", label="Model output") | |
| # | |
| # run_btn = gr.Button("Run model") | |
| # | |
| # | |
| # def on_click_fn(img, text): return generate(1, img, text) | |
| # | |
| # | |
| # run_btn.click(on_click_fn, inputs=[query_image, chat_input], outputs=[text_output, out_image]) | |
| # | |
| # with gr.Tab("🔢 Counting objects"): | |
| # with gr.Row(): | |
| # query_image = gr.Image(type="pil") | |
| # with gr.Row(): | |
| # chat_input = gr.Textbox(lines=1, label="Chat Input") | |
| # text_output = gr.Textbox(value="Output:", label="Model output") | |
| # | |
| # run_btn = gr.Button("Run model") | |
| # | |
| # | |
| # def on_click_fn(img,text): return generate(0, img, text) | |
| # | |
| # | |
| # run_btn.click(on_click_fn, inputs=[query_image, chat_input], outputs=[text_output]) | |
| # | |
| # with gr.Tab("🕵️ Visual Question Answering"): | |
| # with gr.Row(): | |
| # query_image = gr.Image(type="pil") | |
| # with gr.Row(): | |
| # question = gr.Textbox(lines=1, label="Question") | |
| # text_output = gr.Textbox(value="Output:", label="Model output") | |
| # | |
| # run_btn = gr.Button("Run model") | |
| # | |
| # | |
| # def on_click_fn(img, txt): return generate(2, img, txt) | |
| # | |
| # | |
| # run_btn.click( | |
| # on_click_fn, inputs=[query_image, question], outputs=[text_output] | |
| # ) | |
| # | |
| # with gr.Tab("🌎 Custom"): | |
| # gr.Markdown( | |
| # """### Customize the demonstration by uploading your own images and text samples. | |
| # ### **Note: Any text prompt you use will be prepended with an 'Output:', so you don't need to include it in your prompt.**""" | |
| # ) | |
| # with gr.Row(): | |
| # query_image = gr.Image(type="pil") | |
| # with gr.Row(): | |
| # question = gr.Textbox(lines=1, label="Question") | |
| # text_output = gr.Textbox(value="Output:", label="Model output") | |
| # | |
| # run_btn = gr.Button("Run model") | |
| # | |
| # | |
| # def on_click_fn(img, txt): return generate(2, img, txt) | |
| # | |
| # | |
| # run_btn.click( | |
| # on_click_fn, inputs=[query_image, question], outputs=[text_output] | |
| # ) | |
| # | |
| # demo.queue(concurrency_count=1) | |
| # demo.launch() | |