import os from numpy import true_divide import gradio as gr import warnings warnings.filterwarnings("ignore") os.system("python setup.py build develop --user") from maskrcnn_benchmark.config import cfg from maskrcnn_benchmark.engine.predictor_glip import GLIPDemo import vqa import cv2 from PIL import Image import numpy as np # Use this command for evaluate the GLIP-T model config_file = "configs/glip_Swin_T_O365_GoldG.yaml" weight_file = "checkpoints/glip_tiny_model_o365_goldg_cc_sbu.pth" # manual override some options cfg.local_rank = 0 cfg.num_gpus = 1 cfg.merge_from_file(config_file) cfg.merge_from_list(["MODEL.WEIGHT", weight_file]) cfg.merge_from_list(["MODEL.DEVICE", "cuda"]) glip_demo = GLIPDemo( cfg, min_image_size=800, confidence_threshold=0.7, show_mask_heatmaps=False ) blip_demo = vqa.VQA( model_path = 'checkpoints/model_base_vqa_capfilt_large.pth') def predict_image(image, object, question): result, _ = glip_demo.run_on_web_image(image[:, :, [2, 1, 0]], object, 0.5) result = result[:, :, [2, 1, 0]] answer = blip_demo.vqa_demo(image, question) return result, answer def predict_video(video, object, question, frame_drop_value): vid = cv2.VideoCapture(video) count = 0 while True: ret, frame = vid.read() if ret: count+=1 if count % frame_drop_value == 0: # image = Image.fromarray(frame) image = frame cv2.putText( img = image, text = str(count), org = (20, 20), fontFace = cv2.FONT_HERSHEY_DUPLEX, fontScale = 0.5, color = (125, 246, 55), thickness = 1) result, _ = glip_demo.run_on_web_image(image[:, :, [2, 1, 0]], object, 0.5) answer = blip_demo.vqa_demo(image, question) yield result, answer else: break yield result, answer with gr.Blocks() as demo: gr.Markdown("Text-Based Object Detection and Visual Question Answering") with gr.Tab("Image"): with gr.Row(): with gr.Column(): image_input = gr.Image(label='input image') obj_input = gr.Textbox(label='Objects', lines=1, placeholder="Objects here..") vqa_input = gr.Textbox(label='Question', lines=1, placeholder="Question here..") image_button = gr.Button("Submit") with gr.Column(): image_output = gr.outputs.Image(type="pil", label="grounding results") vqa_output = gr.Textbox(label="Answer") with gr.Tab("Video"): with gr.Row(): with gr.Column(): video_input = gr.PlayableVideo(label='input video', mirror_webcam=False) obj_input_video = gr.Textbox(label='Objects', lines=1, placeholder="Objects here..") vqa_input_video = gr.Textbox(label='Question', lines=1, placeholder="Question here..") frame_drop_input = gr.Slider(label='Frames drop value', minimum=0, maximum=30, step=1, value=5) video_button = gr.Button("Submit") with gr.Column(): video_output = gr.outputs.Image(type="pil", label="grounding results") vqa_output_video = gr.Textbox(label="Answer") image_button.click(predict_image, inputs=[image_input, obj_input, vqa_input], outputs=[image_output, vqa_output]) video_button.click(predict_video, inputs=[video_input, obj_input_video, vqa_input_video, frame_drop_input], outputs=[video_output, vqa_output_video]) demo.queue() demo.launch()