app.py

import os
import gradio as gr
import torch
import numpy as np
from transformers import pipeline

name_list = ['microsoft/biogpt', 'stanford-crfm/BioMedLM', 'facebook/galactica-1.3b']

examples = [['COVID-19 is'],['A 65-year-old female patient with a past medical history of']] 

print(f"Is CUDA available: {torch.cuda.is_available()}")
print(f"CUDA device: {torch.cuda.get_device_name(torch.cuda.current_device())}")

pipe_biogpt = pipeline("text-generation", model="microsoft/BioGPT-Large", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
pipe_biomedlm = pipeline("text-generation", model="stanford-crfm/BioMedLM", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})
pipe_galactica = pipeline("text-generation", model="facebook/galactica-1.3b", device="cuda:0", model_kwargs={"torch_dtype":torch.bfloat16})

title = "Compare generative biomedical LLMs!"
description = "**Disclaimer:** this demo was made for research purposes only and should not be used for medical purposes."

def inference(text):
  output_biogpt = pipe_biogpt(text, max_length=100)[0]["generated_text"]
  output_biomedlm = pipe_biomedlm(text, max_length=100)[0]["generated_text"]
  output_galactica = pipe_galactica(text, max_length=100)[0]["generated_text"]
  return [
      output_biogpt, 
      output_biomedlm,
      output_galactica
  ]

io = gr.Interface(
  inference,
  gr.Textbox(lines=3),
  outputs=[
    gr.Textbox(lines=3, label="BioGPT-Large"),
    gr.Textbox(lines=3, label="BioMedLM (fka PubmedGPT)"),
    gr.Textbox(lines=3, label="Galactica 1.3B"),
  ],
  title=title,
  description=description,
  examples=examples
)
io.launch()