File size: 3,393 Bytes
c49a56c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 |
from threading import Thread
import gradio as gr
import spaces
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
BANNER_HTML = """
<p align="center">
<a href="https://github.com/ymcui/Chinese-LLaMA-Alpaca-3">
<img src="https://ymcui.com/images/chinese-llama-alpaca-3-banner.png" width="600"/>
</a>
</p>
<h3>
<center>Check our
<a href='https://github.com/ymcui/Chinese-LLaMA-Alpaca-3' target='_blank'>Chinese-LLaMA-Alpaca-3 GitHub Project</a>
for more information.
</center>
</h3>
<p>
<center><em>The demo is mainly for academic purposes and users are not expected to use this demo for illegal activities.</em></center>
</p>
"""
DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant. 你是一个乐于助人的助手。"
# Load different instruct models based on the selected version
def load_model(version):
global tokenizer, model
if version == "v1":
model_name = "hfl/llama-3-chinese-8b-instruct"
elif version == "v2":
model_name = "hfl/llama-3-chinese-8b-instruct-v2"
elif version == "v3":
model_name = "hfl/llama-3-chinese-8b-instruct-v3"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto")
return f"Model {model_name} loaded."
@spaces.GPU(duration=50)
def stream_chat(message: str, history: list, system_prompt: str, model_version: str, temperature: float, max_new_tokens: int):
conversation = [{"role": "system", "content": system_prompt or DEFAULT_SYSTEM_PROMPT}]
for prompt, answer in history:
conversation.extend([{"role": "user", "content": prompt}, {"role": "assistant", "content": answer}])
conversation.append({"role": "user", "content": message})
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True)
generate_kwargs = {
"input_ids": input_ids,
"streamer": streamer,
"max_new_tokens": max_new_tokens,
"temperature": temperature,
"do_sample": temperature != 0,
}
generation_thread = Thread(target=model.generate, kwargs=generate_kwargs)
generation_thread.start()
output = ""
for new_token in streamer:
output += new_token
yield output
chatbot = gr.Chatbot(height=500)
with gr.Blocks() as demo:
gr.HTML(BANNER_HTML)
gr.ChatInterface(
fn=stream_chat,
chatbot=chatbot,
fill_height=True,
additional_inputs_accordion=gr.Accordion(label="Parameters / 参数设置", open=False, render=False),
additional_inputs=[
gr.Text(value=DEFAULT_SYSTEM_PROMPT, label="System Prompt / 系统提示词", render=False),
gr.Radio(choices=["v1", "v2", "v3"], label="Model Version / 模型版本", value="v3", interactive=False, render=False),
gr.Slider(minimum=0, maximum=1, step=0.1, value=0.5, label="Temperature / 温度系数", render=False),
gr.Slider(minimum=128, maximum=2048, step=1, value=256, label="Max new tokens / 最大生成长度", render=False),
],
cache_examples=False,
)
if __name__ == "__main__":
load_model("v3") # Load the default model
demo.launch()
|