|
from threading import Thread |
|
|
|
import gradio as gr |
|
import spaces |
|
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer |
|
|
|
BANNER_HTML = """ |
|
<p align="center"> |
|
<a href="https://github.com/ymcui/Chinese-LLaMA-Alpaca-3"> |
|
<img src="https://ymcui.com/images/chinese-llama-alpaca-3-banner.png" width="600"/> |
|
</a> |
|
</p> |
|
<h3> |
|
<center>Check our |
|
<a href='https://github.com/ymcui/Chinese-LLaMA-Alpaca-3' target='_blank'>Chinese-LLaMA-Alpaca-3 GitHub Project</a> |
|
for more information. |
|
</center> |
|
</h3> |
|
<p> |
|
<center><em>The demo is mainly for academic purposes and users are not expected to use this demo for illegal activities.</em></center> |
|
</p> |
|
""" |
|
|
|
DEFAULT_SYSTEM_PROMPT = "You are a helpful assistant. 你是一个乐于助人的助手。" |
|
|
|
|
|
def load_model(version): |
|
global tokenizer, model |
|
if version == "v1": |
|
model_name = "hfl/llama-3-chinese-8b-instruct" |
|
elif version == "v2": |
|
model_name = "hfl/llama-3-chinese-8b-instruct-v2" |
|
elif version == "v3": |
|
model_name = "hfl/llama-3-chinese-8b-instruct-v3" |
|
|
|
tokenizer = AutoTokenizer.from_pretrained(model_name) |
|
model = AutoModelForCausalLM.from_pretrained(model_name, device_map="auto") |
|
return f"Model {model_name} loaded." |
|
|
|
@spaces.GPU(duration=50) |
|
def stream_chat(message: str, history: list, system_prompt: str, model_version: str, temperature: float, max_new_tokens: int): |
|
conversation = [{"role": "system", "content": system_prompt or DEFAULT_SYSTEM_PROMPT}] |
|
for prompt, answer in history: |
|
conversation.extend([{"role": "user", "content": prompt}, {"role": "assistant", "content": answer}]) |
|
|
|
conversation.append({"role": "user", "content": message}) |
|
|
|
input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device) |
|
streamer = TextIteratorStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True) |
|
|
|
generate_kwargs = { |
|
"input_ids": input_ids, |
|
"streamer": streamer, |
|
"max_new_tokens": max_new_tokens, |
|
"temperature": temperature, |
|
"do_sample": temperature != 0, |
|
} |
|
|
|
generation_thread = Thread(target=model.generate, kwargs=generate_kwargs) |
|
generation_thread.start() |
|
|
|
output = "" |
|
for new_token in streamer: |
|
output += new_token |
|
yield output |
|
|
|
chatbot = gr.Chatbot(height=500) |
|
|
|
with gr.Blocks() as demo: |
|
gr.HTML(BANNER_HTML) |
|
gr.ChatInterface( |
|
fn=stream_chat, |
|
chatbot=chatbot, |
|
fill_height=True, |
|
additional_inputs_accordion=gr.Accordion(label="Parameters / 参数设置", open=False, render=False), |
|
additional_inputs=[ |
|
gr.Text(value=DEFAULT_SYSTEM_PROMPT, label="System Prompt / 系统提示词", render=False), |
|
gr.Radio(choices=["v1", "v2", "v3"], label="Model Version / 模型版本", value="v3", interactive=False, render=False), |
|
gr.Slider(minimum=0, maximum=1, step=0.1, value=0.5, label="Temperature / 温度系数", render=False), |
|
gr.Slider(minimum=128, maximum=2048, step=1, value=256, label="Max new tokens / 最大生成长度", render=False), |
|
], |
|
cache_examples=False, |
|
) |
|
|
|
if __name__ == "__main__": |
|
load_model("v3") |
|
demo.launch() |
|
|
