Spaces:
Sleeping
Sleeping
File size: 2,901 Bytes
05ac0b1 25635a4 02b33e8 05ac0b1 02b33e8 05ac0b1 02b33e8 05ac0b1 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 |
# import gradio as gr
# def greet(name):
# return "Hello " + name + "!!"
# import torch
# from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
# from peft import PeftModel, PeftConfig
# # class InferenceFineTunning:
# # def __init__(self, model_path):
# # peft_model_id = f"hyang0503/{model_path}"
# # config = PeftConfig.from_pretrained(peft_model_id)
# # bnb_config = BitsAndBytesConfig(
# # load_in_4bit=True,
# # bnb_4bit_use_double_quant=True,
# # bnb_4bit_quant_type="nf4",
# # bnb_4bit_compute_dtype=torch.bfloat16
# # )
# # self.model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path, quantization_config=bnb_config, device_map="auto")
# # self.model = PeftModel.from_pretrained(self.model, peft_model_id)
# # # self.tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
# # self.tokenizer = AutoTokenizer.from_pretrained(peft_model_id)
# # self.tokenizer.pad_token = self.tokenizer.eos_token
# # self.model.eval()
# # def generate(self, q): # 실습 노트북과 내용 다름
# # outputs = self.model.generate(
# # **self.tokenizer(
# # f"### 질문: {q}\n\n### 답변:",
# # return_tensors='pt',
# # return_token_type_ids=False
# # ).to("cuda"),
# # max_new_tokens=256,
# # early_stopping=True,
# # do_sample=True,
# # eos_token_id=2,
# # )
# # print(self.tokenizer.decode(outputs[0]))
# # ifg = InferenceFineTunning("qlora-koalpaca")
# # iface = gr.Interface(fn=ifg.generate, inputs="text", outputs="text")
# iface = gr.Interface(fn=greet, inputs="text", outputs="text")
# iface.launch()
import torch
import gradio as gr
from peft import PeftModel, PeftConfig
from transformers import AutoModelForCausalLM, AutoTokenizer
device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu")
peft_model_id = "hyang0503/qlora-koalpaca"
config = PeftConfig.from_pretrained(peft_model_id)
model = AutoModelForCausalLM.from_pretrained(config.base_model_name_or_path)
model = PeftModel.from_pretrained(model, peft_model_id).to(device)
tokenizer = AutoTokenizer.from_pretrained(config.base_model_name_or_path)
def generate(q):
inputs = tokenizer(f"### 질문: {q}\n\n### 답변:", return_tensors='pt', return_token_type_ids=False)
outputs = model.generate(
**{k: v.to(device) for k, v in inputs.items()},
max_new_tokens=256,
do_sample=True,
eos_token_id=2,
)
result = tokenizer.decode(outputs[0])
answer_idx = result.find("### 답변:")
answer = result[answer_idx + 7:].strip()
return answer
gr.Interface(generate, "text", "text").launch(share=True) |