Upload 2 files

app.py

@@ -0,0 +1,107 @@
+import torch
+from transformers import AutoTokenizer, AutoModelForCausalLM, TextIteratorStreamer, TextStreamer
+from threading import Thread
+import gradio as gr
+from peft import PeftModel
+
+model_name_or_path = "sarvamai/OpenHathi-7B-Hi-v0.1-Base"
+peft_model_id = "shuvom/OpenHathi-7B-FT-v0.1_SI"
+model = AutoModelForCausalLM.from_pretrained(model_name_or_path, load_in_4bit=True, device_map="auto")
+
+# tokenizer.chat_template = chat_template
+tokenizer = AutoTokenizer.from_pretrained(peft_model_id)
+# make embedding resizing configurable?
+model.resize_token_embeddings(len(tokenizer), pad_to_multiple_of=8)
+
+model = PeftModel.from_pretrained(model, peft_model_id)
+
+class ChatCompletion:
+  def __init__(self, model, tokenizer, system_prompt=None):
+    self.model = model
+    self.tokenizer = tokenizer
+    self.streamer = TextIteratorStreamer(self.tokenizer, skip_prompt=True)
+    self.print_streamer = TextStreamer(self.tokenizer, skip_prompt=True)
+    # set the model in inference mode
+    self.model.eval()
+    self.system_prompt = system_prompt
+
+  def get_completion(self, prompt, system_prompt=None, message_history=None, max_new_tokens=512, temperature=0.0):
+    if temperature < 1e-2:
+      temperature = 1e-2
+    messages = []
+    if message_history is not None:
+      messages.extend(message_history)
+    elif system_prompt or self.system_prompt:
+      system_prompt = system_prompt or self.system_prompt
+      messages.append({"role": "system", "content":system_prompt})
+    messages.append({"role": "user", "content": prompt})
+    chat_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+    inputs = self.tokenizer(chat_prompt, return_tensors="pt", add_special_tokens=False)
+    # Run the generation in a separate thread, so that we can fetch the generated text in a non-blocking way.
+    generation_kwargs = dict(max_new_tokens=max_new_tokens,
+                             temperature=temperature,
+                             top_p=0.95,
+                             do_sample=True,
+                             eos_token_id=tokenizer.eos_token_id,
+                             repetition_penalty=1.2
+                             )
+    generated_text = self.model.generate(**inputs, streamer=self.print_streamer, **generation_kwargs)
+    return generated_text
+
+  def get_chat_completion(self, message, history):
+    messages = []
+    if self.system_prompt:
+      messages.append({"role": "system", "content":self.system_prompt})
+    for user_message, assistant_message in history:
+        messages.append({"role": "user", "content": user_message})
+        messages.append({"role": "system", "content": assistant_message})
+    messages.append({"role": "user", "content": message})
+    chat_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+    inputs = self.tokenizer(chat_prompt, return_tensors="pt")
+    # Run the generation in a separate thread, so that we can fetch the generated text in a non-blocking way.
+    generation_kwargs = dict(inputs,
+                             streamer=self.streamer,
+                             max_new_tokens=2048,
+                             temperature=0.2,
+                             top_p=0.95,
+                             eos_token_id=tokenizer.eos_token_id,
+                             do_sample=True,
+                             repetition_penalty=1.2,
+                             )
+    thread = Thread(target=self.model.generate, kwargs=generation_kwargs)
+    thread.start()
+    generated_text = ""
+    for new_text in self.streamer:
+        generated_text += new_text.replace(self.tokenizer.eos_token, "")
+        yield generated_text
+    thread.join()
+    return generated_text
+
+  def get_completion_without_streaming(self, prompt, system_prompt=None, message_history=None, max_new_tokens=512, temperature=0.0):
+    if temperature < 1e-2:
+      temperature = 1e-2
+    messages = []
+    if message_history is not None:
+      messages.extend(message_history)
+    elif system_prompt or self.system_prompt:
+      system_prompt = system_prompt or self.system_prompt
+      messages.append({"role": "system", "content":system_prompt})
+    messages.append({"role": "user", "content": prompt})
+    chat_prompt = self.tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
+
+    inputs = self.tokenizer(chat_prompt, return_tensors="pt", add_special_tokens=False)
+    # Run the generation in a separate thread, so that we can fetch the generated text in a non-blocking way.
+    generation_kwargs = dict(max_new_tokens=max_new_tokens,
+                             temperature=temperature,
+                             top_p=0.95,
+                             do_sample=True,
+                             repetition_penalty=1.1)
+    outputs = self.model.generate(**inputs, **generation_kwargs)
+    generated_text = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
+    return generated_text
+
+text_generator = ChatCompletion(model, tokenizer, system_prompt="You are a native Hindi speaker who can converse at expert level in both Hindi and colloquial Hinglish.")
+
+gr.ChatInterface(text_generator.get_chat_completion).queue().launch(debug=True)

requirement.txt

@@ -0,0 +1,76 @@
+accelerate==0.25.0
+aiofiles==23.2.1
+altair==5.2.0
+annotated-types==0.6.0
+anyio==4.2.0
+attrs==23.2.0
+bitsandbytes==0.41.3.post2
+certifi==2023.11.17
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+contourpy==1.2.0
+cycler==0.12.1
+fastapi==0.108.0
+ffmpy==0.3.1
+filelock==3.13.1
+fonttools==4.47.0
+fsspec==2023.12.2
+gradio==4.12.0
+gradio_client==0.8.0
+h11==0.14.0
+httpcore==1.0.2
+httpx==0.26.0
+huggingface-hub==0.20.1
+idna==3.6
+importlib-resources==6.1.1
+Jinja2==3.1.2
+jsonschema==4.20.0
+jsonschema-specifications==2023.12.1
+kiwisolver==1.4.5
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+matplotlib==3.8.2
+mdurl==0.1.2
+mpmath==1.3.0
+networkx==3.2.1
+numpy==1.26.2
+orjson==3.9.10
+packaging==23.2
+pandas==2.1.4
+peft==0.7.1
+Pillow==10.1.0
+psutil==5.9.7
+pydantic==2.5.3
+pydantic_core==2.14.6
+pydub==0.25.1
+Pygments==2.17.2
+pyparsing==3.1.1
+python-dateutil==2.8.2
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+referencing==0.32.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.0
+rpds-py==0.16.2
+safetensors==0.4.1
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+starlette==0.32.0.post1
+sympy==1.12
+tokenizers==0.15.0
+tomlkit==0.12.0
+toolz==0.12.0
+torch==2.1.2
+tqdm==4.66.1
+transformers==4.36.2
+typer==0.9.0
+typing_extensions==4.9.0
+tzdata==2023.4
+urllib3==2.1.0
+uvicorn==0.25.0
+websockets==11.0.3