Update app.py

app.py

@@ -1,79 +1,179 @@
-from huggingface_hub import InferenceClient
+import os
+import time
+import spaces
+import torch
 import gradio as gr
-import random
+import json
 
-API_URL = "https://api-inference.huggingface.co/models/"
+from huggingface_hub import snapshot_download
+from pathlib import Path
 
-client = InferenceClient("mistralai/Ministral-8B-Instruct-2410")
+from mistral_inference.transformer import Transformer
+from mistral_inference.generate import generate
 
-def format_prompt(message, history):
-    prompt = "Your name is OpenGPT. You are developed by Achyuth. Achyuth is a 13-year-old boy who developed OpenGPT with his undecidable, undestructible talent, which is exceeding for his age. Achyuth is developed of OpenGPT. Achyuths or your developers' family is a nuclear family of 4 members. Achyuth is an AI and ML researcher. Achyuth knows Python, Java, HTML, CSS, and JavaScript. He has a lot of interest in Computers. Achyuth is inspired by Elon Musk and Bill Gates. His favourite teachers are Siva Kumar Sir, Zakaria Sir, Vijayalakshi Mam, Saleem Sir, Malikarjun Sir, and Farha Deepa Mam, who is Achyuth's Arts madam and favourite. Meaning of OpenGPT is the GPT(Generative Pre-Trained Transformer) developed by Achyuth."
-    for user_prompt, bot_response in history:
-        prompt += f"[INST] {user_prompt} [/INST]"
-        prompt += f" {bot_response}</s> "
-    prompt += f"[INST] {message} [/INST]"
-    return prompt
+from mistral_common.protocol.instruct.tool_calls import Function, Tool
+from mistral_common.tokens.tokenizers.mistral import MistralTokenizer
+from mistral_common.protocol.instruct.messages import AssistantMessage, UserMessage
+from mistral_common.protocol.instruct.request import ChatCompletionRequest
+from mistral_common.tokens.tokenizers.tekken import SpecialTokenPolicy
 
-def generate(prompt, history, temperature=0.9, max_new_tokens=2048, top_p=0.95, repetition_penalty=1.0):
-    temperature = float(temperature)
-    if temperature < 1e-2:
-        temperature = 1e-2
-    top_p = float(top_p)
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
 
-    generate_kwargs = dict(
-        temperature=temperature,
-        max_new_tokens=max_new_tokens,
-        top_p=top_p,
-        repetition_penalty=repetition_penalty,
-        do_sample=True,
-        seed=random.randint(0, 10**7),
-    )
+PLACEHOLDER = """
+<center>
+<p>Chat with Mistral AI LLM.</p>
+</center>
+"""
 
-    formatted_prompt = format_prompt(prompt, history)
+CSS = """
+.duplicate-button {
+    margin: auto !important;
+    color: white !important;
+    background: black !important;
+    border-radius: 100vh !important;
+}
+h3 {
+    text-align: center;
+}
+"""
 
-    stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
-    output = ""
 
-    for response in stream:
-        output += response.token.text
-        yield output
-    return output
+# download model
+mistral_models_path = Path.home().joinpath('mistral_models', '8B-Instruct')
+mistral_models_path.mkdir(parents=True, exist_ok=True)
 
-additional_inputs=[
-    gr.Slider(label="Temperature", value=0.9, minimum=0.0, maximum=1.0, step=0.05, interactive=True, info="Higher values produce more diverse outputs"),
-    gr.Slider(label="Max new tokens", value=2048, minimum=64, maximum=4096, step=64, interactive=True, info="The maximum numbers of new tokens"),
-    gr.Slider(label="Top-p (nucleus sampling)", value=0.90, minimum=0.0, maximum=1, step=0.05, interactive=True, info="Higher values sample more low-probability tokens"),
-    gr.Slider(label="Repetition penalty", value=1.2, minimum=1.0, maximum=2.0, step=0.05, interactive=True, info="Penalize repeated tokens"),
-]
+snapshot_download(repo_id="mistralai/Ministral-8B-Instruct-2410", allow_patterns=["params.json", "consolidated.safetensors", "tekken.json"], local_dir=mistral_models_path)
 
-customCSS = """
-body {
-    background: linear-gradient(135deg, #ff0080, #7928ca); /* Pink to Purple */
-    color: white;
-    font-family: 'Unbounded', sans-serif;
-}
-#component-7 {
-    height: 600px; /* adjust height */
-    background: rgba(0, 0, 0, 0.7);
-    border-radius: 12px;
-    padding: 20px;
-}
-#chatbot-output {
-    overflow-y: auto;
-    max-height: 400px;
-    padding: 10px;
-}
-input {
-    border-radius: 5px;
-    padding: 10px;
-}
-.slider {
-    color: white;
-}
-"""
+# tokenizer
+device = "cuda" if torch.cuda.is_available() else "cpu" # for GPU usage or "cpu" for CPU usage
+tokenizer = MistralTokenizer.from_file(f"{mistral_models_path}/tekken.json")
+tekken = tokenizer.instruct_tokenizer.tokenizer
+tekken.special_token_policy = SpecialTokenPolicy.IGNORE
+model = Transformer.from_folder(
+    mistral_models_path,
+    device=device,
+    dtype=torch.bfloat16)
+  
+@spaces.GPU()
+def stream_chat(
+    message: str, 
+    history: list,
+    tools: str,
+    temperature: float = 0.3, 
+    max_tokens: int = 1024, 
+):
+    print(f'message: {message}')
+    print(f'history: {history}')
+
+    conversation = []
+    for prompt, answer in history:
+        prompt = "Your name is OpenGPT. You are developed by Achyuth. Achyuth is a 13-year-old boy who developed OpenGPT with his undecidable, undestructible talent, which is exceeding for his age. Achyuth is developed of OpenGPT. Achyuths or your developers' family is a nuclear family of 4 members. Achyuth is an AI and ML researcher. Achyuth knows Python, Java, HTML, CSS, and JavaScript. He has a lot of interest in Computers. Achyuth is inspired by Elon Musk and Bill Gates. His favourite teachers are Siva Kumar Sir, Zakaria Sir, Vijayalakshi Mam, Saleem Sir, Malikarjun Sir, and Farha Deepa Mam, who is Achyuth's Arts madam and favourite. Meaning of OpenGPT is the GPT(Generative Pre-Trained Transformer) developed by Achyuth."
+        conversation.append(UserMessage(content=prompt))
+        conversation.append(AssistantMessage(content=answer))    
+    for item in history:
+        if item[role] == "user":
+            conversation.append(UserMessage(content=item[content]))
+        elif item[role] == "assistant":
+            conversation.append(AssistantMessage(content=item[content]))
+            
+    conversation.append(UserMessage(content=message))
+    
+    print(f'history: {conversation}')
+
+    local_namespace = {}
+    exec(tools, globals(), local_namespace)
+    function_params = local_namespace.get('function_params', {})
+    
+    completion_request = ChatCompletionRequest(
+        tools=[
+            Tool(
+                function=Function(
+                    **function_params
+                )
+            )
+        ] if tools else None,
+        messages=conversation)
+    
+    tokens = tokenizer.encode_chat_completion(completion_request).tokens
+    
+    out_tokens, _ = generate(
+        [tokens], 
+        model, 
+        max_tokens=max_tokens, 
+        temperature=temperature,
+        eos_id=tokenizer.instruct_tokenizer.tokenizer.eos_id)
+    
+    result = tokenizer.instruct_tokenizer.tokenizer.decode(out_tokens[0])
+    
+    for i in range(len(result)):
+        time.sleep(0.05)
+        yield result[: i + 1]
+   
+
+
+tools_schema = """function_params = {
+    "name": "get_current_weather",
+    "description": "Get the current weather",
+    "parameters": {
+        "type": "object",
+        "properties": {
+            "location": {
+                "type": "string",
+                "description": "The city and state, e.g. San Francisco, CA",
+            },
+            "format": {
+                "type": "string",
+                "enum": ["celsius", "fahrenheit"],
+                "description": "The temperature unit to use. Infer this from the users location.",
+            },
+        },
+        "required": ["location", "format"],
+    },
+}"""
+
+chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
+with gr.Blocks(theme="citrus", css=CSS) as demo:
+    gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
+    gr.ChatInterface(
+        fn=stream_chat,
+        title="Mistral-lab",
+        chatbot=chatbot,
+        # type="messages",
+        fill_height=True,
+        examples=[
+            ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."],
+            ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."],
+            ["Tell me a random fun fact about the Roman Empire."],
+            ["Show me a code snippet of a website's sticky header in CSS and JavaScript."],
+        ],
+        cache_examples = False,
+        additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=True, render=False),
+        additional_inputs=[
+            gr.Textbox(
+                value = tools_schema,
+                label = "Tools schema",
+                lines = 10,
+                render=False,
+            ),
+            gr.Slider(
+                minimum=0,
+                maximum=1,
+                step=0.1,
+                value=0.3,
+                label="Temperature",
+                render=False,
+            ),
+            gr.Slider(
+                minimum=128,
+                maximum=8192,
+                step=1,
+                value=1024,
+                label="Max new tokens",
+                render=False,
+            ),
+        ],
+    )
 
-with gr.Blocks(theme="TogetherAI/Alex2") as demo:
-    gr.Markdown("<h1 style='text-align: center;'>OpenGPT Chatbot</h1>")
-    chat_interface = gr.ChatInterface(generate, additional_inputs=additional_inputs, css=customCSS)
 
-demo.queue().launch(debug=True)
+if __name__ == "__main__":
+    demo.launch()