import spaces import json import subprocess from llama_cpp import Llama from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType from llama_cpp_agent.providers import LlamaCppPythonProvider from llama_cpp_agent.chat_history import BasicChatHistory from llama_cpp_agent.chat_history.messages import Roles import gradio as gr from huggingface_hub import hf_hub_download, snapshot_download import os #### space_link = "https://huggingface.co/spaces/Weyaxi/chat" all_repo_ggufs = ["bartowski/Einstein-v6.1-Llama3-8B-GGUF", "LoneStriker/Einstein-v4-7B-GGUF"] text = "Chat with various Einstein models based on various base models. \n\nSelect your model in the additional input place", ### for gguf_model_id in all_repo_ggufs: download_gguf_model_dir = snapshot_download(repo_id=gguf_model_id, allow_patterns=["*Q6_*.gguf", "*Q5_*.gguf"], local_dir = "./models") files = [f for f in os.listdir(download_gguf_model_dir) if os.path.isfile(os.path.join(download_gguf_model_dir, f))] css = """ .message-row { justify-content: space-evenly !important; } .message-bubble-border { border-radius: 6px !important; } .dark.message-bubble-border { border-color: #180e25 !important; } .dark.user { background: #120a20 !important; } .dark.assistant { background: transparent !important; } """ PLACEHOLDER = f"""
Logo

Cognitive Computations

{text}

Einstein collection
Discord GitHub
""" @spaces.GPU(duration=120) def respond( message, history: list[tuple[str, str]], model, max_tokens, temperature, top_p, top_k, repeat_penalty, ): from llama_cpp import Llama from llama_cpp_agent import LlamaCppAgent from llama_cpp_agent import MessagesFormatterType from llama_cpp_agent.providers import LlamaCppPythonProvider from llama_cpp_agent.chat_history import BasicChatHistory from llama_cpp_agent.chat_history.messages import Roles llm = Llama( model_path=f"models/{model}", flash_attn=True, n_threads=40, n_gpu_layers=81, n_batch=1024, n_ctx=8192, ) provider = LlamaCppPythonProvider(llm) agent = LlamaCppAgent( provider, system_prompt="You are Einstein an AI assistant that helps humanity.", predefined_messages_formatter_type=MessagesFormatterType.CHATML, debug_output=True ) settings = provider.get_provider_default_settings() settings.temperature = temperature settings.top_k = top_k settings.top_p = top_p settings.max_tokens = max_tokens settings.repeat_penalty = repeat_penalty settings.stream = True messages = BasicChatHistory() for msn in history: user = { 'role': Roles.user, 'content': msn[0] } assistant = { 'role': Roles.assistant, 'content': msn[1] } messages.add_message(user) messages.add_message(assistant) stream = agent.get_chat_response(message, llm_sampling_settings=settings, chat_history=messages, returns_streaming_generator=True, print_output=False) outputs = "" for output in stream: outputs += output yield outputs demo = gr.ChatInterface( respond, additional_inputs=[ gr.Dropdown(files, value=files[0], label="Model"), gr.Slider(minimum=1, maximum=8192, value=8192, step=1, label="Max tokens"), gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"), gr.Slider( minimum=0.1, maximum=1.0, value=0.95, step=0.05, label="Top-p", ), gr.Slider( minimum=0, maximum=100, value=40, step=1, label="Top-k", ), gr.Slider( minimum=0.0, maximum=2.0, value=1.1, step=0.1, label="Repetition penalty", ), ], theme=gr.themes.Soft(primary_hue="indigo", secondary_hue="blue", neutral_hue="gray",font=[gr.themes.GoogleFont("Exo"), "ui-sans-serif", "system-ui", "sans-serif"]).set( body_background_fill_dark="#0c0615", block_background_fill_dark="#0c0615", block_border_width="1px", block_title_background_fill_dark="#1a1127", input_background_fill_dark="#1a1127", button_secondary_background_fill_dark="#1a1127", border_color_primary_dark="#180e25", background_fill_secondary_dark="#0c0615", color_accent_soft_dark="transparent" ), css=css, retry_btn="Retry", undo_btn="Undo", clear_btn="Clear", submit_btn="Send", title="Chat with Einstein 🧑‍🔬 series", chatbot=gr.Chatbot(scale=1, placeholder=PLACEHOLDER) ) if __name__ == "__main__": demo.launch()