vptq

app.py

@@ -4,7 +4,11 @@ from huggingface_hub import InferenceClient
 """
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
-client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
+#VPTQ-community/Qwen2.5-72B-Instruct-v8-k65536-0-woft
+
+from vptq.app_utils import get_chat_loop_generator
+
+chat_completion = get_chat_loop_generator("VPTQ-community/Meta-Llama-3.1-70B-Instruct-v8-k32768-0-woft")
 
 
 def respond(
@@ -27,18 +31,20 @@ def respond(
 
     response = ""
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
+    for message in chat_completion(
+            messages,
+            max_tokens=max_tokens,
+            stream=True,
+            temperature=temperature,
+            top_p=top_p,
     ):
-        token = message.choices[0].delta.content
+        token = message
 
         response += token
         yield response
 
+
+
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 """

requirements.txt

@@ -1 +1,2 @@
-huggingface_hub==0.22.2
+huggingface_hub==0.22.2
+https://github.com/microsoft/VPTQ/releases/download/v0.0.1/vptq-0.0.1-cp310-cp310-manylinux1_x86_64.whl