Update app.py

app.py

@@ -1,18 +1,22 @@
 import gradio as gr
-from transformers import AutoModelForCausalLM, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
 from huggingface_hub import InferenceClient
 import os
 # Environment variable for HF token
 hf_token = os.environ.get("HF_TOKEN")
 
 # Your model ID
-model_id = "mistralai/Mistral-7B-Instruct-v0.1"
+model_id = "mistralai/Mistral-7B-Instruct-v0.2"
 
 
-tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_token)
+quantization_config = BitsAndBytesConfig(
+    load_in_4bit=True,
+    bnb_4bit_quant_type="nf4",
+    bnb_4bit_compute_dtype="torch.float16",
+)
 
-# Load the model
-model = AutoModelForCausalLM.from_pretrained(model_id, token=hf_token, device_map="auto")
+model = AutoModelForCausalLM.from_pretrained(model_id, quantization_config=True, device_map="auto", token=hf_token)
+tokenizer = AutoTokenizer.from_pretrained(model_id, token=hf_token)
 
 def format_prompt(message, history):
   prompt = "<s>"