Update app.py

app.py

@@ -1,3 +1,32 @@
 import gradio as gr
+from transformers import AutoTokenizer, AutoModelForCausalLM
+import torch
 
-gr.load("models/mattshumer/Reflection-Llama-3.1-70B").launch()
+# Load the model and tokenizer
+model_name = "mattshumer/Reflection-Llama-3.1-70B"
+tokenizer = AutoTokenizer.from_pretrained(model_name)
+model = AutoModelForCausalLM.from_pretrained(model_name, torch_dtype=torch.float16, device_map="auto")
+
+def generate_response(message, history):
+    # Combine history and new message
+    prompt = "\n".join([f"Human: {h[0]}\nAI: {h[1]}" for h in history])
+    prompt += f"\nHuman: {message}\nAI:"
+    
+    # Tokenize and generate
+    inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+    outputs = model.generate(**inputs, max_new_tokens=500, temperature=0.7)
+    response = tokenizer.decode(outputs[0], skip_special_tokens=True)
+    
+    # Extract only the AI's response
+    ai_response = response.split("AI:")[-1].strip()
+    return ai_response
+
+# Create the Gradio interface
+iface = gr.ChatInterface(
+    fn=generate_response,
+    title="Chat with Reflection-Llama-3.1-70B",
+    description="Ask me anything!",
+)
+
+# Launch the interface
+iface.launch()