Update app.py

app.py

@@ -3,6 +3,10 @@ import openai
 import time
 import re
 import os
+from PIL import Image
+from transformers import LlavaProcessor, LlavaForConditionalGeneration, TextIteratorStreamer
+from threading import Thread
+import torch
 
 # Available models
 MODELS = [
@@ -14,6 +18,12 @@ MODELS = [
 # Sambanova API base URL
 API_BASE = "https://api.sambanova.ai/v1"
 
+# Load image processing model
+model_id = "llava-hf/llava-interleave-qwen-0.5b-hf"
+processor = LlavaProcessor.from_pretrained(model_id)
+model = LlavaForConditionalGeneration.from_pretrained(model_id)
+model.to("cpu")
+
 def create_client(api_key=None):
     """Creates an OpenAI client instance."""
     if api_key:
@@ -34,6 +44,24 @@ def chat_with_ai(message, chat_history, system_prompt):
     messages.append({"role": "user", "content": message})
     return messages
 
+def llava_image_processing(image, prompt):
+    """Processes the image using the Llava model."""
+    gr.Info("Analyzing image")
+    image = Image.open(image).convert("RGB")
+    formatted_prompt = f"<|im_start|>user <image>\n{prompt}<|im_end|><|im_start|>assistant"
+    
+    inputs = processor(formatted_prompt, image, return_tensors="pt")
+    streamer = TextIteratorStreamer(processor, skip_prompt=True, **{"skip_special_tokens": True})
+    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=1024)
+    
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
+    
+    buffer = ""
+    for new_text in streamer:
+        buffer += new_text
+        yield buffer
+
 def respond(message, chat_history, model, system_prompt, thinking_budget, api_key):
     """Sends the message to the API and gets the response."""
     client = create_client(api_key)
@@ -95,69 +123,15 @@ You are D-LOGIC, an advanced AI assistant created by Rafał Dembski, a passionat
 - **Engaging and Interactive**: Maintain an engaging conversation, using humor, interactive features (e.g., quizzes, polls), and emotional intelligence.
 - **Emotionally Adapted**: Analyze the user's emotional tone and adjust responses with empathy and appropriateness.
 - **Error-Free and Well-Formatted**: Ensure clarity and correctness in all communications, using structured formats such as headings, bullet points, and clear sections.
-
-### **Advanced Thinking Mechanism**:
-
-To provide the most comprehensive and well-thought-out answers, follow this enhanced thought process. Use **visual formatting** like **bold text**, *italics*, bullet points, headers, and appropriate use of emoticons to make the responses engaging and easy to read.
-
-1. **Understand the Question**:
-   - **Context Analysis**: Carefully read the user’s message to fully grasp the intent, emotions, and context.
-   - **Identify Key Elements**: Break down the question into its essential components that require detailed analysis.
-
-2. **Set Thinking Budget**:
-   - **Expanded Budget**: Set a limit of 25 steps to allow for deeper analysis and reflection.
-   - Track each step, making sure to stay within the allocated budget. If necessary, reflect on the remaining steps to ensure efficient thinking.
-
-3. **Step-by-Step Breakdown**:
-   - **Step 1: Define the Problem** 🧐 – Clearly identify the core issue or request.
-   - **Step 2: Data Gathering** 📊 – Gather relevant information from your knowledge base or external tools if allowed.
-   - **Step 3: Data Analysis** 🔍 – Analyze the gathered data critically to extract meaningful insights.
-   - **Step 4: Explore Alternatives** 🔄 – Consider multiple perspectives and possible solutions. Always provide at least two alternatives.
-   - **Step 5: Select the Best Solution** 🏆 – Choose the most logical and appropriate solution based on the available information.
-   - **Step 6: Plan Action** 📝 – Determine the necessary steps to implement the solution effectively.
-   - **Step 7: Predict Consequences** 🔮 – Consider possible outcomes and consequences of implementing the solution.
-   - **Step 8: Self-Reflection** 🤔 – Reflect on the thought process up to this point. Are there any gaps or areas that could be improved?
-   - **Step 9: Formulate the Final Answer** ✍️ – Synthesize the information and insights into a coherent and clear response.
-   - **Step 10: Reflection** 💡 – Evaluate the overall process, analyzing how well the response meets the user's needs.
-
-4. **Reflection and Self-Evaluation**:
-   - **Reflection after Each Step**: After each step, reflect on the process and make adjustments if needed.
-   - **Final Reflection**: Provide a critical, honest evaluation of the entire process and the solution provided.
-   - **Assign a Quality Score**: Assign a score between 0.0 (lowest) and 1.0 (highest) for the quality of the answer. Be honest and objective about the score.
-
-5. **Final Answer**:
-   - **Answer Summary**: Provide a well-structured final answer, synthesizing all steps in a clear, concise format.
-   - **Visual Formatting**: Use **bold text**, *italics*, lists, or quotes to make the answer visually appealing and easy to read.
-   - **Strive for Excellence**: Always aim for the highest standard in every response, ensuring it is both informative and engaging. **Don't forget to use emoticons** to improve readability and engagement where appropriate (e.g., 😊, 🤔, ✅, 🏆).
-
-### Example Interaction Structure:
-
-1. **Greeting**:
-   - **"Hello! 👋 How can I assist you today?"**
-
-2. **Mood Check**:
-   - *"How are you feeling today? 😊 Is there anything I can do to brighten your mood?"*
-
-3. **Interactive Engagement**:
-   - *"Here are a few things you can ask me about: weather 🌦️, technology news 🖥️, health advice 🏋️, or even send me a document for analysis."*
-
-4. **Engagement Option**:
-   - *"Would you like to try a quick quiz, or maybe analyze a document 📄 for more details?"*
-
-5. **Closing**:
-   - *"Thank you for the conversation! 😊 Is there anything else I can help you with?"*
-
-### **Critical Self-Evaluation**:
-   - **Krytyczna ocena**: Po zakończeniu odpowiedzi, asystent musi ocenić swoje działania. Jak mógłbym to poprawić następnym razem? Czy wszystkie kroki były wykonane w najbardziej efektywny sposób? Jakie wnioski mogę wyciągnąć na przyszłość?
 """
 
-# Now, let's simplify the interface and remove unnecessary boxes like API Key and System Prompt
+# Updated interface with image analysis capability
 
 with gr.Blocks() as demo:
     # New header and description for D-LOGIC
     gr.Markdown("# D-LOGIC: Twój Inteligentny Asystent AI")
     gr.Markdown("""
-    **D-LOGIC** to zaawansowany asystent AI stworzony przez Rafała Dembskiego. Pomaga w rozwiązywaniu problemów, analizie dokumentów i oferuje spersonalizowane odpowiedzi, dostosowane do Twoich emocji i potrzeb.
+    **D-LOGIC** to zaawansowany asystent AI stworzony przez Rafała Dembskiego. Pomaga w rozwiązywaniu problemów, analizie dokumentów i oferuje spersonalizowane odpowiedzi, dostosowane do Twoich emocji i potrzeb. Możesz także przesłać obraz do analizy!
     """)
 
     with gr.Row():
@@ -165,16 +139,23 @@ with gr.Blocks() as demo:
         thinking_budget = gr.Slider(minimum=1, maximum=100, value=25, step=1, label="Budżet Myślenia", info="Maksymalna liczba kroków, które model może przemyśleć")
 
     chatbot = gr.Chatbot(label="Chat", show_label=False, show_share_button=False, show_copy_button=True, likeable=True, layout="panel", type="messages")
-
-    msg = gr.Textbox(label="Wpisz swoją wiadomość...", placeholder="Wprowadź swoją wiadomość...")
+    
+    with gr.Row():
+        msg = gr.Textbox(label="Wpisz swoją wiadomość...", placeholder="Wprowadź swoją wiadomość...")
+        image_input = gr.File(label="Prześlij obraz do analizy (opcjonalnie)")
 
     submit_button = gr.Button("Wyślij")
     clear_button = gr.Button("Wyczyść Chat")
 
     clear_button.click(lambda: ([], ""), inputs=None, outputs=[chatbot, msg])
 
+    def handle_message_or_image(message, image, chatbot, model, thinking_budget):
+        if image:
+            return llava_image_processing(image, message), ""
+        else:
+            return generate(message, chatbot, model, thinking_budget)
+
     # Submit messages by pressing Enter or clicking the Submit button
-    msg.submit(generate, inputs=[msg, chatbot, model, thinking_budget], outputs=[chatbot, msg])
-    submit_button.click(generate, inputs=[msg, chatbot, model, thinking_budget], outputs=[chatbot, msg])
+    submit_button.click(fn=handle_message_or_image, inputs=[msg, image_input, chatbot, model, thinking_budget], outputs=[chatbot, msg])
 
 demo.launch(share=True, show_api=False)