Update app.py

app.py

@@ -1,90 +1,3 @@
-import gradio as gr
-
-from langchain_community.llms import LlamaCpp
-from langchain.prompts import PromptTemplate
-from langchain.chains import LLMChain
-from langchain_core.callbacks import StreamingStdOutCallbackHandler
-from langchain.retrievers import TFIDFRetriever
-from langchain.chains import RetrievalQA
-from langchain.memory import ConversationBufferMemory
-
-
-callbacks = [StreamingStdOutCallbackHandler()]
-print("creating ll started")
-llm = LlamaCpp(
-    model_path="finbrov1.gguf",
-    temperature=0.75,
-    max_tokens=100,
-    top_p=4,
-    callback_manager=callbacks,
-    verbose=True,  # Verbose is required to pass to the callback manager
-)
-print("creating ll ended")
-
-
-
-
-
-
-def greet(question, model_type):
-    print(f"question is {question}")
-    if model_type == "With memory":
-        retriever = TFIDFRetriever.from_texts(
-            ["Finatial AI"])
-        
-        
-        template = """You are the Finiantial expert:
-        {history}
-        {context}
-        ### Instruction:
-        {question}
-        
-        ### Input:
-        
-        
-        ### Response:
-        """
-        
-        prompt1 = PromptTemplate(
-            input_variables=["history", "context", "question"],
-            template=template,
-        )
-        
-        llm_chain_model = RetrievalQA.from_chain_type(
-            llm=llm,
-            chain_type='stuff',
-            retriever=retriever,
-            verbose=False,
-            chain_type_kwargs={
-                "verbose": False,
-                "prompt": prompt1,
-                "memory": ConversationBufferMemory(
-                    memory_key="history",
-                    input_key="question"),
-            }
-        )
-        print("creating model created")
-    else:
-        template = """You are the Finiantial expert:
-        ### Instruction:
-        {question}
-        ### Input:
-        ### Response:
-        """
-        
-        prompt = PromptTemplate(template=template, input_variables=["question"])
-        
-        llm_chain_model = LLMChain(prompt=prompt, llm=llm)
-    out_gen = llm_chain_model.run(question)  
-    print(f"out is: {out_gen}")
-    return out_gen
-
-demo = gr.Interface(fn=greet, inputs=["text", gr.Dropdown(
-            ["With memory", "Without memory"], label="Memory status", info="With using memory, the output will be slow but strong"
-        ),], outputs="text")
-demo.launch(debug=True, share=True)
-
-
 # import gradio as gr
 
 # from langchain_community.llms import LlamaCpp
@@ -96,11 +9,11 @@ demo.launch(debug=True, share=True)
 # from langchain.memory import ConversationBufferMemory
 # from langchain_community.chat_models import ChatLlamaCpp
 
+
 # callbacks = [StreamingStdOutCallbackHandler()]
 # print("creating ll started")
-# M_NAME = "taddeusb90_finbro-v0.1.0-dolphin-2.9-llama-3-8B-instruct-131k_adapt_basic_model_16bit.gguf"
-# llm = LlamaCpp(
-#     model_path=M_NAME,
+# llm = ChatLlamaCpp(
+#     model_path="finbro-v0.1.0-llama-3-8B-instruct-1m.gguf",
 #     n_batch=8,
 #     temperature=0.85,
 #     max_tokens=256,
@@ -110,7 +23,7 @@ demo.launch(debug=True, share=True)
 #     n_ctx=2048,
 #     verbose=True,  # Verbose is required to pass to the callback manager
 # )
-# # print("creating ll ended")
+# print("creating llm ended")
 
 
 
@@ -118,29 +31,120 @@ demo.launch(debug=True, share=True)
 
 
 # def greet(question, model_type):
-#     print("prompt started ")
 #     print(f"question is {question}")
-#     template = """You are the Finiantial expert:
-    
-#     ### Instruction:
-#     {question}
-    
-#     ### Input:
-    
-    
-#     ### Response:
-#     """
-#     print("test1")
-#     prompt = PromptTemplate(template=template, input_variables=["question"])
-#     print("test2")
-#     llm_chain_model = LLMChain(prompt=prompt, llm=llm)
-#     print("test3")
+#     if model_type == "With memory":
+#         retriever = TFIDFRetriever.from_texts(
+#             ["Finatial AI"])
+        
+        
+#         template = """You are the Finiantial expert:
+#         {history}
+#         {context}
+#         ### Instruction:
+#         {question}
+        
+#         ### Input:
+        
+        
+#         ### Response:
+#         """
+        
+#         prompt1 = PromptTemplate(
+#             input_variables=["history", "context", "question"],
+#             template=template,
+#         )
+        
+#         llm_chain_model = RetrievalQA.from_chain_type(
+#             llm=llm,
+#             chain_type='stuff',
+#             retriever=retriever,
+#             verbose=False,
+#             chain_type_kwargs={
+#                 "verbose": False,
+#                 "prompt": prompt1,
+#                 "memory": ConversationBufferMemory(
+#                     memory_key="history",
+#                     input_key="question"),
+#             }
+#         )
+#         print("creating model created")
+#     else:
+#         template = """You are the Finiantial expert:
+#         ### Instruction:
+#         {question}
+#         ### Input:
+#         ### Response:
+#         """
+        
+#         prompt = PromptTemplate(template=template, input_variables=["question"])
+        
+#         llm_chain_model = LLMChain(prompt=prompt, llm=llm)
 #     out_gen = llm_chain_model.run(question)  
-#     print("test4")
 #     print(f"out is: {out_gen}")
 #     return out_gen
 
 # demo = gr.Interface(fn=greet, inputs=["text", gr.Dropdown(
-#             ["Without memory", "With memory"], label="Memory status", info="With using memory, the output will be slow but strong"
+#             ["With memory", "Without memory"], label="Memory status", info="With using memory, the output will be slow but strong"
 #         ),], outputs="text")
-# demo.launch(debug=True, share=True)
+# demo.launch(debug=True, share=True)
+
+
+import gradio as gr
+
+from langchain_community.llms import LlamaCpp
+from langchain.prompts import PromptTemplate
+from langchain.chains import LLMChain
+from langchain_core.callbacks import StreamingStdOutCallbackHandler
+from langchain.retrievers import TFIDFRetriever
+from langchain.chains import RetrievalQA
+from langchain.memory import ConversationBufferMemory
+from langchain_community.chat_models import ChatLlamaCpp
+
+callbacks = [StreamingStdOutCallbackHandler()]
+print("creating ll started")
+M_NAME = "finbro-v0.1.0-llama-3-8B-instruct-1m.gguf"
+llm = ChatLlamaCpp(
+    model_path=M_NAME,
+    n_batch=8,
+    temperature=0.85,
+    max_tokens=256,
+    top_p=0.95,
+    top_k = 10,
+    callback_manager=callbacks,
+    n_ctx=2048,
+    verbose=True,  # Verbose is required to pass to the callback manager
+)
+# print("creating ll ended")
+
+
+
+
+
+
+def greet(question, model_type):
+    print("prompt started ")
+    print(f"question is {question}")
+    template = """You are the Finiantial expert:
+    
+    ### Instruction:
+    {question}
+    
+    ### Input:
+    
+    
+    ### Response:
+    """
+    print("test1")
+    prompt = PromptTemplate(template=template, input_variables=["question"])
+    print("test2")
+    llm_chain_model = LLMChain(prompt=prompt, llm=llm)
+    print("test3")
+    out_gen = llm_chain_model.run(question)  
+    print("test4")
+    print(f"out is: {out_gen}")
+    return out_gen
+
+demo = gr.Interface(fn=greet, inputs=["text", gr.Dropdown(
+            ["Without memory", "With memory"], label="Memory status", info="With using memory, the output will be slow but strong"
+        ),], outputs="text")
+demo.launch(debug=True, share=True)