app.py

import gradio as gr

from langchain_community.llms import LlamaCpp
from langchain.prompts import PromptTemplate
from langchain.chains import LLMChain
from langchain_core.callbacks import StreamingStdOutCallbackHandler
from langchain.retrievers import TFIDFRetriever
from langchain.chains import RetrievalQA
from langchain.memory import ConversationBufferMemory
from langchain_community.chat_models import ChatLlamaCpp


callbacks = [StreamingStdOutCallbackHandler()]
print("creating ll started")
llm = ChatLlamaCpp(
    model_path="finbro-v0.1.0-llama-3-8B-instruct-1m.gguf",
    n_batch=8,
    temperature=0.85,
    max_tokens=256,
    top_p=0.95,
    top_k = 10,
    callback_manager=callbacks,
    n_ctx=2048,
    verbose=True,  # Verbose is required to pass to the callback manager
)
print("creating llm ended")






def greet(question, model_type):
    print(f"question is {question}")
    out_gen = "testsetestestetsetsets"
    return out_gen

demo = gr.Interface(fn=greet, inputs=["text", gr.Dropdown(
            ["With memory", "Without memory"], label="Memory status", info="With using memory, the output will be slow but strong"
        ),], outputs="text")
demo.launch(debug=True, share=True)


# import gradio as gr

# from langchain_community.llms import LlamaCpp
# from langchain.prompts import PromptTemplate
# from langchain.chains import LLMChain
# from langchain_core.callbacks import StreamingStdOutCallbackHandler
# from langchain.retrievers import TFIDFRetriever
# from langchain.chains import RetrievalQA
# from langchain.memory import ConversationBufferMemory
# from langchain_community.chat_models import ChatLlamaCpp

# callbacks = [StreamingStdOutCallbackHandler()]
# print("creating ll started")
# M_NAME = "finbro-v0.1.0-llama-3-8B-instruct-1m.gguf"
# llm = ChatLlamaCpp(
#     model_path=M_NAME,
#     n_batch=8,
#     temperature=0.85,
#     max_tokens=256,
#     top_p=0.95,
#     top_k = 10,
#     callback_manager=callbacks,
#     n_ctx=2048,
#     verbose=True,  # Verbose is required to pass to the callback manager
# )
# # print("creating ll ended")






# def greet(question, model_type):
#     print("prompt started ")
#     print(f"question is {question}")
#     template = """You are the Finiantial expert:
    
#     ### Instruction:
#     {question}
    
#     ### Input:
    
    
#     ### Response:
#     """
#     print("test1")
#     prompt = PromptTemplate(template=template, input_variables=["question"])
#     print("test2")
#     llm_chain_model = LLMChain(prompt=prompt, llm=llm)
#     print("test3")
#     out_gen = llm_chain_model.run(question)  
#     print("test4")
#     print(f"out is: {out_gen}")
#     return out_gen

# demo = gr.Interface(fn=greet, inputs=["text", gr.Dropdown(
#             ["Without memory", "With memory"], label="Memory status", info="With using memory, the output will be slow but strong"
#         ),], outputs="text")
# demo.launch(debug=True, share=True)