initial commit

app.py

@@ -0,0 +1,74 @@
+import streamlit as st
+import os
+import json
+import time
+
+# Import the ConfluenceQA class
+from confluence_qa import ConfluenceQA
+
+st.set_page_config(
+    page_title='Q&A Bot for Confluence Page',
+    page_icon='⚡',
+    layout='wide',
+    initial_sidebar_state='auto',
+)
+if "config" not in st.session_state:
+    st.session_state["config"] = {}
+if "confluence_qa" not in st.session_state:
+    st.session_state["confluence_qa"] = None
+
+@st.cache_resource
+def load_confluence(config):
+    # st.write("loading the confluence page")
+    confluence_qa = ConfluenceQA(config=config)
+    confluence_qa.init_embeddings()
+    confluence_qa.define_model()
+    confluence_qa.store_in_vector_db()
+    confluence_qa.retrieve_qa_chain()
+    return confluence_qa
+
+with st.sidebar.form(key ='Form1'):
+    st.markdown('## Add your configs')
+    confluence_url = st.text_input("paste the confluence URL", "https://templates.atlassian.net/wiki/")
+    username = st.text_input(label="confluence username",
+                             help="leave blank if confluence page is public",
+                             type="password")
+    space_key = st.text_input(label="confluence space",
+                             help="Space of Confluence",
+                             value="RD")
+    api_key = st.text_input(label="confluence api key",
+                            help="leave blank if confluence page is public",
+                            type="password")
+    submitted1 = st.form_submit_button(label='Submit')
+
+    # if submitted1 and confluence_url and space_key:
+    #     st.session_state["config"] = {
+    #         "persist_directory": None,
+    #         "confluence_url": confluence_url,
+    #         "username": username if username != "" else None,
+    #         "api_key": api_key if api_key != "" else None,
+    #         "space_key": space_key,
+    #     }
+    #     with st.spinner(text="Ingesting Confluence..."):
+    #         ### Hardcoding for https://templates.atlassian.net/wiki/ and space RD to avoid multiple OpenAI calls.
+    #         config = st.session_state["config"]
+    #         if  config["confluence_url"] == "https://templates.atlassian.net/wiki/" and config["space_key"] =="RD":
+    #             config["persist_directory"] = "chroma_db"
+    #         st.session_state["config"] = config
+
+    #         st.session_state["confluence_qa"]  = load_confluence(st.session_state["config"])
+    #     st.write("Confluence Space Ingested")
+        
+
+st.title("Confluence Q&A Demo")
+
+question = st.text_input('Ask a question', "How do I make a space public?")
+
+if st.button('Get Answer', key='button2'):
+    with st.spinner(text="Asking LLM..."):
+        confluence_qa = st.session_state.get("confluence_qa")
+        if confluence_qa is not None:
+            result = confluence_qa.answer_confluence(question)
+            st.write(result)
+        else:
+            st.write("Please load Confluence page first.")

confluence_qa.py

@@ -0,0 +1,52 @@
+from langchain.document_loaders import ConfluenceLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter,TokenTextSplitter
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM,pipeline
+from langchain import HuggingFacePipeline
+from langchain.prompts import PromptTemplate
+from langchain.chains import RetrievalQA
+from langchain.embeddings import HuggingFaceEmbeddings
+from langchain.vectorstores import FAISS
+
+class ConfluenceQA:
+    def init_embeddings(self) -> None:
+        self.embeddings = HuggingFaceEmbeddings(model_name="all-MiniLM-L6-v2")
+    
+    def define_model(self) -> None:
+        tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large")
+        model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-large")
+        pipe = pipeline("text2text-generation", model=model, tokenizer=tokenizer, max_new_tokens=1024)
+        self.llm = HuggingFacePipeline(pipeline = pipe,model_kwargs={"temperature": 0, "max_length": 1024},)
+
+    def store_in_vector_db(self) -> None:
+        config = self.config        
+        loader = ConfluenceLoader(
+            url=config.url, username=config.username, api_key=config.apiKey
+        )
+        documents = loader.load(include_attachments=config.includeAttachements, limit=50, page_ids=config.page_ids)
+        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
+        documents = text_splitter.split_documents(documents)
+        # text_splitter = TokenTextSplitter(chunk_size=1000, chunk_overlap=10)  # This the encoding for text-embedding-ada-002
+        # texts = text_splitter.split_documents(texts)
+        self.db = FAISS.from_documents(documents, self.embeddings)
+
+    def retrieve_qa_chain(self) -> None:
+        template = """Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. Keep the answer as concise as possible.
+        {context}
+        Question: {question}
+        Helpful Answer:"""
+        QA_CHAIN_PROMPT = PromptTemplate(
+            template=template, input_variables=["context", "question"]
+        )
+        chain_type_kwargs = {"prompt": QA_CHAIN_PROMPT}
+        self.qa = RetrievalQA.from_chain_type(llm=self.llm, chain_type="stuff", retriever=self.db.as_retriever(), chain_type_kwargs=chain_type_kwargs)
+
+    def __init__(self,config) -> None:
+        self.db=None
+        self.embeddings=None
+        self.llm=None
+        self.config=config
+        self.qa=None
+
+    def qa_bot(self, query:str):
+        result = self.qa.run(query)
+        return result

requirements.txt

@@ -0,0 +1,12 @@
+langchain 
+torch 
+transformers 
+faiss-cpu 
+pypdf 
+sentence-transformers 
+atlassian-python-api 
+pytesseract 
+reportlab 
+Pillow 
+svglib 
+tiktoken