change embedding

confluence_qa.py

@@ -1,5 +1,5 @@
 from langchain.document_loaders import ConfluenceLoader
-from langchain.text_splitter import RecursiveCharacterTextSplitter,TokenTextSplitter
+from langchain.text_splitter import CharacterTextSplitter, TokenTextSplitter
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM,pipeline
 from langchain.llms.huggingface_pipeline import HuggingFacePipeline
 from langchain.prompts import PromptTemplate
@@ -14,8 +14,8 @@ class ConfluenceQA:
     def define_model(self) -> None:
         tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large")
         model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-large")
-        pipe = pipeline("text2text-generation", model=model, tokenizer=tokenizer, max_new_tokens=1024)
-        self.llm = HuggingFacePipeline(pipeline = pipe,model_kwargs={"temperature": 0, "max_length": 1024},)
+        pipe = pipeline("text2text-generation", model=model, tokenizer=tokenizer)
+        self.llm = HuggingFacePipeline(pipeline = pipe,model_kwargs={"temperature": 1},)
 
     def store_in_vector_db(self) -> None:
         persist_directory = self.config.get("persist_directory",None)
@@ -23,17 +23,19 @@ class ConfluenceQA:
         username = self.config.get("username",None)
         api_key = self.config.get("api_key",None)
         space_key = self.config.get("space_key",None)  
-        include_attachment = self.config.get("include_attachment", False)      
+        include_attachment = self.config.get("include_attachment", True)      
         loader = ConfluenceLoader(
             url=confluence_url, username=username, api_key=api_key
         )
-        documents = loader.load(include_attachments=include_attachment, limit=50, space_key=space_key)
-        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=150)
+        documents = loader.load(include_attachments=include_attachment, limit=100, space_key=space_key)
+        text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=10)
         documents = text_splitter.split_documents(documents)
-        # text_splitter = TokenTextSplitter(chunk_size=1000, chunk_overlap=10)  # This the encoding for text-embedding-ada-002
-        # texts = text_splitter.split_documents(texts)
+        print(documents)
+        text_splitter = TokenTextSplitter(chunk_size=1000, chunk_overlap=10, encoding_name="cl100k_base")  # This the encoding for text-embedding-ada-002
+        texts = text_splitter.split_documents(texts)
         self.db = Chroma.from_documents(documents, self.embeddings)
 
+
     def retrieve_qa_chain(self) -> None:
         template = """Use the following pieces of context to answer the question at the end. If you don't know the answer, just say that you don't know, don't try to make up an answer. Keep the answer as concise as possible.
         {context}