edit

confluence_qa.py

@@ -1,5 +1,5 @@
 from langchain.document_loaders import ConfluenceLoader
-from langchain.text_splitter import CharacterTextSplitter, TokenTextSplitter,RecursiveCharacterTextSplitter
+from langchain.text_splitter import CharacterTextSplitter, TokenTextSplitter,RecursiveCharacterTextSplitter,SentenceTransformersTokenTextSplitter
 from transformers import AutoTokenizer, AutoModelForSeq2SeqLM,pipeline
 from langchain.llms.huggingface_pipeline import HuggingFacePipeline
 from langchain.prompts import PromptTemplate
@@ -28,12 +28,14 @@ class ConfluenceQA:
             url=confluence_url, username=username, api_key=api_key
         )
         documents = loader.load(include_attachments=include_attachment, limit=100, space_key=space_key)
-        text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=10)
+        # text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=10)
+        # documents = text_splitter.split_documents(documents)
+        # print(documents)
+        text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=10)
+        documents = text_splitter.split_documents(documents)
+        text_splitter = SentenceTransformersTokenTextSplitter(chunck_size=1000,chunk_overlap=10)
         documents = text_splitter.split_documents(documents)
         print(documents)
-        # text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=10)
-        # documents = text_splitter.split_documents(documents)
-        
         # text_splitter = TokenTextSplitter(chunk_size=1000, chunk_overlap=10, encoding_name="cl100k_base")  # This the encoding for text-embedding-ada-002
         # documents = text_splitter.split_documents(documents)
         self.db = Chroma.from_documents(documents, self.embeddings)