added S3 locations to example.env

expanded test use case in test_this.py
generate summaries in German now

example.env

@@ -3,13 +3,14 @@ HUGGINGFACEHUB_API_TOKEN=""
 GOOGLE_CSE_ID=""
 GOOGLE_API_KEY=""
 
-# AWS S3 object storage 
-S3_LOCATION=""
-S3_FILE_NAME=""
-
-# Local vectorstore storage
-FAISS_INDEX_PATH = ""
+# Vectorstore storage on S3 and locally
+S3_LOCATION="rad-rag-demos"
+FAISS_VS_NAME="vectorstores/faiss-insurance-agent-500.zip"
+CHROMA_VS_NAME=""
+FAISS_INDEX_PATH = "./vectorstore/faiss-insurance-agent-500"
+CHROMA_DIRECTORY = "./vectorstore/chroma-insurance-agent-500"
 
 # llm and embedding models
-embedding_model=""
-llm_model=""
+EMBEDDING_MODEL="sentence-transformers/multi-qa-mpnet-base-dot-v1"
+LLM_MODEL="mistralai/Mixtral-8x7B-Instruct-v0.1"
+LLM_MODEL_ARGS=

rag_app/generate_summary.py

@@ -24,10 +24,10 @@ def generate_keywords(document:dict,
     
     template = (
         """
-        You are a SEO expert bot. Your task is to craft a meaningful list of 5 keywords to organize documents. 
-        The keywords should help us in searching and retrieving the documents later.
+        Du bist ein SEO-Experten-Bot. Deine Aufgabe ist es, 5 aussagekräftige Schlüsselwörtern zu erstellen, um Dokumente zu organisieren.
+        Die Schlüsselwörter sollen uns später beim Suchen, Filtern und Abrufen der Dokumente helfen.
 
-        You will only respond with the clear, concise and meaningful 5 of keywords separated by comma. 
+        Antworte nur mit 5 klaren, prägnanten und aussagekräftigen Schlüsselwörtern, getrennt durch Kommas.
           
         <<<
         Document: {document}
@@ -49,10 +49,10 @@ def generate_description(document:dict,
     
     template = (
         """
-        You are a SEO expert bot. Your task is to craft a meaningful summary to descripe and organize documents. 
-        The description should be a meaningful summary of the document's content and help us in searching and retrieving the documents later.
+        Du bist ein SEO-Experten-Bot. Deine Aufgabe ist es, eine aussagekräftige Zusammenfassung zu erstellen, um Dokumente zu beschreiben und zu organisieren.
+        Die Beschreibung sollte eine aussagekräftige Zusammenfassung des Dokumentinhalts sein und uns später beim Suchen und Abrufen der Dokumente helfen.
 
-        You will only respond with the clear, concise and meaningful description. 
+        Antworte nur mit einer klaren, prägnanten und aussagekräftigen Beschreibung in Deutsch.
           
         <<<
         Document: {document}

rag_app/handle_vector_store.py

@@ -29,11 +29,11 @@ def build_vector_store(
         FAISS_INDEX_PATH = db_path
 
     embeddings,chunks = create_embeddings(docs, chunk_size, chunk_overlap, embedding_model)
-    for chunk in chunks:
-        keywords=generate_keywords(chunk)
-        description=generate_description(chunk)
-        chunk.metadata['chunk_keywords']=keywords
-        chunk.metadata['chunk_description']=description
+    # for chunk in chunks:
+    #     keywords=generate_keywords(chunk)
+    #     description=generate_description(chunk)
+    #     chunk.metadata['keywords']=keywords
+    #     chunk.metadata['description']=description
 
     #load chunks into vector store
     print(f'Loading chunks into faiss vector store ...')

test_this.py

@@ -3,13 +3,19 @@ from rag_app.create_embedding import create_embeddings
 from rag_app.generate_summary import generate_description, generate_keywords
 from rag_app.handle_vector_store import build_vector_store
 
-docs = load_docs_from_urls(["https://www.wuerttembergische.de/"],5)
+# 1. load the urls
+# 2. build the vectorstore -> the function will create the chunking and embeddings
+# 3. initialize the db retriever
+# 4. 
 
-for doc in docs:
-    keywords=generate_keywords(doc)
-    description=generate_description(doc)
-    doc.metadata['keywords']=keywords
-    doc.metadata['description']=description
+docs = load_docs_from_urls(["https://www.wuerttembergische.de/"],6)
+
+# for doc in docs:
+#     keywords=generate_keywords(doc)
+#     description=generate_description(doc)
+#     doc.metadata['keywords']=keywords
+#     doc.metadata['description']=description
+#     print(doc.metadata)
 
 build_vector_store(docs, './vectorstore/faiss-insurance-agent-1500','sentence-transformers/multi-qa-mpnet-base-dot-v1',True,1500,150)