rag_study_merge2 (#2)

- Adding rag study updates (148b40923b7125d9750b4c2e8e058ccb3b4b5efb)

.gitattributes

@@ -32,4 +32,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
+*.pdf filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,3 +1,10 @@
+
+.env
+*.log
+# *.pdf
+*.DS_Store
+.ragatouille/
 .venv/
-__pycache__/
-.env
+db/
+scripts/__pycache__
+scripts/tmp_trainer

Dockerfile

@@ -2,22 +2,24 @@
 FROM python:3.11.1
 
 # Set the working directory in the container
-WORKDIR /app
+WORKDIR /usr/src/app
 
 # Install poetry
-# RUN pip3 install poetry==1.7.1
+RUN pip3 install poetry
 
-# Copy the current directory contents into the container at /usr/src/app
-COPY . .
+# Copy only the necessary files for installing dependencies
+COPY pyproject.toml poetry.lock ./
 
-# Install dependencies
-# RUN poetry config virtualenvs.create false \
-#     && poetry install --no-interaction --no-ansi
-# Streamlit must be installed separately. Potentially this will cause an issue with dependencies in the future, but it's the only way it works.
-# RUN pip3 install streamlit  
+# Disable virtual environments creation by Poetry
+# as the Docker container itself is an isolated environment
+RUN poetry config virtualenvs.create false
 
 #  Install dependencies
-RUN pip3 install -r requirements.txt
+# RUN pip3 install -r requirements.txt
+RUN poetry install
+
+# Copy the current directory contents into the container at /usr/src/app
+COPY . .
 
 # Make a port available to the world outside this container
 # The EXPOSE instruction informs Docker that the container listens on the specified network ports at runtime. Your container needs to listen to Streamlit’s (default) port 8501.
@@ -27,7 +29,7 @@ EXPOSE 8501
 HEALTHCHECK CMD curl --fail http://localhost:8501/_stcore/health
 
 # An ENTRYPOINT allows you to configure a container that will run as an executable. Here, it also contains the entire streamlit run command for your app, so you don’t have to call it from the command line
-ENTRYPOINT ["streamlit", "run", "app.py", "--server.port=8501", "--server.address=0.0.0.0"]
+ENTRYPOINT ["streamlit", "run", "Start.py", "--server.port=8501", "--server.address=0.0.0.0"]
 
 # Execute with:
 # docker build -t <image_name> .    

README.md

@@ -8,4 +8,10 @@ pinned: false
 app_port: 8501
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# aerospace_chatbot
+Aerospace discipline-specific chatbots and AI tools.
+
+## Dependencies
+Dependencies are managed with [poetry](https://python-poetry.org/). Detailed install instructions are located [here](https://www.evernote.com/shard/s84/sh/f37de730-ce37-cd28-789c-86c3dc024a7c/90VLNref38KARua10p4am7IZkwsOxo93fXuBNqba-HpeIkMqGpRZrRkmjw)
+* Once poetry is installed, run the following to install all dependencies: <code>poetry install</code>
+* poetry.lock and pyproject.toml are committed to this directory and are the working dependencies.

Start.py

@@ -0,0 +1,27 @@
+import streamlit as st
+import os
+
+# Set up page
+st.set_page_config(
+    page_title="Aerospace Chatbot: AMS",
+)
+st.title("Aerospace Chatbot Homepage")
+st.markdown("Code base: https://github.com/dsmueller3760/aerospace_chatbot/tree/rag_study")
+st.markdown('---')
+st.title("Chatbots")
+st.markdown("""
+Chatbots for aerospace mechanisms symposia, using all available papers published since 2000
+* Modular version meant to study retrieval methods
+""")
+st.subheader("AMS")
+'''
+This chatbot will look up from all Aerospace Mechanism Symposia in the following location: https://github.com/dsmueller3760/aerospace_chatbot/tree/main/data/AMS
+* Available models: https://platform.openai.com/docs/models
+* Model parameters: https://platform.openai.com/docs/api-reference/chat/create
+* Pinecone: https://docs.pinecone.io/docs/projects#api-keys
+* OpenAI API: https://platform.openai.com/api-keys
+'''
+
+# # Establish secrets
+# PINECONE_ENVIRONMENT=os.getenv('PINECONE_ENVIRONMENT')
+# PINECONE_API_KEY=os.getenv('PINECONE_API_KEY')

config/config.json

@@ -0,0 +1,44 @@
+{
+    "databases": [
+        {
+            "name": "Pinecone",
+            "embedding_models": ["Openai", "Voyage"]
+        },
+        {
+            "name": "ChromaDB",
+            "embedding_models": ["Openai"]
+        },
+        {
+            "name": "RAGatouille",
+            "hf_rag_models": [
+                "colbert-ir/colbertv2.0"
+            ]
+        }
+    ],
+    "llms": [
+        {
+            "name": "OpenAI",
+            "models": [   
+                "gpt-3.5-turbo-1106",
+                "gpt-3.5-turbo-instruct",
+                "gpt-4",
+                "gpt-4-32k",
+                "gpt-4-1106-preview"
+            ]
+        },
+        {
+            "name": "Hugging Face",
+            "models": [
+                "mistralai/Mixtral-8x7B-Instruct-v0.1",
+                "ai-aerospace/autotrain-ams_v0.1_100_Mistral-7B-Instruct-v0.1",
+                "meta-llama/Llama-2-7b-chat-hf"
+            ]
+        }
+    ],
+    "rag_types": [
+        "Standard",
+        "Parent-Child",
+        "Hypothetical Questions",
+        "Summaries"
+    ]
+}

config/index_data.json

@@ -0,0 +1,13 @@
+{
+    "Pinecone": {
+        "Openai": "pinecone-openai-ams",
+        "Voyage": "pinecone-voyage-ams"
+    },
+    "ChromaDB": {
+        "Openai": "chromadb-openai-ams",
+        "Voyage": "chromadb-voyage-ams"
+    },
+    "RAGatouille": {
+        "colbert-ir/colbertv2.0": "RAGatouille-colbertv2.0-ams"
+    }
+}

data/AMS/AMS_1996.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3626fd4a0769b8a73a12ee79a1bec7c264c541a5bf90df6f6c13c1ff00011b24
+size 152158068

data/AMS/AMS_1997.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:34442bbc794415ea8d778ebd57e1dd368e20c5e6f65aff35fa008af54dbb900a
+size 22719877

data/AMS/AMS_1998.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1523ca03cd1254b81cd0cb285182b7ac40208cba7932972ca00e0942e43f3539
+size 122280718

data/AMS/AMS_1999.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c631364761565d749e6bafb0ab1e84611e773ccdb640ab08f6b32b1fcc49e1e
+size 27850919

data/AMS/AMS_2000.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddf89c5cd9ddbd225e77198b19274535d4f003fdc20b5823239f51ad48230549
+size 24061146

data/AMS/AMS_2001.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c63b2bba5a892759a7298097ee2388f353cc974285a73bfd8635d48af9f7d945
+size 23264984

data/AMS/AMS_2002.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b8b60c30ea9843face46e021a80bd1072901596b8e0f98a63601b31ecac2076
+size 41615570

data/AMS/AMS_2004.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:986a7f046ba336d35d9d0db974931940543d612dad2c9bb6d5976d778777b659
+size 28914300

data/AMS/AMS_2006.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:af4fb8e67c1ebf7b51fddd947d531d68ab05ff187fe915528811676ae0083d55
+size 61039456

data/AMS/AMS_2008.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d74dcd8ef68ae324f9246a35e1ccf538c4fd676d8b1ae733191c8ad6a055c90
+size 31961158

data/AMS/AMS_2010.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30d9ad0b75d0d41c75926dd97361f1548b79920df61d8d7486978d4b69a00ef6
+size 30161812

data/AMS/AMS_2012.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e26a981f74c9d0c3526ad5152c83ad9fabde8f197f69cb24a0fd1d4004c1f026
+size 31088140

data/AMS/AMS_2014.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:73dea6c8c45d0103404e3e3bd764e6efcd0f5bf5f45d505ce98e6c07528d9322
+size 35199422

data/AMS/AMS_2016.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f7d8a0e558abd59b94abcbe013f263755f3c525eaf73702662293a3d8b5e2ec5
+size 35244294

data/AMS/AMS_2018.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7b929f0c6d71116e23d4f52011e82eda07280aabb177300e37419ca38b047c60
+size 30251124

data/AMS/AMS_2020.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb6aaaa2cb700bc7d460a1f222756e6a795b629780087a477acd9713982fc0b9
+size 45426669

data/AMS/AMS_2022.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ccc90819f501fca9445d415c1ceca8d3991300f8e08724cf7043f1a103aa4231
+size 17636761

data/AMS/README.txt

@@ -0,0 +1,18 @@
+Documents are not uploaded to git. The list of documents which were uploaded to pinecone database AMS:
+AMS_1996, https://ntrs.nasa.gov/citations/19960025595
+AMS_1997, https://ntrs.nasa.gov/citations/19970021613
+AMS_1998, https://ntrs.nasa.gov/citations/19980193156
+AMS_1999, https://ntrs.nasa.gov/citations/19990053852
+AMS_2000, https://ntrs.nasa.gov/citations/20000048380
+AMS_2001, https://ntrs.nasa.gov/citations/20010071164
+AMS_2002, https://ntrs.nasa.gov/citations/20020050182
+AMS_2004, https://ntrs.nasa.gov/citations/20040084272
+AMS_2006, https://ntrs.nasa.gov/citations/20060028221
+AMS_2008, https://ntrs.nasa.gov/citations/20080023060
+AMS_2010, https://ntrs.nasa.gov/citations/20100021914
+AMS_2012, https://ntrs.nasa.gov/citations/20130008824
+AMS_2014, https://ntrs.nasa.gov/citations/20140008875
+AMS_2016, https://ntrs.nasa.gov/citations/20160004038
+AMS_2018, https://ntrs.nasa.gov/citations/20180002828
+AMS_2020, https://ntrs.nasa.gov/citations/20205009766
+AMS_2022, https://ntrs.nasa.gov/citations/20220006415

data/AMS/ams_data-400-0.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef248f60645d1def4d3624351c90cbb5d91554d0a8bfd35615514f4a71a20159
+size 18183603

data/AMS/ams_data-5000-0.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0472930c89ad2c13f997789b070049c99640c6ddcd114cc635110409854435b5
+size 17283048

data_import.py

@@ -0,0 +1,262 @@
+import os
+import re
+import logging
+import shutil
+import string
+
+import pinecone
+import chromadb
+
+import json, jsonlines
+from tqdm import tqdm
+
+from langchain_community.vectorstores import Pinecone
+from langchain_community.vectorstores import Chroma
+
+from langchain.text_splitter import RecursiveCharacterTextSplitter, CharacterTextSplitter
+
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import VoyageEmbeddings
+
+from langchain_community.document_loaders import PyPDFLoader
+from langchain_core.documents import Document as lancghain_Document
+
+from ragatouille import RAGPretrainedModel
+
+from dotenv import load_dotenv,find_dotenv
+load_dotenv(find_dotenv(),override=True)
+
+# Set secrets from environment file
+OPENAI_API_KEY=os.getenv('OPENAI_API_KEY')
+VOYAGE_API_KEY=os.getenv('VOYAGE_API_KEY')
+PINECONE_API_KEY=os.getenv('PINECONE_API_KEY')
+HUGGINGFACEHUB_API_TOKEN=os.getenv('HUGGINGFACEHUB_API_TOKEN') 
+
+def chunk_docs(docs,
+               chunk_method='tiktoken_recursive',
+               file=None,
+               chunk_size=500,
+               chunk_overlap=0,
+               use_json=False):
+    docs_out=[]
+    if file:
+        logging.info('Jsonl file to be used: '+file)
+    if use_json and os.path.exists(file):
+            logging.info('Jsonl file found, using this instead of parsing docs.')
+            with open(file, "r") as file_in:
+                file_data = [json.loads(line) for line in file_in]
+            # Process the file data and put it into the same format as docs_out
+            for line in file_data:
+                doc_temp = lancghain_Document(page_content=line['page_content'],
+                                              source=line['metadata']['source'],
+                                              page=line['metadata']['page'],
+                                              metadata=line['metadata'])
+                if has_meaningful_content(doc_temp):
+                    docs_out.append(doc_temp)
+            logging.info('Parsed: '+file)
+            logging.info('Number of entries: '+str(len(docs_out)))
+            logging.info('Sample entries:')
+            logging.info(str(docs_out[0]))
+            logging.info(str(docs_out[-1]))
+    else:
+        logging.info('No jsonl found. Reading and parsing docs.')
+        logging.info('Chunk size (tokens): '+str(chunk_size))
+        logging.info('Chunk overlap (tokens): '+str(chunk_overlap))
+        for doc in tqdm(docs,desc='Reading and parsing docs'):
+            logging.info('Parsing: '+doc)
+            loader = PyPDFLoader(doc)
+            data = loader.load_and_split()
+
+            if chunk_method=='tiktoken_recursive':
+                text_splitter = RecursiveCharacterTextSplitter.from_tiktoken_encoder(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+            else:
+                raise NotImplementedError
+            pages = text_splitter.split_documents(data)
+
+            # Tidy up text by removing unnecessary characters
+            for page in pages:
+                page.metadata['source']=os.path.basename(page.metadata['source'])   # Strip path
+                page.metadata['page']=int(page.metadata['page'])+1   # Pages are 0 based, update
+                page.page_content=re.sub(r"(\w+)-\n(\w+)", r"\1\2", page.page_content)   # Merge hyphenated words
+                page.page_content = re.sub(r"(?<!\n\s)\n(?!\s\n)", " ", page.page_content.strip())  # Fix newlines in the middle of sentences
+                page.page_content = re.sub(r"\n\s*\n", "\n\n", page.page_content)   # Remove multiple newlines
+                # Add metadata to the end of the page content, some RAG models don't have metadata.
+                page.page_content += str(page.metadata)
+                doc_temp=lancghain_Document(page_content=page.page_content,
+                                            source=page.metadata['source'],
+                                            page=page.metadata['page'],
+                                            metadata=page.metadata)
+                if has_meaningful_content(page):
+                    docs_out.append(doc_temp)
+        logging.info('Parsed: '+doc)
+        logging.info('Sample entries:')
+        logging.info(str(docs_out[0]))
+        logging.info(str(docs_out[-1]))
+        if file:
+            # Write to a jsonl file, save it.
+            logging.info('Writing to jsonl file: '+file)
+            with jsonlines.open(file, mode='w') as writer:
+                for doc in docs_out: 
+                    writer.write(doc.dict())
+            logging.info('Written: '+file)
+    return docs_out
+def load_docs(index_type,
+              docs,
+              query_model,
+              index_name=None,
+              chunk_method='tiktoken_recursive',
+              chunk_size=500,
+              chunk_overlap=0,
+              clear=False,
+              use_json=False,
+              file=None,
+              batch_size=50):
+    """
+    Loads PDF documents. If index_name is blank, it will return a list of the data (texts). If it is a name of a pinecone storage, it will return the vector_store.    
+    """
+    # Chunk docs
+    docs_out=chunk_docs(docs,
+                        chunk_method=chunk_method,
+                        file=file,
+                        chunk_size=chunk_size,
+                        chunk_overlap=chunk_overlap,
+                        use_json=use_json)
+    # Initialize client
+    db_path='../db/'
+    if index_name:
+        if index_type=="Pinecone":
+            # Import and initialize Pinecone client
+            pinecone.init(
+                api_key=PINECONE_API_KEY
+            )
+            # Find the existing index, clear for new start
+            if clear:
+                try:
+                    pinecone.describe_index(index_name)
+                except:
+                    raise Exception(f"Cannot clear index {index_name} because it does not exist.")
+                index=pinecone.Index(index_name)
+                index.delete(delete_all=True) # Clear the index first, then upload
+                logging.info('Cleared database '+index_name)
+            # Upsert docs
+            try:
+                pinecone.describe_index(index_name)
+            except:
+                logging.info(f"Index {index_name} does not exist. Creating new index.")
+                logging.info('Size of embedding used: '+str(embedding_size(query_model)))  # TODO: set this to be backed out of the embedding size
+                pinecone.create_index(index_name,dimension=embedding_size(query_model))
+                logging.info(f"Index {index_name} created. Adding {len(docs_out)} entries to index.")
+                pass
+            else:
+                logging.info(f"Index {index_name} exists. Adding {len(docs_out)} entries to index.")
+            index = pinecone.Index(index_name)
+            vectorstore = Pinecone(index, query_model, "page_content") # Set the vector store to calculate embeddings on page_content
+            vectorstore = batch_upsert(index_type,
+                                       vectorstore,
+                                       docs_out,
+                                       batch_size=batch_size)
+        elif index_type=="ChromaDB":
+            # Upsert docs. Defaults to putting this in the ../db directory
+            logging.info(f"Creating new index {index_name}.")
+            persistent_client = chromadb.PersistentClient(path=db_path+'/chromadb')            
+            vectorstore = Chroma(client=persistent_client,
+                                 collection_name=index_name,
+                                 embedding_function=query_model)
+            logging.info(f"Index {index_name} created. Adding {len(docs_out)} entries to index.")
+            vectorstore = batch_upsert(index_type,
+                                       vectorstore,
+                                       docs_out,
+                                       batch_size=batch_size)
+            logging.info("Documents upserted to f{index_name}.")
+            # Test query
+            test_query = vectorstore.similarity_search('What are examples of aerosapce adhesives to avoid?')
+            logging.info('Test query: '+str(test_query))
+            if not test_query:
+                raise ValueError("Chroma vector database is not configured properly. Test query failed.")       
+        elif index_type=="RAGatouille":
+            logging.info(f'Setting up RAGatouille model {query_model}')
+            vectorstore = RAGPretrainedModel.from_pretrained(query_model)
+            logging.info('RAGatouille model set: '+str(vectorstore))
+
+            # Create an index from the vectorstore.
+            docs_out_colbert = [doc.page_content for doc in docs_out]
+            if chunk_size>500:
+                raise ValueError("RAGatouille cannot handle chunks larger than 500 tokens. Reduce token count.")
+            vectorstore.index(
+                collection=docs_out_colbert,
+                index_name=index_name,
+                max_document_length=chunk_size,
+                overwrite_index=True,
+                split_documents=True,
+            )
+            logging.info(f"Index created: {vectorstore}")
+
+            # Move the directory to the db folder
+            logging.info(f"Moving RAGatouille index to {db_path}")
+            ragatouille_path = os.path.join(db_path, '.ragatouille')
+            if os.path.exists(ragatouille_path):
+                shutil.rmtree(ragatouille_path)
+                logging.info(f"RAGatouille index deleted from {ragatouille_path}")
+            shutil.move('./.ragatouille', db_path)
+            logging.info(f"RAGatouille index created in {db_path}:"+str(vectorstore))
+
+    # Return vectorstore or docs
+    if index_name:
+        return vectorstore
+    else:
+        return docs_out
+def delete_index(index_type,index_name):
+    """
+    Deletes an existing Pinecone index with the given index_name.
+    """
+    if index_type=="Pinecone":
+        # Import and initialize Pinecone client
+        pinecone.init(
+            api_key=PINECONE_API_KEY
+        )
+        try:
+            pinecone.describe_index(index_name)
+            logging.info(f"Index {index_name} exists.")
+        except:
+            raise Exception(f"Index {index_name} does not exist, cannot delete.")
+        else:
+            pinecone.delete_index(index_name)
+            logging.info(f"Index {index_name} deleted.")
+    elif index_type=="ChromaDB":
+        # Delete existing collection
+        logging.info(f"Deleting index {index_name}.")
+        persistent_client = chromadb.PersistentClient(path='../db/chromadb')  
+        persistent_client.delete_collection(name=index_name)  
+        logging.info("Index deleted.")
+    elif index_type=="RAGatouille":
+            raise NotImplementedError
+def batch_upsert(index_type,vectorstore,docs_out,batch_size=50):
+    # Batch insert the chunks into the vector store
+    for i in range(0, len(docs_out), batch_size):
+        chunk_batch = docs_out[i:i + batch_size]
+        if index_type=="Pinecone":
+            vectorstore.add_documents(chunk_batch)
+        elif index_type=="ChromaDB":
+            vectorstore.add_documents(chunk_batch)  # Happens to be same for chroma/pinecone, leaving if statement just in case
+    return vectorstore
+def has_meaningful_content(page):
+    """
+    Test whether the page has more than 30% words and is more than 5 words.
+    """
+    text=page.page_content
+    num_words = len(text.split())
+    alphanumeric_pct = sum(c.isalnum() for c in text) / len(text)
+    if num_words < 5 or alphanumeric_pct < 0.3:
+        return False
+    else:
+        return True
+def embedding_size(embedding_model):
+    """
+    Returns the embedding size of the model.
+    """
+    if isinstance(embedding_model,OpenAIEmbeddings):
+        return 1536 # https://platform.openai.com/docs/models/embeddings, test-embedding-ada-002
+    elif isinstance(embedding_model,VoyageEmbeddings):
+        return 1024 # https://docs.voyageai.com/embeddings/, voyage-02
+    else:
+        raise NotImplementedError

pages/1_Chatbot_AMS_Langchain.py

@@ -1,152 +0,0 @@
-import os
-import queries
-import pinecone
-from dotenv import load_dotenv, find_dotenv
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.llms import OpenAI
-import streamlit as st
-import openai
-import time
-
-from dotenv import load_dotenv,find_dotenv,dotenv_values
-load_dotenv(find_dotenv(),override=True)
-
-# Set secrets
-# PINECONE_ENVIRONMENT=db.secrets.get('PINECONE_ENVIRONMENT')
-# PINECONE_API_KEY=db.secrets.get('PINECONE_API_KEY')
-PINECONE_ENVIRONMENT=os.getenv('PINECONE_ENVIRONMENT')
-PINECONE_API_KEY=os.getenv('PINECONE_API_KEY')
-
-# Set the page title
-st.set_page_config(
-    page_title='Aerospace Chatbot: AMS w/Langchain',
-)
-st.title('Aerospace Mechanisms Chatbot')
-with st.expander('''What's under the hood?'''):
-    st.markdown('''
-    This chatbot will look up from all Aerospace Mechanism Symposia in the following location: https://github.com/dsmueller3760/aerospace_chatbot/tree/main/data/AMS
-    * Source code: https://github.com/dsmueller3760/aerospace_chatbot/blob/main/scripts/setup_page_langchain.py
-    * Uses custom langchain functions with QA retrieval: https://js.langchain.com/docs/modules/chains/popular/chat_vector_db_legacy
-    * All prompts will query entire database unless 'filter response with last received sources' is activated.
-    * **Repsonse time ~10 seconds per prompt**.
-    ''')
-filter_toggle=st.checkbox('Filter response with last received sources?')
-
-# Add a sidebar for input options
-st.title('Input')
-
-# Add input fields in the sidebar
-st.sidebar.title('Input options')
-output_level = st.sidebar.selectbox('Level of Output', ['Concise', 'Detailed'], index=1)
-k = st.sidebar.number_input('Number of items per prompt', min_value=1, step=1, value=4)
-search_type = st.sidebar.selectbox('Search Type', ['similarity', 'mmr'], index=1)
-temperature = st.sidebar.slider('Temperature', min_value=0.0, max_value=2.0, value=0.0, step=0.1)
-verbose = st.sidebar.checkbox('Verbose output')
-chain_type = st.sidebar.selectbox('Chain Type', ['stuff', 'map_reduce'], index=0)
-
-# Vector databases
-st.sidebar.title('Vector database')
-index_type=st.sidebar.selectbox('Index type', ['Pinecone'], index=0)
-index_name=st.sidebar.selectbox('Index name', ['canopy--ams'], index=0)
-
-# Embeddings
-st.sidebar.title('Embeddings')
-embedding_type=st.sidebar.selectbox('Embedding type', ['Openai'], index=0)
-embedding_name=st.sidebar.selectbox('Embedding name', ['text-embedding-ada-002'], index=0)
-
-# Add a section for secret keys
-st.sidebar.title('Secret keys')
-OPENAI_API_KEY = st.sidebar.text_input('OpenAI API Key', type='password')
-
-# Pinecone
-pinecone.init(
-    api_key=PINECONE_API_KEY,
-    environment=PINECONE_ENVIRONMENT
-)
-
-if OPENAI_API_KEY:
-    openai.api_key = OPENAI_API_KEY
-    embeddings_model = OpenAIEmbeddings(model=embedding_name,openai_api_key=OPENAI_API_KEY)
-
-    # Set up chat history
-    qa_model_obj = st.session_state.get('qa_model_obj',[])
-    message_id = st.session_state.get('message_id', 0)
-
-    if 'messages' not in st.session_state:
-        st.session_state.messages = []
-    for message in st.session_state.messages:
-        with st.chat_message(message['role']):
-            st.markdown(message['content'])
-
-    # Process some items
-    if output_level == 'Concise':
-        out_token = 50
-    else:
-        out_token = 516
-
-    # Define LLM parameters and qa model object
-    llm = OpenAI(temperature=temperature,
-                    openai_api_key=OPENAI_API_KEY,
-                    max_tokens=out_token)
-    qa_model_obj=queries.QA_Model(index_name,
-                    embeddings_model,
-                    llm,
-                    k,
-                    search_type,
-                    verbose,
-                    filter_arg=False)
-
-    # Display assistant response in chat message container
-    if prompt := st.chat_input('Prompt here'):
-        st.session_state.messages.append({'role': 'user', 'content': prompt})
-        with st.chat_message('user'):
-            st.markdown(prompt)
-        with st.chat_message('assistant'):
-            message_placeholder = st.empty()
-
-            with st.status('Generating response...') as status:
-                t_start=time.time()
-
-                # Process some items
-                if output_level == 'Concise':
-                    out_token = 50
-                else:
-                    out_token = 516
-
-                # Define LLM parameters and qa model object
-                llm = OpenAI(temperature=temperature,
-                                openai_api_key=OPENAI_API_KEY,
-                                max_tokens=out_token)
-
-                message_id += 1
-                st.write('Message: '+str(message_id))
-                
-                if message_id>1:
-                    qa_model_obj=st.session_state['qa_model_obj']
-                    qa_model_obj.update_model(llm,
-                                        k=k,
-                                        search_type=search_type,
-                                        verbose=verbose,
-                                        filter_arg=filter_toggle)
-                    if filter_toggle:
-                        filter_list = list(set(item['source'] for item in qa_model_obj.sources[-1]))
-                        filter_items=[]
-                        for item in filter_list:
-                            filter_item={'source': item}
-                            filter_items.append(filter_item)
-                        filter={'$or':filter_items}
-                
-                st.write('Searching vector database, generating prompt...')
-                qa_model_obj.query_docs(prompt)
-                ai_response=qa_model_obj.result['answer']
-                message_placeholder.markdown(ai_response)
-                t_delta=time.time() - t_start
-                status.update(label='Prompt generated in '+"{:10.3f}".format(t_delta)+' seconds', state='complete', expanded=False)
-        
-        st.session_state['qa_model_obj'] = qa_model_obj
-        st.session_state['message_id'] = message_id
-        st.session_state.messages.append({'role': 'assistant', 'content': ai_response})
-
-else:
-    st.warning('No API key found. Add your API key in the sidebar under Secret Keys. Find it or create one here: https://platform.openai.com/api-keys')
-    st.info('Your API-key is not stored in any form by this app. However, for transparency it is recommended to delete your API key once used.')

pages/1_Chatbot_AMS_Modular.py

@@ -0,0 +1,158 @@
+import queries, setup
+
+import os
+import time
+import logging
+import json
+
+import pinecone
+import openai
+
+from langchain_community.vectorstores import Pinecone
+from langchain_community.vectorstores import Chroma
+
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import VoyageEmbeddings
+
+from langchain_openai import OpenAI, ChatOpenAI
+from langchain_community.llms import HuggingFaceHub
+
+from ragatouille import RAGPretrainedModel
+
+import streamlit as st
+
+# Set up the page, enable logging
+from dotenv import load_dotenv,find_dotenv
+load_dotenv(find_dotenv(),override=True)
+logging.basicConfig(filename='app_1_chatbot_ams_modular.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s', level=logging.DEBUG)
+
+# Set the page title
+st.set_page_config(
+    page_title='Aerospace Chatbot: Modular',
+)
+st.title('Aerospace Mechanisms Chatbot')
+with st.expander('''What's under the hood?'''):
+    st.markdown('''
+    This chatbot will look up from all Aerospace Mechanism Symposia in the following location: https://github.com/dsmueller3760/aerospace_chatbot/tree/main/data/AMS
+    Example questions:
+    * What are examples of latch failures which have occurred due to improper fitup?
+    * What are examples of lubricants which should be avoided for space mechanism applications?
+    ''')
+filter_toggle=st.checkbox('Filter response with last received sources?')
+
+sb=setup.load_sidebar(config_file='../config/config.json',
+                      index_data_file='../config/index_data.json',
+                      vector_databases=True,
+                      embeddings=True,
+                      rag_type=True,
+                      index_name=True,
+                      llm=True,
+                      model_options=True,
+                      secret_keys=True)
+
+secrets=setup.set_secrets(sb) # Take secrets from .env file first, otherwise from sidebar
+
+# Set up chat history
+if 'qa_model_obj' not in st.session_state:
+    st.session_state.qa_model_obj = []
+if 'message_id' not in st.session_state:
+    st.session_state.message_id = 0
+if 'messages' not in st.session_state:
+    st.session_state.messages = []
+for message in st.session_state.messages:
+    with st.chat_message(message['role']):
+        st.markdown(message['content'])
+
+# Define chat
+if prompt := st.chat_input('Prompt here'):
+    # User prompt
+    st.session_state.messages.append({'role': 'user', 'content': prompt})
+    with st.chat_message('user'):
+        st.markdown(prompt)
+    # Assistant response
+    with st.chat_message('assistant'):
+        message_placeholder = st.empty()
+
+        with st.status('Generating response...') as status:
+            t_start=time.time()
+
+            st.session_state.message_id += 1
+            st.write('Starting reponse generation for message: '+str(st.session_state.message_id))
+            logging.info('Starting reponse generation for message: '+str(st.session_state.message_id))
+
+             # Process some items
+            if sb['model_options']['output_level'] == 'Concise':
+                out_token = 50
+            else:
+                out_token = 516
+            logging.info('Output tokens: '+str(out_token))
+            
+            if st.session_state.message_id==1:
+                # Define embeddings
+                if sb['query_model']=='Openai':
+                    query_model=OpenAIEmbeddings(model=sb['embedding_name'],openai_api_key=secrets['OPENAI_API_KEY'])
+                elif sb['query_model']=='Voyage':
+                    query_model=VoyageEmbeddings(model=sb['embedding_name'],voyage_api_key=secrets['VOYAGE_API_KEY'])
+                elif sb['index_type']=='RAGatouille':
+                    query_model=RAGPretrainedModel.from_index('../db/.ragatouille/colbert/indexes/'+sb['index_name'])
+                logging.info('Query model set: '+str(query_model))
+
+                # Define LLM
+                if sb['llm_source']=='OpenAI':
+                    llm = ChatOpenAI(model_name=sb['llm_model'],
+                                    temperature=sb['model_options']['temperature'],
+                                    openai_api_key=secrets['OPENAI_API_KEY'],
+                                    max_tokens=out_token)
+                elif sb['llm_source']=='Hugging Face':
+                    llm = HuggingFaceHub(repo_id=sb['llm_model'],
+                                        model_kwargs={"temperature": sb['model_options']['temperature'], "max_length": out_token})
+                logging.info('LLM model set: '+str(llm))
+
+                # Initialize QA model object
+                if 'search_type' in sb['model_options']: 
+                    search_type=sb['model_options']['search_type']
+                else:
+                    search_type=None
+                st.session_state.qa_model_obj=queries.QA_Model(sb['index_type'],
+                                                               sb['index_name'],
+                                                               query_model,
+                                                               llm,
+                                                               k=sb['model_options']['k'],
+                                                               search_type=search_type,
+                                                               filter_arg=False)
+                logging.info('QA model object set: '+str(st.session_state.qa_model_obj))
+            if st.session_state.message_id>1:
+                logging.info('Updating model with sidebar settings...')
+                # Update LLM
+                if sb['llm_source']=='OpenAI':
+                    llm = ChatOpenAI(model_name=sb['llm_model'],
+                                    temperature=sb['model_options']['temperature'],
+                                    openai_api_key=secrets['OPENAI_API_KEY'],
+                                    max_tokens=out_token)
+                elif sb['llm_source']=='Hugging Face':
+                    llm = HuggingFaceHub(repo_id=sb['llm_model'],
+                                        model_kwargs={"temperature": sb['model_options']['temperature'], "max_length": out_token})
+                logging.info('LLM model set: '+str(llm))
+
+                st.session_state.qa_model_obj.update_model(llm,
+                                                           k=sb['model_options']['k'],
+                                                           search_type=sb['model_options']['search_type'],
+                                                           filter_arg=filter_toggle)
+                logging.info('QA model object updated: '+str(st.session_state.qa_model_obj))
+            
+            st.write('Searching vector database, generating prompt...')
+            logging.info('Searching vector database, generating prompt...')
+            st.session_state.qa_model_obj.query_docs(prompt)
+            ai_response=st.session_state.qa_model_obj.result['answer'].content
+            message_placeholder.markdown(ai_response)
+            t_delta=time.time() - t_start
+            status.update(label='Prompt generated in '+"{:10.3f}".format(t_delta)+' seconds', state='complete', expanded=False)
+            
+        st.session_state.messages.append({'role': 'assistant', 'content': ai_response})
+        logging.info(f'Messaging complete for {st.session_state.message_id}.')
+
+# Add reset button
+if st.button('Restart session'):
+    st.session_state.qa_model_obj = []
+    st.session_state.message_id = 0
+    st.session_state.messages = []

pages/2_Chatbot_AMS_Canopy.py

@@ -1,157 +0,0 @@
-import os
-import queries
-import pinecone
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.llms import OpenAI
-import streamlit as st
-import openai
-import time
-
-from tqdm.auto import tqdm
-from typing import Tuple
-
-# from dotenv import load_dotenv,find_dotenv,dotenv_values
-# load_dotenv(find_dotenv(),override=True)
-
-from canopy.tokenizer import Tokenizer
-from canopy.knowledge_base import KnowledgeBase
-from canopy.context_engine import ContextEngine
-from canopy.chat_engine import ChatEngine
-from canopy.llm.openai import OpenAILLM
-# from canopy.llm.models import ModelParams
-from canopy.models.data_models import Document, Messages, UserMessage, AssistantMessage
-from canopy.models.api_models import ChatResponse
-
-def chat(new_message: str, history: Messages) -> Tuple[str, Messages, ChatResponse]:
-    messages = history + [UserMessage(content=new_message)]
-    response = chat_engine.chat(messages)
-    assistant_response = response.choices[0].message.content
-    return assistant_response, messages + [AssistantMessage(content=assistant_response)], response
-
-# Set secrets
-# PINECONE_ENVIRONMENT=db.secrets.get('PINECONE_ENVIRONMENT')
-# PINECONE_API_KEY=db.secrets.get('PINECONE_API_KEY')
-PINECONE_ENVIRONMENT=os.getenv('PINECONE_ENVIRONMENT')
-PINECONE_API_KEY=os.getenv('PINECONE_API_KEY')
-
-# Set the page title
-st.set_page_config(
-    page_title='Aerospace Chatbot: AMS w/Langchain',
-)
-st.title('Aerospace Mechanisms Chatbot')
-with st.expander('''What's under the hood?'''):
-    st.markdown('''
-    This chatbot will look up from all Aerospace Mechanism Symposia in the following location: https://github.com/dsmueller3760/aerospace_chatbot/tree/main/data/AMS
-    * Source code: https://github.com/dsmueller3760/aerospace_chatbot/blob/main/scripts/setup_page_canopy.py
-    * Uses pinecone canopy: https://www.pinecone.io/blog/canopy-rag-framework/
-    * **Response time ~45 seconds per prompt**
-    ''')
-
-# Add a sidebar for input options
-st.title('Input')
-st.sidebar.title('Input Options')
-
-# Add input fields in the sidebar
-model_name=st.sidebar.selectbox('Model', ['gpt-3.5-turbo''gpt-3.5-turbo-16k','gpt-3.5-turbo','gpt-3.5-turbo-1106','gpt-4','gpt-4-32k'], index=1)
-model_list={'gpt-3.5-turbo':4096,
-            'gpt-3.5-turbo-16k':16385,
-            'gpt-3.5-turbo-1106':16385, 
-            'gpt-4':8192,
-            'gpt-4-32k':32768}
-temperature = st.sidebar.slider('Temperature', min_value=0.0, max_value=2.0, value=0.0, step=0.1)
-n=None  # Not used. How many chat completion choices to generate for each input message.
-top_p=None  # Not used. Only use this or temperature. Where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.
-
-k=st.sidebar.number_input('Number document chunks per query', min_value=1, step=1, value=15)
-output_level=st.sidebar.selectbox('Level of Output', ['Concise', 'Detailed', 'No Limit'], index=2)
-max_prompt_tokens=model_list[model_name]
-
-# Vector databases
-st.sidebar.title('Vector Database')
-index_name=st.sidebar.selectbox('Index name', ['canopy--ams'], index=0)
-
-# Embeddings
-st.sidebar.title('Embeddings')
-embedding_type=st.sidebar.selectbox('Embedding type', ['Openai'], index=0)
-embedding_name=st.sidebar.selectbox('Embedding name', ['text-embedding-ada-002'], index=0)
-
-# Add a section for secret keys
-st.sidebar.title('Secret Keys')
-OPENAI_API_KEY = st.sidebar.text_input('OpenAI API Key', type='password')
-
-
-
-if OPENAI_API_KEY:
-    openai.api_key = OPENAI_API_KEY
-    embeddings_model = OpenAIEmbeddings(model=embedding_name,openai_api_key=OPENAI_API_KEY)
-
-    # Set up chat history
-    qa_model_obj = st.session_state.get('qa_model_obj',[])
-    message_id = st.session_state.get('message_id', 0)
-    history = st.session_state.get('history',[])
-
-    if 'messages' not in st.session_state:
-        st.session_state.messages = []
-    for message in st.session_state.messages:
-        with st.chat_message(message['role']):
-            st.markdown(message['content'])
-
-    # Process some items
-    if output_level == 'Concise':
-        out_token = 50
-    else:
-        out_token = 516
-
-    # Display assistant response in chat message container
-    if prompt := st.chat_input('Prompt here'):
-        st.session_state.messages.append({'role': 'user', 'content': prompt})
-        with st.chat_message('user'):
-            st.markdown(prompt)
-        with st.chat_message('assistant'):
-            message_placeholder = st.empty()
-
-            with st.status('Generating response...') as status:
-                t_start=time.time()
-                message_id += 1
-                st.write('Message: '+str(message_id))
-                
-                # Process some items
-                if output_level == 'Concise':
-                    max_generated_tokens = 50
-                elif output_level == 'Detailed':
-                    max_generated_tokens = 516
-                else:
-                    max_generated_tokens = None
-                    
-                # Inialize canopy
-                Tokenizer.initialize()
-                pinecone.init(
-                    api_key=PINECONE_API_KEY,
-                    environment=PINECONE_ENVIRONMENT
-                )
-
-                kb = KnowledgeBase(index_name=index_name,
-                                default_top_k=k)
-                kb.connect()
-                context_engine = ContextEngine(kb)
-                llm=OpenAILLM(model_name=model_name)
-                chat_engine = ChatEngine(context_engine,
-                                        llm=llm,
-                                        max_generated_tokens=max_generated_tokens,
-                                        max_prompt_tokens=max_prompt_tokens)
-                
-                st.write('Searching vector database, generating prompt...')
-                response, history, chat_response = chat(prompt, history)
-
-                message_placeholder.markdown(response)
-                t_delta=time.time() - t_start
-                status.update(label='Prompt generated in '+"{:10.3f}".format(t_delta)+' seconds', state='complete', expanded=False)
-        
-        st.session_state['history'] = history
-        st.session_state['qa_model_obj'] = qa_model_obj
-        st.session_state['message_id'] = message_id
-        st.session_state.messages.append({'role': 'assistant', 'content': response})
-
-else:
-    st.warning('No API key found. Add your API key in the sidebar under Secret Keys. Find it or create one here: https://platform.openai.com/api-keys')
-    st.info('Your API-key is not stored in any form by this app. However, for transparency it is recommended to delete your API key once used.')

pages/2_Document_Upload.py

@@ -0,0 +1,109 @@
+import data_import, setup
+
+import os
+import time
+import logging
+import glob
+
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import VoyageEmbeddings
+
+from ragatouille import RAGPretrainedModel
+
+import streamlit as st
+
+# Set up the page, enable logging 
+from dotenv import load_dotenv,find_dotenv
+load_dotenv(find_dotenv(),override=True)
+logging.basicConfig(filename='app_2_document_upload.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s', level=logging.DEBUG)
+
+# Set the page title
+st.set_page_config(
+    page_title='Upload PDFs',
+)
+st.title('Upload PDFs')
+
+sb=setup.load_sidebar(config_file='../config/config.json',
+                      index_data_file='../config/index_data.json',
+                      vector_databases=True,
+                      embeddings=True,
+                      index_name=True,
+                      secret_keys=True)
+
+secrets=setup.set_secrets(sb) # Take secrets from .env file first, otherwise from sidebar
+
+# Populate the main screen
+logging.info(f'index_type test, {sb["index_type"]}')
+
+if sb["index_type"]=='RAGatouille':
+    logging.info('Set hugging face model for queries.')
+    query_model=sb['query_model']
+elif sb['query_model']=='Openai' or 'Voyage':
+    logging.info('Set embeddings model for queries.')
+    if sb['query_model']=='Openai':
+        query_model=OpenAIEmbeddings(model=sb['embedding_name'],openai_api_key=secrets['OPENAI_API_KEY'])
+    elif sb['query_model']=='Voyage':
+        query_model=VoyageEmbeddings(voyage_api_key=secrets['VOYAGE_API_KEY'])
+logging.info('Query model set: '+str(query_model))
+
+# Find docs
+index_name_md=st.markdown('Enter a directory relative to the current directory, or an absolute path.')
+data_folder = st.text_input('Enter a directory','../data/AMS/')
+if not os.path.isdir(data_folder):
+    st.error('The entered directory does not exist')
+docs = glob.glob(data_folder+'*.pdf')   # Only get the PDFs in the directory
+st.markdown('PDFs found: '+str(docs))
+st.markdown('Number of PDFs found: ' + str(len(docs)))
+logging.info('Docs: '+str(docs))
+
+# Add an expandable box for options
+with st.expander("Options"):
+    use_json = st.checkbox('Use existing jsonl, if available (will ignore chunk method, size, and overlap)?', value=True)
+    json_file=st.text_input('Jsonl file',data_folder+'ams_data.jsonl')
+    clear_database = st.checkbox('Clear existing database?')
+    chunk_method= st.selectbox('Chunk method', ['tiktoken_recursive'], index=0)
+    if sb['query_model']=='Openai' or 'ChromaDB':
+        # OpenAI will time out if the batch size is too large
+        batch_size=st.number_input('Batch size for upsert', min_value=1, step=1, value=100)
+    else:
+        batch_size=None
+    if chunk_method=='tiktoken_recursive':
+        chunk_size=st.number_input('Chunk size (tokens)', min_value=1, step=1, value=500)
+        chunk_overlap=st.number_input('Chunk overlap (tokens)', min_value=0, step=1, value=0)
+    else:
+        raise NotImplementedError
+
+# Add a button to run the function
+if st.button('Chunk docs to jsonl file'):
+    start_time = time.time()  # Start the timer
+    data_import.chunk_docs(docs,
+                           file=json_file,
+                           chunk_method=chunk_method,
+                           chunk_size=chunk_size,
+                           chunk_overlap=chunk_overlap,
+                           use_json=False)
+    end_time = time.time()  # Stop the timer
+    elapsed_time = end_time - start_time 
+    st.write(f"Elapsed Time: {elapsed_time:.2f} seconds")
+if st.button('Load docs into vector database'):
+    start_time = time.time()  # Start the timer
+    data_import.load_docs(sb['index_type'],
+                          docs,
+                          query_model=query_model,
+                          index_name=sb['index_name'],
+                          chunk_size=chunk_size,
+                          chunk_overlap=chunk_overlap,
+                          use_json=use_json,
+                          clear=clear_database,
+                          file=json_file,
+                          batch_size=batch_size)
+    end_time = time.time()  # Stop the timer
+    elapsed_time = end_time - start_time 
+    st.write(f"Elapsed Time: {elapsed_time:.2f} seconds")
+# Add a button to delete the index
+if st.button('Delete existing index'):
+    start_time = time.time()  # Start the timer
+    data_import.delete_index(sb['index_type'],sb['index_name'])
+    end_time = time.time()  # Stop the timer
+    elapsed_time = end_time - start_time 
+    st.write(f"Elapsed Time: {elapsed_time:.2f} seconds")

pages/3_Visualize_Data.py

@@ -0,0 +1,110 @@
+import setup
+
+import time
+import logging
+from datetime import datetime
+
+from langchain_openai import OpenAIEmbeddings
+from langchain_community.embeddings import VoyageEmbeddings
+
+from ragxplorer import RAGxplorer
+
+import streamlit as st
+
+# Set up the page, enable logging 
+from dotenv import load_dotenv,find_dotenv
+load_dotenv(find_dotenv(),override=True)
+logging.basicConfig(filename='app_3_visualize_data.log', filemode='w', format='%(name)s - %(levelname)s - %(message)s', level=logging.DEBUG)
+
+# Set the page title
+st.set_page_config(
+    page_title='Visualize Data',
+    layout='wide'
+)
+st.title('Visualize Data')
+
+sb=setup.load_sidebar(config_file='../config/config.json',
+                      index_data_file='../config/index_data.json',
+                      vector_databases=True,
+                      embeddings=True,
+                      index_name=True,
+                      secret_keys=True)
+secrets=setup.set_secrets(sb) # Take secrets from .env file first, otherwise from sidebar
+
+# Set up session state variables
+if 'client' not in st.session_state:
+    st.session_state.client = None
+
+# Populate the main screen
+logging.info(f'index_type test, {sb["index_type"]}')
+
+if sb["index_type"]=='RAGatouille':
+    raise Exception('Only index type ChromaDB is supported for this function.')
+elif sb["index_type"]=='Pinecone':
+    raise Exception('Only index type ChromaDB is supported for this function.')
+elif sb['query_model']=='Openai' or 'Voyage':
+    logging.info('Set embeddings model for queries.')
+    if sb['query_model']=='Openai':
+        query_model=OpenAIEmbeddings(model=sb['embedding_name'],openai_api_key=secrets['OPENAI_API_KEY'])
+    elif sb['query_model']=='Voyage':
+        query_model=VoyageEmbeddings(voyage_api_key=secrets['VOYAGE_API_KEY'])
+logging.info('Query model set: '+str(query_model))
+
+st.info('You must have created a database using Document Upload in ChromaDB for this to work.')
+
+# Add an expandable with description of what's going on.
+with st.expander("Under the hood",expanded=True):
+    st.markdown('''
+                Uses modified version of https://github.com/gabrielchua/RAGxplorer/tree/main?tab=readme-ov-file to connect to existing database created.
+                Assumes that chroma databases are located in ../db/chroma
+                Query size in database: Take a random sample of this size from the database to visualize.
+                ''')
+
+with st.expander("Create visualization data",expanded=True):
+    # Add a button to run the function
+    vector_qty=st.number_input('Query size in database', min_value=1, step=10, value=50)
+    export_df = st.checkbox('Export visualization data?', value=True)
+    if export_df:
+        current_time = datetime.now().strftime("%Y.%m.%d.%H.%M")
+        df_export_path = st.text_input('Export file', f'../data/AMS/ams_data-400-0-{vector_qty}.json')
+    if st.button('Create visualization data'):
+        start_time = time.time()  # Start the timer
+        
+        st.session_state.client = RAGxplorer(embedding_model=sb['embedding_name'])
+        st.session_state.client.load_db(path_to_db='../db/chromadb/',index_name=sb['index_name'],
+                                        df_export_path=df_export_path,
+                                        vector_qty=vector_qty,
+                                        verbose=True)
+
+        end_time = time.time()  # Stop the timer
+        elapsed_time = end_time - start_time 
+        st.write(f"Elapsed Time: {elapsed_time:.2f} seconds")
+
+with st.expander("Visualize data",expanded=True):
+    import_data = st.checkbox('Import visualization data?', value=True)
+    if import_data:
+        import_file = st.file_uploader("Import file", type="json")
+        if import_file is None:
+            # Use a default file
+            import_file_path=st.text_input('Import file',df_export_path)
+        else:
+            # Use the uploaded file
+            import_file_path=st.text_input('Import file',f'../data/AMS/{import_file.name}')
+    else:
+        import_file_path=None
+    
+    query = st.text_input('Query', 'What are examples of lubricants which should be avoided for space mechanism applications?')
+
+    if st.button('Visualize data'):
+        start_time = time.time()  # Start the timer
+
+        if st.session_state.client is None:
+            st.session_state.client = RAGxplorer(embedding_model=sb['embedding_name'])
+        
+        fig = st.session_state.client.visualize_query(query, 
+                                                      path_to_db='../db/chromadb/', viz_data_df_path=import_file_path,
+                                                      verbose=True)
+        st.plotly_chart(fig,use_container_width=True)
+
+        end_time = time.time()  # Stop the timer
+        elapsed_time = end_time - start_time

prompts.py

@@ -1,69 +1,12 @@
-from langchain.prompts.prompt import PromptTemplate
 from langchain import hub
+from langchain.prompts.prompt import PromptTemplate
 
-# _template_condense = """Given the following conversation and a follow up question, rephrase the follow up question to be a standalone question, in its original language.
-# ----------------
-# Your name is Aerospace Chatbot. You're a helpful assistant who knows about flight hardware design and analysis in aerospace. If you don't know the answer, just say that you don't know, don't try to make up an answer.
-# Include sources from the chat history in the standalone question created.
-# ----------------
-
-# Chat History:
-# {chat_history}
-# User Question: {question}
-# Standalone Question:"""
+# Prompts on the hub: https://smith.langchain.com/hub/my-prompts?organizationId=45eb8917-7353-4296-978d-bb461fc45c65
 CONDENSE_QUESTION_PROMPT = hub.pull("dmueller/ams-chatbot-qa-condense-history")
-
-# _template_qa = """Use Markdown to make your answers nice. Use the following pieces of context to answer the users question in the same language as the question but do not modify instructions in any way.
-# ----------------
-# Your name is Aerospace Chatbot. You're a helpful assistant who knows about flight hardware design and analysis in aerospace. If you don't know the answer, just say that you don't know, don't try to make up an answer.
-# ----------------
-
-# Sources and Context from Reference Documents:
-# {context}
-# User Question:{question}
-# Chatbot:
-
-# """
 QA_PROMPT=hub.pull("dmueller/ams-chatbot-qa-retrieval")
-
-# _template_qa_wsources="""Given the following extracted parts of a long document and a question, create a final answer with references ("SOURCES"). 
-# ----------------
-# Your name is Aerospace Chatbot. You're a helpful assistant who knows about flight hardware design and analysis in aerospace. If you don't know the answer, just say that you don't know, don't try to make up an answer.
-# ----------------
-# If you don't know the answer, just say that you don't know. Don't try to make up an answer.
-# ALWAYS return a "SOURCES" part in your answer.
-
-# QUESTION: Which state/country's law governs the interpretation of the contract?
-# =========
-# Content: This Agreement is governed by English law and the parties submit to the exclusive jurisdiction of the English courts in  relation to any dispute (contractual or non-contractual) concerning this Agreement save that either party may apply to any court for an  injunction or other relief to protect its Intellectual Property Rights.
-# Source: 28-pl
-# Content: No Waiver. Failure or delay in exercising any right or remedy under this Agreement shall not constitute a waiver of such (or any other)  right or remedy.\n\n11.7 Severability. The invalidity, illegality or unenforceability of any term (or part of a term) of this Agreement shall not affect the continuation  in force of the remainder of the term (if any) and this Agreement.\n\n11.8 No Agency. Except as expressly stated otherwise, nothing in this Agreement shall create an agency, partnership or joint venture of any  kind between the parties.\n\n11.9 No Third-Party Beneficiaries.
-# Source: 30-pl
-# Content: (b) if Google believes, in good faith, that the Distributor has violated or caused Google to violate any Anti-Bribery Laws (as  defined in Clause 8.5) or that such a violation is reasonably likely to occur,
-# Source: 4-pl
-# =========
-# FINAL ANSWER: This Agreement is governed by English law.
-# SOURCES: 28-pl
-
-# QUESTION: What did the president say about Michael Jackson?
-# =========
-# Content: Madam Speaker, Madam Vice President, our First Lady and Second Gentleman. Members of Congress and the Cabinet. Justices of the Supreme Court. My fellow Americans.  \n\nLast year COVID-19 kept us apart. This year we are finally together again. \n\nTonight, we meet as Democrats Republicans and Independents. But most importantly as Americans. \n\nWith a duty to one another to the American people to the Constitution. \n\nAnd with an unwavering resolve that freedom will always triumph over tyranny. \n\nSix days ago, Russia’s Vladimir Putin sought to shake the foundations of the free world thinking he could make it bend to his menacing ways. But he badly miscalculated. \n\nHe thought he could roll into Ukraine and the world would roll over. Instead he met a wall of strength he never imagined. \n\nHe met the Ukrainian people. \n\nFrom President Zelenskyy to every Ukrainian, their fearlessness, their courage, their determination, inspires the world. \n\nGroups of citizens blocking tanks with their bodies. Everyone from students to retirees teachers turned soldiers defending their homeland.
-# Source: 0-pl
-# Content: And we won’t stop. \n\nWe have lost so much to COVID-19. Time with one another. And worst of all, so much loss of life. \n\nLet’s use this moment to reset. Let’s stop looking at COVID-19 as a partisan dividing line and see it for what it is: A God-awful disease.  \n\nLet’s stop seeing each other as enemies, and start seeing each other for who we really are: Fellow Americans.  \n\nWe can’t change how divided we’ve been. But we can change how we move forward—on COVID-19 and other issues we must face together. \n\nI recently visited the New York City Police Department days after the funerals of Officer Wilbert Mora and his partner, Officer Jason Rivera. \n\nThey were responding to a 9-1-1 call when a man shot and killed them with a stolen gun. \n\nOfficer Mora was 27 years old. \n\nOfficer Rivera was 22. \n\nBoth Dominican Americans who’d grown up on the same streets they later chose to patrol as police officers. \n\nI spoke with their families and told them that we are forever in debt for their sacrifice, and we will carry on their mission to restore the trust and safety every community deserves.
-# Source: 24-pl
-# Content: And a proud Ukrainian people, who have known 30 years  of independence, have repeatedly shown that they will not tolerate anyone who tries to take their country backwards.  \n\nTo all Americans, I will be honest with you, as I’ve always promised. A Russian dictator, invading a foreign country, has costs around the world. \n\nAnd I’m taking robust action to make sure the pain of our sanctions  is targeted at Russia’s economy. And I will use every tool at our disposal to protect American businesses and consumers. \n\nTonight, I can announce that the United States has worked with 30 other countries to release 60 Million barrels of oil from reserves around the world.  \n\nAmerica will lead that effort, releasing 30 Million barrels from our own Strategic Petroleum Reserve. And we stand ready to do more if necessary, unified with our allies.  \n\nThese steps will help blunt gas prices here at home. And I know the news about what’s happening can seem alarming. \n\nBut I want you to know that we are going to be okay.
-# Source: 5-pl
-# Content: More support for patients and families. \n\nTo get there, I call on Congress to fund ARPA-H, the Advanced Research Projects Agency for Health. \n\nIt’s based on DARPA—the Defense Department project that led to the Internet, GPS, and so much more.  \n\nARPA-H will have a singular purpose—to drive breakthroughs in cancer, Alzheimer’s, diabetes, and more. \n\nA unity agenda for the nation. \n\nWe can do this. \n\nMy fellow Americans—tonight , we have gathered in a sacred space—the citadel of our democracy. \n\nIn this Capitol, generation after generation, Americans have debated great questions amid great strife, and have done great things. \n\nWe have fought for freedom, expanded liberty, defeated totalitarianism and terror. \n\nAnd built the strongest, freest, and most prosperous nation the world has ever known. \n\nNow is the hour. \n\nOur moment of responsibility. \n\nOur test of resolve and conscience, of history itself. \n\nIt is in this moment that our character is formed. Our purpose is found. Our future is forged. \n\nWell I know this nation.
-# Source: 34-pl
-# =========
-# FINAL ANSWER: The president did not mention Michael Jackson.
-# SOURCES:
-
-# QUESTION: {question}
-# =========
-# {summaries}
-# =========
-# FINAL ANSWER:"""
 QA_WSOURCES_PROMPT=hub.pull("dmueller/ams-chatbot-qa-retrieval-wsources")
-
 QA_GENERATE_PROMPT=hub.pull("dmueller/generate_qa_prompt")
+
+# Prompts defined here only
+DEFAULT_DOCUMENT_PROMPT = PromptTemplate.from_template(template="{page_content}")
+TEST_QUERY_PROMPT='What are examples of adhesives to use when potting motors for launch vehicle or spacecraft mechanisms?'

queries.py

@@ -1,145 +1,268 @@
-"""
-@author: dsmueller3760
-Query from pinecone embeddings
-"""
-from dotenv import load_dotenv, find_dotenv
-from langchain.vectorstores import Pinecone
-from langchain.embeddings import OpenAIEmbeddings
-from langchain.llms import OpenAI
+import os
+import logging
+import re
 
-from langchain.chains.qa_with_sources import load_qa_with_sources_chain
-from langchain.chains import ConversationalRetrievalChain
-from langchain.memory import ConversationBufferMemory
-from langchain.chains.llm import LLMChain
+from dotenv import load_dotenv, find_dotenv
 
-import os
+import openai
 import pinecone
+import chromadb
 
-from prompts import CONDENSE_QUESTION_PROMPT, QA_PROMPT, QA_WSOURCES_PROMPT
+from langchain_community.vectorstores import Pinecone
+from langchain_community.vectorstores import Chroma
 
+from langchain.memory import ConversationBufferMemory
 
+from operator import itemgetter
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnableLambda, RunnablePassthrough
+from langchain.schema import format_document
+from langchain_core.messages import get_buffer_string
 
+from prompts import CONDENSE_QUESTION_PROMPT, QA_PROMPT, DEFAULT_DOCUMENT_PROMPT, TEST_QUERY_PROMPT
+
+# Set secrets from environment file
+OPENAI_API_KEY=os.getenv('OPENAI_API_KEY')
+VOYAGE_API_KEY=os.getenv('VOYAGE_API_KEY')
+PINECONE_API_KEY=os.getenv('PINECONE_API_KEY')
+HUGGINGFACEHUB_API_TOKEN=os.getenv('HUGGINGFACEHUB_API_TOKEN') 
+
+# Class and functions
 class QA_Model:
     def __init__(self, 
+                 index_type,
                  index_name,
-                 embeddings_model,
+                 query_model,
                  llm,
                  k=6,
                  search_type='similarity',
+                 fetch_k=50,
                  temperature=0,
-                 verbose=False,
                  chain_type='stuff',
                  filter_arg=False):
         
-        self.index_name:str=index_name
-        self.embeddings_model:OpenAIEmbeddings=embeddings_model
+        self.index_type=index_type
+        self.index_name=index_name
+        self.query_model=query_model
         self.llm=llm
-        self.k:int=k
-        self.search_type:str=search_type
-        self.temperature:int=temperature
-        self.verbose:bool=verbose
-        self.chain_type:str=chain_type
-        self.filter_arg:bool=filter_arg
+        self.k=k
+        self.search_type=search_type
+        self.fetch_k=fetch_k
+        self.temperature=temperature
+        self.chain_type=chain_type
+        self.filter_arg=filter_arg
+        self.sources=[]
 
         load_dotenv(find_dotenv(),override=True)
 
+        # Define retriever search parameters
+        search_kwargs = _process_retriever_args(self.filter_arg,
+                                                self.sources,
+                                                self.search_type,
+                                                self.k,
+                                                self.fetch_k)
+
         # Read in from the vector database
-        self.vectorstore = Pinecone.from_existing_index(index_name,embeddings_model)
-
-        # Set up question generator and qa with sources
-        self.question_generator = LLMChain(llm=llm, 
-                                           prompt=CONDENSE_QUESTION_PROMPT,
-                                           verbose=verbose)
-        self.doc_chain = load_qa_with_sources_chain(llm, chain_type=chain_type,prompt=QA_WSOURCES_PROMPT,verbose=verbose)
-
-        # Establish chat history
-        self.chat_history=ConversationBufferMemory(memory_key='chat_history',
-                                            input_key='question',
-                                            output_key='answer',
-                                            return_messages=True)
-
-        # Implement filter
-        if filter_arg:
-            filter_list = list(set(item["source"] for item in self.sources[-1]))
-            filter_items=[]
-            for item in filter_list:
-                filter_item={"source": item}
-                filter_items.append(filter_item)
-            filter={"$or":filter_items}
-        else:
-            filter=None
+        if index_type=='Pinecone':
+            pinecone.init(
+                api_key=PINECONE_API_KEY
+            )
+            logging.info('Chat pinecone index name: '+str(index_name))
+            logging.info('Chat query model: '+str(query_model))
+            index = pinecone.Index(index_name)
+            self.vectorstore = Pinecone(index,query_model,'page_content')
+            logging.info('Chat vectorstore: '+str(self.vectorstore))
 
-        if search_type=='mmr':
-            search_kwargs={'k':k,'fetch_k':50,'filter':filter} # See as_retriever docs for parameters
-        else:
-            search_kwargs={'k':k,'filter':filter} # See as_retriever docs for parameters
-
-        self.qa = ConversationalRetrievalChain(
-                    retriever=self.vectorstore.as_retriever(search_type=search_type,
-                                                            search_kwargs=search_kwargs),  
-                    combine_docs_chain=self.doc_chain, 
-                    question_generator=self.question_generator,
-                    memory=self.chat_history,
-                    verbose=verbose,
-                    return_source_documents=True,
-                    return_generated_question=True,
-                    )
-        
-        self.sources=[]
+            # Test query
+            test_query = self.vectorstore.similarity_search(TEST_QUERY_PROMPT)
+            logging.info('Test query: '+str(test_query))
+            if not test_query:
+                raise ValueError("Pinecone vector database is not configured properly. Test query failed.")
+            else:
+                logging.info('Test query succeeded!')
+            
+            self.retriever=self.vectorstore.as_retriever(search_type=search_type,
+                                                         search_kwargs=search_kwargs)
+            logging.info('Chat retriever: '+str(self.retriever))
+        elif index_type=='ChromaDB':
+            logging.info('Chat chroma index name: '+str(index_name))
+            logging.info('Chat query model: '+str(query_model))
+            persistent_client = chromadb.PersistentClient(path='../db/chromadb')            
+            self.vectorstore = Chroma(client=persistent_client,
+                                      collection_name=index_name,
+                                      embedding_function=query_model)
+            logging.info('Chat vectorstore: '+str(self.vectorstore))
 
-    def query_docs(self,query,tags=None):
-        self.result=self.qa({'question': query},tags=tags)
+            # Test query
+            test_query = self.vectorstore.similarity_search(TEST_QUERY_PROMPT)
+            logging.info('Test query: '+str(test_query))
+            if not test_query:
+                raise ValueError("Chroma vector database is not configured properly. Test query failed.")
+            else:
+                logging.info('Test query succeeded!')
+            
+            self.retriever=self.vectorstore.as_retriever(search_type=search_type,
+                                                         search_kwargs=search_kwargs)
+            logging.info('Chat retriever: '+str(self.retriever))
+        elif index_type=='RAGatouille':
+            # Easy because the index is picked up directly.
+            self.vectorstore=query_model
+            logging.info('Chat query model:'+str(query_model))
 
-        # print('-------------')
-        # print(query+'\n')
-        # print(self.result['answer']+'\n\n'+'Sources:'+'\n')
+             # Test query
+            test_query = self.vectorstore.search(TEST_QUERY_PROMPT)
+            logging.info('Test query: '+str(test_query))
+            if not test_query:
+                raise ValueError("Chroma vector database is not configured properly. Test query failed.")
+            else:
+                logging.info('Test query succeeded!')
+            
+            self.retriever=self.vectorstore.as_langchain_retriever()
+            logging.info('Chat retriever: '+str(self.retriever))
 
-        temp_sources=[]
-        for data in self.result['source_documents']:
-            temp_sources.append(data.metadata)
-            # print(data.metadata)
+        # Intialize memory
+        self.memory = ConversationBufferMemory(
+                        return_messages=True, output_key='answer', input_key='question')
+        logging.info('Memory: '+str(self.memory))
 
-        self.sources.append(temp_sources)
-        # print('\nGenerated question: '+self.result['generated_question'])
-        # print('-------------\n')
+        # Assemble main chain
+        self.conversational_qa_chain=_define_qa_chain(self.llm,
+                                                      self.retriever,
+                                                      self.memory,
+                                                      self.search_type,
+                                                      search_kwargs)
+    def query_docs(self,query):        
+        self.memory.load_memory_variables({})
+        logging.info('Memory content before qa result: '+str(self.memory))
+
+        logging.info('Query: '+str(query))
+        self.result = self.conversational_qa_chain.invoke({'question': query})
+        logging.info('QA result: '+str(self.result))
+
+        if self.index_type!='RAGatouille':
+            self.sources = '\n'.join(str(data.metadata) for data in self.result['references'])
+            self.result['answer'].content += '\nSources: \n'+self.sources
+            logging.info('Sources: '+str(self.sources))
+            logging.info('Response with sources: '+str(self.result['answer'].content))
+        else:
+            # RAGatouille doesn't have metadata, need to extract from context first.
+            extracted_metadata = []
+            pattern = r'\{([^}]*)\}(?=[^{}]*$)' # Regular expression pattern to match the last curly braces
 
-    def update_model(self,llm,
-                    k=6,
-                    search_type='similarity',
-                    fetch_k=50,
-                    verbose=None,
-                    filter_arg=False):
+            for ref in self.result['references']:
+                match = re.search(pattern, ref.page_content)
+                if match:
+                    extracted_metadata.append("{"+match.group(1)+"}")
+            self.sources = '\n'.join(extracted_metadata)
+            self.result['answer'].content += '\nSources: \n'+self.sources
+            logging.info('Sources: '+str(self.sources))
+            logging.info('Response with sources: '+str(self.result['answer'].content))
+
+        self.memory.save_context({'question': query}, {'answer': self.result['answer'].content})
+        logging.info('Memory content after qa result: '+str(self.memory))
+
+    def update_model(self,
+                     llm,
+                     k=6,
+                     search_type='similarity',
+                     fetch_k=50,
+                     filter_arg=False):
 
         self.llm=llm
+        self.k=k
+        self.search_type=search_type
+        self.fetch_k=fetch_k
+        self.filter_arg=filter_arg
+        
+        # Define retriever search parameters
+        search_kwargs = _process_retriever_args(self.filter_arg,
+                                                self.sources,
+                                                self.search_type,
+                                                self.k,
+                                                self.fetch_k)
+        # Update conversational retrieval chain
+        self.conversational_qa_chain=_define_qa_chain(self.llm,
+                                                      self.retriever,
+                                                      self.memory,
+                                                      self.search_type,
+                                                      search_kwargs)
+        logging.info('Updated qa chain: '+str(self.conversational_qa_chain))
 
-        # Set up question generator and qa with sources
-        self.question_generator = LLMChain(llm=self.llm, prompt=CONDENSE_QUESTION_PROMPT,verbose=verbose)
-        self.doc_chain = load_qa_with_sources_chain(self.llm, chain_type=self.chain_type,prompt=QA_WSOURCES_PROMPT,verbose=verbose)
-
-        # Implement filter
-        if filter_arg:
-            print(self.sources)
-            filter_list = list(set(item["source"] for item in self.sources[-1]))
-            filter_items=[]
-            for item in filter_list:
-                filter_item={"source": item}
-                filter_items.append(filter_item)
-            filter={"$or":filter_items}
-        else:
-            filter=None
+# Internal functions
+def _combine_documents(docs, 
+                        document_prompt=DEFAULT_DOCUMENT_PROMPT, 
+                        document_separator='\n\n'):
+    '''
+    Combine a list of documents into a single string.
+    '''
+    # TODO: this would be where stuff, map reduce, etc. would go
+    doc_strings = [format_document(doc, document_prompt) for doc in docs]
+    return document_separator.join(doc_strings)
+def _define_qa_chain(llm,
+                     retriever,
+                     memory,
+                     search_type,
+                     search_kwargs):
+    '''
+    Define the conversational QA chain.
+    '''
+    # This adds a 'memory' key to the input object
+    loaded_memory = RunnablePassthrough.assign(
+                        chat_history=RunnableLambda(memory.load_memory_variables) 
+                        | itemgetter('history'))  
+    logging.info('Loaded memory: '+str(loaded_memory))
+    
+    # Assemble main chain
+    standalone_question = {
+        'standalone_question': {
+            'question': lambda x: x['question'],
+            'chat_history': lambda x: get_buffer_string(x['chat_history'])}
+        | CONDENSE_QUESTION_PROMPT
+        | llm
+        | StrOutputParser()}
+    logging.info('Condense inputs as a standalong question: '+str(standalone_question))
+    retrieved_documents = {
+        'source_documents': itemgetter('standalone_question') 
+                            | retriever,
+        'question': lambda x: x['standalone_question']}
+    logging.info('Retrieved documents: '+str(retrieved_documents))
+    # Now we construct the inputs for the final prompt
+    final_inputs = {
+        'context': lambda x: _combine_documents(x['source_documents']),
+        'question': itemgetter('question')}
+    logging.info('Combined documents: '+str(final_inputs))
+    # And finally, we do the part that returns the answers
+    answer = {
+        'answer': final_inputs 
+                    | QA_PROMPT 
+                    | llm,
+        'references': itemgetter('source_documents')}
+    conversational_qa_chain = loaded_memory | standalone_question | retrieved_documents | answer
+    logging.info('Conversational QA chain: '+str(conversational_qa_chain))
+    return conversational_qa_chain
+def _process_retriever_args(filter_arg,
+                            sources,
+                            search_type,
+                            k,
+                            fetch_k):
+    '''
+    Process arguments for retriever.
+    '''
+    # Implement filter
+    if filter_arg:
+        filter_list = list(set(item['source'] for item in sources[-1]))
+        filter_items=[]
+        for item in filter_list:
+            filter_item={'source': item}
+            filter_items.append(filter_item)
+        filter={'$or':filter_items}
+    else:
+        filter=None
 
-        if search_type=='mmr':
-            search_kwargs={'k':k,'fetch_k':fetch_k,'filter':filter} # See as_retriever docs for parameters
-        else:
-            search_kwargs={'k':k,'filter':filter} # See as_retriever docs for parameters
-
-        self.qa = ConversationalRetrievalChain(
-            retriever=self.vectorstore.as_retriever(search_type=search_type,
-                                                            search_kwargs=search_kwargs),
-            combine_docs_chain=self.doc_chain, 
-            question_generator=self.question_generator,
-            memory=self.chat_history,
-            verbose=verbose,
-            return_source_documents=True,
-            return_generated_question=True,
-            )
+    # Impement filtering and number of documents to return
+    if search_type=='mmr':
+        search_kwargs={'k':k,'fetch_k':fetch_k,'filter':filter} # See as_retriever docs for parameters
+    else:
+        search_kwargs={'k':k,'filter':filter} # See as_retriever docs for parameters
+    
+    return search_kwargs

setup.py

@@ -0,0 +1,140 @@
+import os
+import logging
+import json
+
+import openai
+
+import streamlit as st
+
+# Set up the page, enable logging 
+from dotenv import load_dotenv,find_dotenv
+load_dotenv(find_dotenv(),override=True)
+
+def load_sidebar(config_file,
+                 index_data_file,
+                 vector_databases=False,
+                 embeddings=False,
+                 rag_type=False,
+                 index_name=False,
+                 llm=False,
+                 model_options=False,
+                 secret_keys=False):
+    """
+    Sets up the sidebar based no toggled options. Returns variables with options.
+    """
+    sb_out={}
+    with open(config_file, 'r') as f:
+        config = json.load(f)
+        databases = {db['name']: db for db in config['databases']}
+        llms  = {m['name']: m for m in config['llms']}
+        logging.info('Loaded: '+config_file)
+    with open(index_data_file, 'r') as f:
+        index_data = json.load(f)
+        logging.info('Loaded: '+index_data_file)
+
+    if vector_databases:
+        # Vector databases
+        st.sidebar.title('Vector database')
+        sb_out['index_type']=st.sidebar.selectbox('Index type', list(databases.keys()), index=1)
+        logging.info('Index type: '+sb_out['index_type'])
+
+    if embeddings:
+        # Embeddings
+        st.sidebar.title('Embeddings')
+        if sb_out['index_type']=='RAGatouille':    # Default to selecting hugging face model for RAGatouille, otherwise select alternates
+           sb_out['query_model']=st.sidebar.selectbox('Hugging face rag models', databases[sb_out['index_type']]['hf_rag_models'], index=0)
+        else:
+            sb_out['query_model']=st.sidebar.selectbox('Embedding models', databases[sb_out['index_type']]['embedding_models'], index=0)
+
+        if sb_out['query_model']=='Openai':
+            sb_out['embedding_name']='text-embedding-ada-002'
+        elif sb_out['query_model']=='Voyage':
+            sb_out['embedding_name']='voyage-02'
+        logging.info('Query type: '+sb_out['query_model'])
+        if 'embedding_name' in locals() or 'embedding_name' in globals():
+            logging.info('Embedding name: '+sb_out['embedding_name'])
+    if rag_type:
+        if sb_out['index_type']!='RAGatouille': # RAGatouille doesn't have a rag_type
+            # RAG Type
+            st.sidebar.title('RAG Type')
+            sb_out['rag_type']=st.sidebar.selectbox('RAG type', config['rag_types'], index=0)
+            sb_out['smart_agent']=st.sidebar.checkbox('Smart agent?')
+            logging.info('RAG type: '+sb_out['rag_type'])
+            logging.info('Smart agent: '+str(sb_out['smart_agent']))
+    if index_name:
+        # Index Name 
+        st.sidebar.title('Index Name')  
+        sb_out['index_name']=index_data[sb_out['index_type']][sb_out['query_model']]
+        st.sidebar.markdown('Index name: '+sb_out['index_name'])
+        logging.info('Index name: '+sb_out['index_name'])
+    if llm:
+        # LLM
+        st.sidebar.title('LLM')
+        sb_out['llm_source']=st.sidebar.selectbox('LLM model', list(llms.keys()), index=0)
+        logging.info('LLM source: '+sb_out['llm_source'])
+        if sb_out['llm_source']=='OpenAI':
+            sb_out['llm_model']=st.sidebar.selectbox('OpenAI model', llms[sb_out['llm_source']]['models'], index=0)
+        if sb_out['llm_source']=='Hugging Face':
+            sb_out['llm_model']=st.sidebar.selectbox('Hugging Face model', llms[sb_out['llm_source']]['models'], index=0)
+    if model_options:
+        # Add input fields in the sidebar
+        st.sidebar.title('LLM Options')
+        temperature = st.sidebar.slider('Temperature', min_value=0.0, max_value=2.0, value=0.0, step=0.1)
+        output_level = st.sidebar.selectbox('Level of Output', ['Concise', 'Detailed'], index=1)
+
+        st.sidebar.title('Retrieval Options')
+        k = st.sidebar.number_input('Number of items per prompt', min_value=1, step=1, value=4)
+        if sb_out['index_type']!='RAGatouille':
+            search_type = st.sidebar.selectbox('Search Type', ['similarity', 'mmr'], index=0)
+            sb_out['model_options']={'output_level':output_level,
+                                    'k':k,
+                                    'search_type':search_type,
+                                    'temperature':temperature}
+        else:
+            sb_out['model_options']={'output_level':output_level,
+                                    'k':k,
+                                    'temperature':temperature}
+        logging.info('Model options: '+str(sb_out['model_options']))
+    if secret_keys:
+        # Add a section for secret keys
+        st.sidebar.title('Secret keys')
+        st.sidebar.markdown('If .env file is in directory, will use that first.')
+        sb_out['keys']={}
+        if 'llm_source' in sb_out and sb_out['llm_source'] == 'OpenAI':
+            sb_out['keys']['OPENAI_API_KEY'] = st.sidebar.text_input('OpenAI API Key', type='password')
+        elif 'query_model' in sb_out and sb_out['query_model'] == 'Openai':
+            sb_out['keys']['OPENAI_API_KEY'] = st.sidebar.text_input('OpenAI API Key', type='password')
+        if 'llm_source' in sb_out and sb_out['llm_source']=='Hugging Face':
+            sb_out['keys']['HUGGINGFACEHUB_API_TOKEN'] = st.sidebar.text_input('Hugging Face API Key', type='password')
+        if 'query_model' in sb_out and sb_out['query_model']=='Voyage':
+            sb_out['keys']['VOYAGE_API_KEY'] = st.sidebar.text_input('Voyage API Key', type='password')
+        if 'index_type' in sb_out and sb_out['index_type']=='Pinecone':
+            sb_out['keys']['PINECONE_API_KEY']=st.sidebar.text_input('Pinecone API Key',type='password')
+    return sb_out
+def set_secrets(sb):
+    """
+    Sets secrets from environment file, or from sidebar if not available.
+    """
+    secrets={}
+    secrets['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY')
+    openai.api_key = secrets['OPENAI_API_KEY']
+    if not secrets['OPENAI_API_KEY']:
+        secrets['OPENAI_API_KEY'] = sb['keys']['OPENAI_API_KEY']
+        os.environ['OPENAI_API_KEY'] = secrets['OPENAI_API_KEY']
+        openai.api_key = secrets['OPENAI_API_KEY']
+
+    secrets['VOYAGE_API_KEY'] = os.getenv('VOYAGE_API_KEY')
+    if not secrets['VOYAGE_API_KEY']:
+        secrets['VOYAGE_API_KEY'] = sb['keys']['VOYAGE_API_KEY']
+        os.environ['VOYAGE_API_KEY'] = secrets['VOYAGE_API_KEY']
+
+    secrets['PINECONE_API_KEY'] = os.getenv('PINECONE_API_KEY')
+    if not secrets['PINECONE_API_KEY']:
+        secrets['PINECONE_API_KEY'] = sb['keys']['PINECONE_API_KEY']
+        os.environ['PINECONE_API_KEY'] = secrets['PINECONE_API_KEY']
+
+    secrets['HUGGINGFACEHUB_API_TOKEN'] = os.getenv('HUGGINGFACEHUB_API_TOKEN')
+    if not secrets['HUGGINGFACEHUB_API_TOKEN']:
+        secrets['HUGGINGFACEHUB_API_TOKEN'] = sb['keys']['HUGGINGFACEHUB_API_TOKEN']
+        os.environ['HUGGINGFACEHUB_API_TOKEN'] = secrets['HUGGINGFACEHUB_API_TOKEN']
+    return secrets