Fixed vectodb collection issue

app/api/userchat.py

@@ -1,6 +1,7 @@
 from fastapi import APIRouter, Depends, HTTPException, Body
 from ..dependencies import get_current_user
 from ..utils.chat_rag import llm_infer
+from ..utils.chat_rag import sanitize_collection_name
 from typing import Any
 
 router = APIRouter()
@@ -10,7 +11,7 @@ async def chat_with_llama(user_input: str = Body(..., embed=True), current_user:
     # Example logic for model inference (pseudo-code, adjust as necessary)
     try:
         user_id = current_user["user_id"]
-        model_response = llm_infer(user_id=user_id, prompt=user_input)
+        model_response = llm_infer(user_collection_name=sanitize_collection_name(user_id), prompt=user_input)
         # Optionally, store chat history
         # chromadb_face_helper.store_chat_history(user_id=current_user["user_id"], user_input=user_input, model_response=model_response)
     except Exception as e:

app/api/userupload.py

@@ -4,6 +4,7 @@ import os
 from app.dependencies import get_current_user
 # Assuming a utility for processing PDFs and generating embeddings
 from ..utils.doc_ingest import ingest_document
+from ..utils.chat_rag import sanitize_collection_name
 
 router = APIRouter()
 
@@ -22,7 +23,7 @@ async def upload_file(file: UploadFile = File(...), current_user: Any = Depends(
     
     try:
         # Process PDF and store embeddings 
-        ingest_document(file_location, current_user["user_id"])
+        ingest_document(file_location, sanitize_collection_name(current_user["user_id"]))
     except Exception as e:
         # If processing fails, attempt to clean up the file before re-raising the error
         os.remove(file_location)

app/utils/chat_rag.py

@@ -1,5 +1,7 @@
 #list of librarys for requirement.txt
 import os
+import re
+import hashlib
 from langchain.document_loaders import PyPDFLoader
 
 # Import embeddings module from langchain for vector representations of text
@@ -29,18 +31,32 @@ from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 
 CHROMADB_LOC = "/home/user/data/chromadb"
 
+def sanitize_collection_name(email):
+    # Replace invalid characters with an underscore
+    sanitized = re.sub(r'[^a-zA-Z0-9_-]', '_', email)
+    # Ensure the name is within the length limits
+    if len(sanitized) > 63:
+        # Hashing the name to ensure uniqueness and length constraint
+        hash_suffix = hashlib.sha256(email.encode()).hexdigest()[:8]
+        sanitized = sanitized[:55] + "_" + hash_suffix
+    # Ensure it starts and ends with an alphanumeric character
+    if not re.match(r'^[a-zA-Z0-9].*[a-zA-Z0-9]$', sanitized):
+        sanitized = "a" + sanitized + "1"
+    return sanitized
+
+
 # Modify vectordb initialization to be dynamic based on user_id
-def get_vectordb_for_user(user_id):
-    collection_name = f"user_{user_id}_collection"
+def get_vectordb_for_user(user_collection_name):
+   
     vectordb = Chroma(
-        collection_name=collection_name,
+        collection_name=user_collection_name,
         embedding_function=HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2'),
-        persist_directory=f"{CHROMADB_LOC}/{collection_name}", # Optional: Separate directory for each user's data
+        persist_directory=f"{CHROMADB_LOC}/{user_collection_name}", # Optional: Separate directory for each user's data
     )
     return vectordb
 
 
-def pdf_to_vec(filename, collection_name):
+def pdf_to_vec(filename, user_collection_name):
     document = []
     loader = PyPDFLoader(filename)
     document.extend(loader.load()) #which library is this from?
@@ -55,7 +71,7 @@ def pdf_to_vec(filename, collection_name):
     document_chunks = document_splitter.split_documents(document) #which library is this from?
 
     # Create a Chroma vector database from the document chunks with the specified embeddings, and set a directory for persistence
-    vectordb = Chroma.from_documents(document_chunks, embedding=embeddings, collection_name=collection_name, persist_directory=CHROMADB_LOC) ## change to GUI path
+    vectordb = Chroma.from_documents(document_chunks, embedding=embeddings, collection_name=user_collection_name, persist_directory=CHROMADB_LOC) ## change to GUI path
 
     # Persist the created vector database to disk in the specified directory
     vectordb.persist() #this is mandatory?
@@ -94,8 +110,8 @@ def load_llm():
 
 
 #step 5, to instantiate once to create default_chain,router_chain,destination_chains into chain and set vectordb. so will not re-create per prompt
-def default_chain(llm, user_id):
-    vectordb = get_vectordb_for_user(user_id)  # Use the dynamic vectordb based on user_id
+def default_chain(llm, user_collection_name):
+    vectordb = get_vectordb_for_user(user_collection_name)  # Use the dynamic vectordb based on user_id
     sum_template = """
     As a machine learning education specialist, our expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
 
@@ -169,13 +185,13 @@ def default_chain(llm, user_id):
     return default_chain,router_chain,destination_chains
 
 # Adjust llm_infer to accept user_id and use it for user-specific processing
-def llm_infer(user_id, prompt):
+def llm_infer(user_collection_name, prompt):
     
     llm = load_llm()  # load_llm is singleton for entire system
     
-    vectordb = get_vectordb_for_user(user_id) # Vector collection for each us. 
+    vectordb = get_vectordb_for_user(user_collection_name) # Vector collection for each us. 
     
-    default_chain, router_chain, destination_chains = get_or_create_chain(user_id, llm)  # Now user-specific
+    default_chain, router_chain, destination_chains = get_or_create_chain(user_collection_name, llm)  # Now user-specific
 
     chain = MultiPromptChain(
         router_chain=router_chain,
@@ -191,13 +207,13 @@ def llm_infer(user_id, prompt):
 # Assuming a simplified caching mechanism for demonstration
 chain_cache = {}
 
-def get_or_create_chain(user_id, llm):
+def get_or_create_chain(user_collection_name, llm):
     if 'default_chain' in chain_cache and 'router_chain' in chain_cache:
         default_chain = chain_cache['default_chain']
         router_chain = chain_cache['router_chain']
         destination_chains = chain_cache['destination_chains']
     else:
-        vectordb = get_vectordb_for_user(user_id)  # User-specific vector database
+        vectordb = get_vectordb_for_user(user_collection_name)  # User-specific vector database
         sum_template = """
         As a machine learning education specialist, our expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
 

requirements.txt

@@ -1,7 +1,7 @@
 fastapi==0.95.2                # Core framework for building APIs.
 uvicorn[standard]==0.18.3      # ASGI server for FastAPI, supports live reloading.
 requests==2.28.*               # For making HTTP requests, if needed by our app.
-torch==1.11.*                  # PyTorch, for handling deep learning models.
+torch                          # PyTorch, for handling deep learning models.
 sentencepiece==0.1.*           # For chat text processing
 mtcnn==0.1.1                   # For face detection in images.
 python-jose[cryptography]==3.3.*  # For creating, parsing, and verifying JWT tokens.