Implemented LLM model and wired it to APIs

README.md

@@ -42,6 +42,6 @@ EduConnect/
 │   └── images/                      # UI rendering images for administration page
 ├── Dockerfile                       # Docker configuration for setting up the environment
 ├── requirements.txt                 # Lists all Python library dependencies
-├── entrypoint.sh
+├── entrypoint.sh                    # To download model from hugging face
 └── .env                             # Environment variables for configuration settings
 ```

app/admin/admin_functions.py

@@ -20,7 +20,7 @@ async def register_user(db, email: str, name: str, role: str, file: UploadFile =
     :return: email 
     """
     unique_filename = f"{email}.jpg"  # Use the email as the filename
-    file_path = f"/home/user/data/tmp/{unique_filename}"  # Specify your upload directory
+    file_path = f"/home/user/data/tmp/{unique_filename}"  # Specify our upload directory
 
     # Ensure the directory exists
     os.makedirs(os.path.dirname(file_path), exist_ok=True)
@@ -35,7 +35,7 @@ async def register_user(db, email: str, name: str, role: str, file: UploadFile =
     
     if cropped_face is not None:
         
-        # Here you can store the embeddings along with user details in ChromaDB
+        # Here we can store the embeddings along with user details in ChromaDB
         # chroma_db.save_embeddings(user_id, embeddings)
         db.upsert(images=[cropped_face], ids=[email], metadatas=[{"name":name, "role":role}])
         return {"status": "User registered successfully", "image": cropped_face}
@@ -68,7 +68,7 @@ def verify_admin_password(submitted_user: str, submitted_password: str) -> bool:
     return False
 
 # Additional Admin Functions
-# You could include other administrative functionalities here, such as:
+# we could include other administrative functionalities here, such as:
 # - Listing all registered users.
 # - Moderating chat messages or viewing chat history.
 # - Managing system settings or configurations.

app/api/userchat.py

@@ -1,16 +1,23 @@
 from fastapi import APIRouter, Depends, HTTPException, Body
 from ..dependencies import get_current_user
+from ..utils.chat_rag import llm_infer
 from typing import Any
 
 router = APIRouter()
 
 @router.post("/user/chat")
 async def chat_with_llama(user_input: str = Body(..., embed=True), current_user: Any = Depends(get_current_user)):
-    # Implement your logic to interact with LlamaV2 LLM here.
-    # Example response, replace with actual chat logic
-    chat_response = "Hello, how can I assist you today?"
+    # Example logic for model inference (pseudo-code, adjust as necessary)
+    try:
+        user_id = current_user["user_id"]
+        model_response = llm_infer(user_id=user_id, prompt=user_input)
+        # Optionally, store chat history
+        # chromadb_face_helper.store_chat_history(user_id=current_user["user_id"], user_input=user_input, model_response=model_response)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+    
     return {
-        "response": chat_response,
+        "ai_response": model_response,
         "user_id": current_user["user_id"],
         "name": current_user["name"],
         "role": current_user["role"]

app/api/userlogout.py

@@ -1,5 +1,5 @@
 from fastapi import APIRouter, Depends, HTTPException
-from ..utils.db import tinydb_helper  # Ensure this import is correct based on your project structure
+from ..utils.db import tinydb_helper  # Ensure this import is correct based on our project structure
 from ..dependencies import oauth2_scheme
 
 router = APIRouter()

app/api/userupload.py

@@ -1,22 +1,38 @@
 from typing import Any
-from fastapi import APIRouter, Depends, UploadFile, File
+from fastapi import APIRouter, Depends, UploadFile, File, HTTPException
 import os
 from app.dependencies import get_current_user
+# Assuming a utility for processing PDFs and generating embeddings
+from ..utils.doc_ingest import ingest_document
 
 router = APIRouter()
 
 @router.post("/user/upload")
 async def upload_file(file: UploadFile = File(...), current_user: Any = Depends(get_current_user)):
+    if file.content_type != "application/pdf":
+        raise HTTPException(status_code=400, detail="Unsupported file type. Please upload a PDF.")
+    
     upload_dir = "/home/user/data/uploads"
-    # Ensure the upload directory exists
-    os.makedirs(upload_dir, exist_ok=True)  # This creates the directory if it doesn't exist, does nothing otherwise
+    os.makedirs(upload_dir, exist_ok=True)
     
     file_location = f"{upload_dir}/{file.filename}"
     with open(file_location, "wb") as buffer:
         contents = await file.read()
         buffer.write(contents)
+    
+    try:
+        # Process PDF and store embeddings 
+        ingest_document(file_location, current_user["user_id"])
+    except Exception as e:
+        # If processing fails, attempt to clean up the file before re-raising the error
+        os.remove(file_location)
+        raise HTTPException(status_code=500, detail=f"Failed to process file: {e}")
+
+    # Clean up file in uploads directory after successful processing
+    os.remove(file_location)
+
     return {
-        "status": "File uploaded successfully.",
+        "status": "File uploaded and processed successfully.",
         "user_id": current_user["user_id"],
         "name": current_user["name"],
         "role": current_user["role"]

app/dependencies.py

@@ -2,7 +2,7 @@ from fastapi import Depends, HTTPException, status
 from fastapi.security import OAuth2PasswordBearer
 from jose import jwt, JWTError
 from .utils.db import tinydb_helper  # Ensure correct import path
-from .utils.jwt_utils import SECRET_KEY, ALGORITHM  # Ensure these are defined in your jwt_utils.py
+from .utils.jwt_utils import SECRET_KEY, ALGORITHM  # Ensure these are defined in our jwt_utils.py
 
 oauth2_scheme = OAuth2PasswordBearer(tokenUrl="token")
 

app/main.py

@@ -16,7 +16,7 @@ CHROMADB_LOC = "/home/user/data/chromadb"
 app = FastAPI()
 
 # Add middleware
-# Set all origins to wildcard for simplicity, but you should limit this in production
+# Set all origins to wildcard for simplicity, but we should limit this in production
 app.add_middleware(
     CORSMiddleware,
     allow_origins=["*"],

app/utils/chat_rag.py

@@ -1,4 +1,5 @@
 #list of librarys for requirement.txt
+import os
 from langchain.document_loaders import PyPDFLoader
 
 # Import embeddings module from langchain for vector representations of text
@@ -26,7 +27,20 @@ from langchain.chains import ConversationalRetrievalChain
 from langchain.callbacks.manager import CallbackManager
 from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
 
-def pdf_to_vec(filename):
+CHROMADB_LOC = "/home/user/data/chromadb"
+
+# Modify vectordb initialization to be dynamic based on user_id
+def get_vectordb_for_user(user_id):
+    collection_name = f"user_{user_id}_collection"
+    vectordb = Chroma(
+        collection_name=collection_name,
+        embedding_function=HuggingFaceEmbeddings(model_name='sentence-transformers/all-MiniLM-L6-v2'),
+        persist_directory=f"{CHROMADB_LOC}/{collection_name}", # Optional: Separate directory for each user's data
+    )
+    return vectordb
+
+
+def pdf_to_vec(filename, collection_name):
     document = []
     loader = PyPDFLoader(filename)
     document.extend(loader.load()) #which library is this from?
@@ -41,7 +55,7 @@ def pdf_to_vec(filename):
     document_chunks = document_splitter.split_documents(document) #which library is this from?
 
     # Create a Chroma vector database from the document chunks with the specified embeddings, and set a directory for persistence
-    vectordb = Chroma.from_documents(document_chunks, embedding=embeddings, persist_directory='./data') ## change to GUI path
+    vectordb = Chroma.from_documents(document_chunks, embedding=embeddings, collection_name=collection_name, persist_directory=CHROMADB_LOC) ## change to GUI path
 
     # Persist the created vector database to disk in the specified directory
     vectordb.persist() #this is mandatory?
@@ -49,11 +63,19 @@ def pdf_to_vec(filename):
     return(vectordb)
     #return collection  # Return the collection as the asset
 
-def load_llm():
-    #callback_manager = CallbackManager([StreamingStdOutCallbackHandler()])
-    llm = LlamaCpp(
+class LlamaModelSingleton:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            print('Loading LLM model...')
+            cls._instance = super(LlamaModelSingleton, cls).__new__(cls)
+            
+            # Model loading logic
+            model_path = os.getenv("MODEL_PATH")
+            cls._instance.llm = LlamaCpp(
                 #streaming = True,
-                model_path="/content/llama-2-7b-mini-ibased.Q5_K_M.gguf", #/content/data/llama-2-7b-mcq_2-gguf.gguf. # change to GUI path. llama-2-7b-mini-ibased.Q5_K_M.gguf llama-2-7b-mcq_2.Q5_K_M.gguf
+                model_path=model_path,
                 #n_gpu_layers=-1,
                 n_batch=512,
                 temperature=0.1,
@@ -61,17 +83,23 @@ def load_llm():
                 #verbose=False,
                 #callback_manager=callback_manager,
                 max_tokens=2000,
-                )
-    return llm
+            )
+            print(f'Model loaded from {model_path}')
+        return cls._instance.llm
 
 
-#step 5, to instantiate once to create default_chain,router_chain,destination_chains into chain and set vectordb. so will not re-create per prompt
-def default_chain():
+def load_llm():
+    return LlamaModelSingleton()
+
 
+
+#step 5, to instantiate once to create default_chain,router_chain,destination_chains into chain and set vectordb. so will not re-create per prompt
+def default_chain(llm, user_id):
+    vectordb = get_vectordb_for_user(user_id)  # Use the dynamic vectordb based on user_id
     sum_template = """
-    As a machine learning education specialist, your expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
+    As a machine learning education specialist, our expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
 
-    Your role entails:
+    our role entails:
 
     Providing Detailed Explanations: Deliver comprehensive answers to these questions, elucidating the underlying technical principles.
     Assisting in Exam Preparation: Support educators in formulating sophisticated exam and quiz questions, including MCQs, accompanied by thorough explanations.
@@ -82,9 +110,9 @@ def default_chain():
     {input}"""
 
     mcq_template = """
-    As a machine learning education specialist, your expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
+    As a machine learning education specialist, our expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
 
-    Your role entails:
+    our role entails:
     Crafting Insightful Questions: Develop thought-provoking questions that explore the intricacies of machine learning topics.
     Generating MCQs: Create MCQs for each machine learning topic, comprising a question, four choices (A-D), and the correct answer, along with a rationale explaining the answer.
 
@@ -140,7 +168,14 @@ def default_chain():
 
     return default_chain,router_chain,destination_chains
 
-def llm_infer(default_chain,router_chain,destination_chains,prompt):
+# Adjust llm_infer to accept user_id and use it for user-specific processing
+def llm_infer(user_id, prompt):
+    
+    llm = load_llm()  # load_llm is singleton for entire system
+    
+    vectordb = get_vectordb_for_user(user_id) # Vector collection for each us. 
+    
+    default_chain, router_chain, destination_chains = get_or_create_chain(user_id, llm)  # Now user-specific
 
     chain = MultiPromptChain(
         router_chain=router_chain,
@@ -151,4 +186,93 @@ def llm_infer(default_chain,router_chain,destination_chains,prompt):
     )
     response = chain.run(prompt)
 
-    return response
+    return response
+
+# Assuming a simplified caching mechanism for demonstration
+chain_cache = {}
+
+def get_or_create_chain(user_id, llm):
+    if 'default_chain' in chain_cache and 'router_chain' in chain_cache:
+        default_chain = chain_cache['default_chain']
+        router_chain = chain_cache['router_chain']
+        destination_chains = chain_cache['destination_chains']
+    else:
+        vectordb = get_vectordb_for_user(user_id)  # User-specific vector database
+        sum_template = """
+        As a machine learning education specialist, our expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
+
+        our role entails:
+
+        Providing Detailed Explanations: Deliver comprehensive answers to these questions, elucidating the underlying technical principles.
+        Assisting in Exam Preparation: Support educators in formulating sophisticated exam and quiz questions, including MCQs, accompanied by thorough explanations.
+        Summarizing Course Material: Distill key information from course materials, articulating complex ideas within the context of advanced machine learning practices.
+
+        Objective: to summarize and explain the key points.
+        Here the question:
+        {input}"""
+
+        mcq_template = """
+        As a machine learning education specialist, our expertise is pivotal in deepening the comprehension of complex machine learning concepts for both educators and students.
+
+        our role entails:
+        Crafting Insightful Questions: Develop thought-provoking questions that explore the intricacies of machine learning topics.
+        Generating MCQs: Create MCQs for each machine learning topic, comprising a question, four choices (A-D), and the correct answer, along with a rationale explaining the answer.
+
+        Objective: to create multiple choice question in this format
+        [question:
+        options A:
+        options B:
+        options C:
+        options D:
+        correct_answer:
+        explanation:]
+
+        Here the question:
+        {input}"""
+
+        prompt_infos = [
+            {
+                "name": "SUMMARIZE",
+                "description": "Good for summarizing and explaination ",
+                "prompt_template": sum_template,
+            },
+            {
+                "name": "MCQ",
+                "description": "Good for creating multiple choices questions",
+                "prompt_template": mcq_template,
+            },
+        ]
+
+        destination_chains = {}
+
+        for p_info in prompt_infos:
+            name = p_info["name"]
+            prompt_template = p_info["prompt_template"]
+            prompt = PromptTemplate(template=prompt_template, input_variables=["input"])
+            chain = LLMChain(llm=llm, prompt=prompt)
+            destination_chains[name] = chain
+        #default_chain = ConversationChain(llm=llm, output_key="text")
+        #memory = ConversationBufferMemory(memory_key='chat_history', return_messages=True)
+
+        default_chain = ConversationalRetrievalChain.from_llm(llm=llm,
+                                                    retriever=vectordb.as_retriever(search_kwargs={'k': 3}),
+                                                    verbose=True, output_key="text" )
+
+        destinations = [f"{p['name']}: {p['description']}" for p in prompt_infos]
+        destinations_str = "\n".join(destinations)
+        router_template = MULTI_PROMPT_ROUTER_TEMPLATE.format(destinations=destinations_str)
+        router_prompt = PromptTemplate(
+            template=router_template,
+            input_variables=["input"],
+            output_parser=RouterOutputParser(),
+        )
+        router_chain = LLMRouterChain.from_llm(llm, router_prompt)
+#
+        chain_cache['default_chain'] = default_chain
+        chain_cache['router_chain'] = router_chain
+        chain_cache['destination_chains'] = destination_chains
+    
+    # Here we can adapt the chains if needed based on the user_id, for example, by adjusting the vectordb retriever
+    # This is where user-specific adaptations occur
+
+    return default_chain, router_chain, destination_chains

app/utils/db.py

@@ -20,7 +20,7 @@ class TinyDBHelper:
     def query_token(self, user_id: str, token: str) -> bool:
         """Query to check if the token exists and is valid."""
         User = Query()
-        # Assuming your tokens table contains 'user_id', 'token', and 'expires_at'
+        # Assuming our tokens table contains 'user_id', 'token', and 'expires_at'
         result = self.tokens_table.search((User.user_id == user_id) & (User.token == token))
         # Optionally, check if the token is expired
         expires_at = datetime.fromisoformat(result[0]['expires_at'])

app/utils/doc_ingest.py

@@ -0,0 +1,19 @@
+# utils/doc_ingest.py
+from .chat_rag import pdf_to_vec
+
+def ingest_document(file_location: str, user_id: str):
+    """
+    Process and ingest a document into a user-specific vector database.
+    
+    :param file_location: The location of the uploaded file on the server.
+    :param user_id: The ID of the user uploading the document.
+    """
+    # Construct a unique collection name based on user_id
+    collection_name = f"user_{user_id}_collection"
+    
+    try:
+        vectordb = pdf_to_vec(file_location, collection_name)
+        print("Document processed and ingested successfully into user-specific collection.")
+    except Exception as e:
+        print(f"Error processing document for user {user_id}: {e}")
+        raise

docs/design_principles.md

@@ -0,0 +1,101 @@
+# DESIGN PRINCIPLES
+
+## VectorDB collections
+
+Given that our EduConnect project is an academic project with a scale limited to not more than 10 users, the approach of creating separate vector stores (ChromaDB collections) for each user becomes highly feasible and manageable. This small scale alleviates concerns about scalability and management overhead that would be significant in a larger, production-level system. Here's how we can effectively implement and manage user-specific vector stores under these conditions:
+
+### Implementation Strategy for Small Scale
+
+1. **Simplified Database Management**: With a maximum of 10 users, managing separate ChromaDB collections becomes straightforward. we can manually monitor and maintain these collections without the need for automated scalability solutions.
+
+2. **Personalized Data Handling**: This setup allows for a high degree of personalization in data handling and retrieval. Each user's interactions and uploads can be contained within their dedicated collection, ensuring data isolation and relevance.
+
+3. **Performance Considerations**: Performance issues related to managing multiple collections are negligible at this scale. Each user's collection will be relatively small, ensuring quick access and query times.
+
+4. **Security and Privacy**: Maintaining separate collections for each user naturally enhances data privacy, as there is a clear separation of data at the database level.
+
+### Example Adjustments
+
+Given the small scale of our project, we might not need to implement complex dynamic collection management. Instead, we can hard-code the logic to create or select a collection based on the user ID. Here is a simplified example adjustment to our document ingestion logic:
+
+```python
+# utils/doc_ingest.py
+def ingest_document(file_location: str, user_id: str):
+    """
+    Process and ingest a document into a user-specific vector database.
+    
+    :param file_location: The location of the uploaded file on the server.
+    :param user_id: The ID of the user uploading the document.
+    """
+    # Construct a unique collection name based on user_id
+    collection_name = f"user_{user_id}_collection"
+    
+    try:
+        vectordb = pdf_to_vec(file_location, collection_name)
+        print("Document processed and ingested successfully into user-specific collection.")
+    except Exception as e:
+        print(f"Error processing document for user {user_id}: {e}")
+        raise
+```
+
+For `pdf_to_vec`, ensure it uses the `collection_name` to store the embeddings in the correct user-specific collection:
+
+```python
+def pdf_to_vec(filename, collection_name):
+    # Logic to process the PDF and store its embeddings in vectordb
+    # Use collection_name for ChromaDB collection
+    # This function will now be more aware of user-specific storage requirements
+```
+
+### Final Notes
+
+Given the academic nature and small scale of our project, focusing on implementing clean, maintainable code that clearly demonstrates the functionality and benefits of user-specific data handling is more valuable than worrying about scalability. This approach also serves as a good model for how similar systems could be architected to scale with more users, by introducing more automated and dynamic management of resources and collections.
+
+## DEFAULT CHAIN
+
+Configuring `default_chain` for each chat interaction, especially when it involves setting up multiple components like template parsing, vector database retrieval, and language model routing for every single request, could indeed introduce overhead and potentially impact performance. This overhead is particularly concerning if the configuration process is resource-intensive, involving complex computations or significant memory allocation.
+
+### Strategies to Optimize Performance
+
+1. **Caching Common Components**: Components that don't change frequently, such as prompt templates and certain chain configurations, can be cached. This way, we avoid re-initializing these components for every chat interaction. we can initialize these components once and reuse them across chat sessions.
+
+2. **Lazy Initialization**: Only initialize certain parts of the chain when they are actually needed. If certain prompts or chains are used more frequently than others, we could prioritize their initialization and delay others until they're required.
+
+3. **Preconfigured Chain Templates**: If the customization per user is limited to a few parameters (such as the vector database they're interacting with), consider creating a preconfigured template for the chains that can be quickly cloned or adapted per user session with minimal overhead.
+
+4. **Efficient Retrieval Mechanism**: For the vector database retriever used in `ConversationalRetrievalChain`, ensure that the mechanism to switch between user-specific databases is optimized. This might mean having a lightweight way of switching context without needing to reload or reinitialize the entire database connection or retrieval logic.
+
+### Implementation Example
+
+Here's an example of how we might implement a caching mechanism for `default_chain` components that are common across users:
+
+```python
+# Assuming a simplified caching mechanism for demonstration
+chain_cache = {}
+
+def get_or_create_chain(user_id, llm):
+    if 'default_chain' in chain_cache and 'router_chain' in chain_cache:
+        default_chain = chain_cache['default_chain']
+        router_chain = chain_cache['router_chain']
+        destination_chains = chain_cache['destination_chains']
+    else:
+        vectordb = get_vectordb_for_user(user_id)  # User-specific vector database
+        # Configuration for default_chain, router_chain, and destination_chains as before
+        # [...]
+        chain_cache['default_chain'] = default_chain
+        chain_cache['router_chain'] = router_chain
+        chain_cache['destination_chains'] = destination_chains
+    
+    # Here we can adapt the chains if needed based on the user_id, for example, by adjusting the vectordb retriever
+    # This is where user-specific adaptations occur
+
+    return default_chain, router_chain, destination_chains
+```
+
+### Key Points
+
+- **Reuse and Cache**: Reuse components wherever possible, caching configurations that are static or common across interactions.
+- **Minimize Dynamic Configuration**: Minimize the amount of dynamic configuration needed per interaction by using templates and parameters that can be easily switched out.
+- **Optimize Data Layer**: Ensure the data layer (e.g., user-specific vector databases) is optimized for quick switching or context updates to prevent it from becoming a bottleneck.
+
+Adopting these strategies will help maintain responsiveness and efficiency in our chat application, ensuring that overhead from setting up `default_chain` for each interaction is minimized.

requirements.txt

@@ -1,6 +1,6 @@
 fastapi==0.95.2                # Core framework for building APIs.
 uvicorn[standard]==0.18.3      # ASGI server for FastAPI, supports live reloading.
-requests==2.28.*               # For making HTTP requests, if needed by your app.
+requests==2.28.*               # For making HTTP requests, if needed by our app.
 torch==1.11.*                  # PyTorch, for handling deep learning models.
 sentencepiece==0.1.*           # For chat text processing
 mtcnn==0.1.1                   # For face detection in images.