Update Repository

api/function.py

@@ -1,57 +1,62 @@
-from script.build_vector import IndexManager
+from script.vector_db import IndexManager
 from script.document_uploader import Uploader
 from db.save_data import InsertDatabase
 from db.get_data import GetDatabase
 from db.delete_data import DeleteDatabase
 from db.update_data import UpdateDatabase
-from typing import Any
+
+from typing import Any, Optional, List
 from fastapi import UploadFile
 from fastapi import HTTPException
+
+from service.dto import ChatMessage
 from core.chat.engine import Engine
+from core.chat.chatstore import ChatStore
 from core.parser import clean_text, update_response, renumber_sources, seperate_to_list
-from llama_index.core.composability import QASummaryQueryEngineBuilder
-from service.dto import BotResponseStreaming, TestStreaming
+from llama_index.core.llms import MessageRole
+from service.dto import BotResponseStreaming
 from service.aws_loader import Loader
 
 import logging
 import re
+import json
 
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)
 
-
 # async def data_ingestion(
 #     db_conn, reference, file: UploadFile, content_table: UploadFile
 # ) -> Any:
 
-async def data_ingestion(
-    db_conn, reference, file: UploadFile
-) -> Any:
-
-    insert_database = InsertDatabase(db_conn)
-    
-    file_name = f"{reference['title']}.pdf"
-    aws_loader = Loader()
-    
-    file_obj = file
-    aws_loader.upload_to_s3(file_obj, file_name)
-    
-    print("Uploaded Success")
 
+async def data_ingestion(db_conn, reference, file: UploadFile) -> Any:
     try:
-        # Insert data into the database
-        await insert_database.insert_data(reference)
-        
-        # uploader = Uploader(reference, file, content_table)
-        uploader = Uploader(reference, file)
-        print("uploader : ", uploader)
+
+        # insert_database = InsertDatabase(db_conn)
+
+        file_name = f"{reference['title']}"
+        aws_loader = Loader()
+
+        file_obj = file
+        aws_loader.upload_to_s3(file_obj, file_name)
+
+        print("Uploaded Success")
         
-        nodes_with_metadata = await uploader.process_documents()
+        response = json.dumps({"status": "success", "message": "Vector Index loaded successfully."})
+
+        # Insert data into the database
+        # await insert_database.insert_data(reference)
+
+        # # uploader = Uploader(reference, file, content_table)
+        # uploader = Uploader(reference, file)
+        # print("uploader : ", uploader)
 
-        # Build indexes using IndexManager
-        index = IndexManager()
-        response = index.build_indexes(nodes_with_metadata)
+        # nodes_with_metadata = await uploader.process_documents()
+
+        # # Build indexes using IndexManager
+        # index = IndexManager()
+        # response = index.build_indexes(nodes_with_metadata)
 
         return response
 
@@ -63,6 +68,7 @@ async def data_ingestion(
             detail="An internal server error occurred in data ingestion.",
         )
 
+
 async def get_data(db_conn, title="", fetch_all_data=True):
     get_database = GetDatabase(db_conn)
     print(get_database)
@@ -118,21 +124,31 @@ async def delete_data(id: int, db_conn):
         )
 
 
-def generate_completion_non_streaming(user_request, chat_engine):
+def generate_completion_non_streaming(
+    session_id, user_request, chat_engine, title=None, category=None, type="general"
+):
     try:
         engine = Engine()
         index_manager = IndexManager()
+        chatstore = ChatStore()
 
         # Load existing indexes
         index = index_manager.load_existing_indexes()
 
-        # Retrieve the chat engine with the loaded index
-        chat_engine = engine.get_chat_engine(index)
+        if type == "general":
+            # Retrieve the chat engine with the loaded index
+            chat_engine = engine.get_chat_engine(session_id, index)
+        else:
+            # Retrieve the chat engine with the loaded index
+            chat_engine = engine.get_chat_engine(
+                session_id, index, title=title, category=category
+            )
 
         # Generate completion response
         response = chat_engine.chat(user_request)
 
         sources = response.sources
+        print(sources)
 
         number_reference = list(set(re.findall(r"\[(\d+)\]", str(response))))
         number_reference_sorted = sorted(number_reference)
@@ -153,10 +169,8 @@ def generate_completion_non_streaming(user_request, chat_engine):
 
                     # Pastikan number valid sebagai indeks
                     if 0 <= number - 1 < len(node):
-                        print(node[number - 1].node.get_text())
-                        raw_content = seperate_to_list(
-                            node[number - 1].node.get_text()
-                        )
+
+                        raw_content = seperate_to_list(node[number - 1].node.get_text())
                         raw_contents.append(raw_content)
 
                         content = clean_text(node[number - 1].node.get_text())
@@ -176,7 +190,7 @@ def generate_completion_non_streaming(user_request, chat_engine):
 
         response = update_response(str(response))
         contents = renumber_sources(contents)
-        
+
         # Check the lengths of content and metadata
         num_content = len(contents)
         num_metadata = len(metadata_collection)
@@ -185,6 +199,14 @@ def generate_completion_non_streaming(user_request, chat_engine):
         for i in range(min(num_content, num_metadata)):
             metadata_collection[i]["content"] = re.sub(r"source \d+\:", "", contents[i])
 
+        message = ChatMessage(
+            role=MessageRole.ASSISTANT, content=response, metadata=metadata_collection
+        )
+
+        chatstore.delete_last_message(session_id)
+        chatstore.add_message(session_id, message)
+        chatstore.clean_message(session_id)
+
         return str(response), raw_contents, contents, metadata_collection, scores
     except Exception as e:
         # Log the error and raise HTTPException for FastAPI

api/router/bot.py

@@ -1,16 +1,49 @@
-from fastapi import APIRouter
+from fastapi import APIRouter, HTTPException, Depends
 from service.dto import UserPromptRequest, BotResponse
+from core.chat.chatstore import ChatStore
 
 from api.function import (
     generate_streaming_completion,
     generate_completion_non_streaming,
 )
 from sse_starlette.sse import EventSourceResponse
+from utils.utils import generate_uuid
 
 router = APIRouter(tags=["Bot"])
 
+def get_chat_store():
+    return ChatStore()
 
-@router.post("/bot")
+@router.post("/bot/new")
+async def create_new_session():
+    session_id = generate_uuid()
+    return {"session_id" : session_id}
+
+@router.get("/bot/{session_id}")
+async def get_session_id(session_id: str, chat_store: ChatStore = Depends(get_chat_store)):    
+    chat_history = chat_store.get_messages(session_id)
+
+    if not chat_history:
+        raise HTTPException(status_code=404, detail="Session not found or empty.")
+    
+    return chat_history
+
+@router.get("/bot")
+async def get_all_session_ids():
+    try:
+        chat_store = ChatStore()
+        all_keys = chat_store.get_keys()
+        print(all_keys)
+        return all_keys
+    except Exception as e:
+    # Log the error and raise HTTPException for FastAPI
+        print(f"An error occurred in update data.: {e}")
+        raise HTTPException(
+            status_code=400, detail="the error when get all session ids"
+        )
+        
+
+@router.post("/bot/{session_id}")
 async def bot_generator_general(user_prompt_request: UserPromptRequest):
 
     if user_prompt_request.streaming:
@@ -22,7 +55,7 @@ async def bot_generator_general(user_prompt_request: UserPromptRequest):
     else:
         response, raw_references, references, metadata, scores = (
             generate_completion_non_streaming(
-                user_prompt_request.prompt, user_prompt_request.streaming
+                user_prompt_request.session_id, user_prompt_request.prompt, user_prompt_request.streaming
             )
         )
 
@@ -35,12 +68,24 @@ async def bot_generator_general(user_prompt_request: UserPromptRequest):
         )
 
 
-@router.post("/bot/{category_id}/{title}")
+@router.post("/bot/{category_id}/{title}") #Ganti router
 async def bot_generator_spesific(
     category_id: int, title: str, user_prompt_request: UserPromptRequest
 ):
     pass
 
+@router.delete("/bot/{session_id}")
+async def delete_bot(session_id: str, chat_store: ChatStore = Depends(get_chat_store)):
+    try:
+        chat_store.delete_messages(session_id)
+        return {"info": f"Delete {session_id} successful"}
+    except Exception as e:
+        # Log the error and raise HTTPException for FastAPI
+        print(f"An error occurred in update data.: {e}")
+        raise HTTPException(
+            status_code=400, detail="the error when deleting message"
+        )
+        
 
 @router.get("/bot/{category_id}/{title}")
 async def get_favourite_data(category_id: int, title: str, human_template):

api/router/reader.py

@@ -0,0 +1,17 @@
+from fastapi import APIRouter, File, UploadFile, HTTPException
+from core.journal_reading.upload import upload_file
+
+router = APIRouter(tags=["Journal Reading"])
+
+
+
+@router.post("/upload")
+async def upload_journal(file: UploadFile = File(...)):
+    try :
+        documents = await upload_file(file)
+        
+        return {"Success"}
+    except Exception as e:
+        raise HTTPException(
+            status_code=400, detail=f"Error processing file: {str(e)}"
+        )

api/router/topic.py

@@ -1,12 +1,18 @@
 from fastapi import Form, APIRouter, File, UploadFile, HTTPException, Request
 from db.repository import get_db_conn
+from db.get_data import GetDatabase
+from db.save_data import InsertDatabase
 from config import MYSQL_CONFIG
 from api.function import data_ingestion, get_data, delete_data, update_data
+from script.vector_db import IndexManager
 from service.dto import MetadataRequest
 
 router = APIRouter(tags=["Topics"])
 
 db_conn = get_db_conn(MYSQL_CONFIG)
+get_database = GetDatabase(db_conn)
+index_manager = IndexManager()
+
 
 @router.post("/topic")
 async def upload_file(
@@ -40,11 +46,24 @@ async def get_metadata():
 
 @router.put("/topic/{id}")
 async def update_metadata(id: int, reference: MetadataRequest):
-    response = await update_data(id, reference, db_conn)
-    return response
+    try :
+        old_reference = await get_database.get_data_by_id(id)
+        index_manager.update_vector_database(old_reference, reference)
+
+        return await update_data(id, reference, db_conn)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail="An error occurred while updating metadata")
 
 
 @router.delete("/topic/{id}")
 async def delete_metadata(id: int):
-    response = await delete_data(id, db_conn)
-    return response
+    try:
+        old_reference = await get_database.get_data_by_id(id)
+        index_manager.delete_vector_database(old_reference)
+
+        return await delete_data(id, db_conn)
+    
+    except Exception as e:
+        print(e)
+        raise HTTPException(status_code=500, detail="An error occurred while delete metadata")
+

app.py

@@ -1,5 +1,5 @@
 from fastapi.applications import FastAPI
-from api.router import health, topic, user, bot, trial, role
+from api.router import health, topic, user, bot, trial, role, reader
 from fastapi.middleware.cors import CORSMiddleware
 from api.events import register_events
 from utils.utils import pipe
@@ -27,6 +27,7 @@ def register_routers(app: FastAPI) -> FastAPI:
     app.include_router(bot.router)
     app.include_router(trial.router)
     app.include_router(role.router)
+    app.include_router(reader.router)
     app.include_router(health.router)
 
     return app

core/chat/chatstore.py

@@ -0,0 +1,93 @@
+import redis
+import os
+import json
+from fastapi import HTTPException
+from uuid import uuid4
+from typing import Optional, List
+from llama_index.storage.chat_store.redis import RedisChatStore
+from llama_index.core.memory import ChatMemoryBuffer
+from service.dto import ChatMessage
+
+
+class ChatStore:
+    def __init__(self):
+        self.redis_client = redis.Redis(
+            host="redis-10365.c244.us-east-1-2.ec2.redns.redis-cloud.com",
+            port=10365,
+            password=os.environ.get("REDIS_PASSWORD"),
+        )
+        
+    def generate_uuid(use_hex=False):
+        if use_hex:
+            return str(uuid4().hex)
+        else:
+            return str(uuid4())
+        
+    def initialize_memory_bot(self, session_id=None):
+        if session_id is None:
+            session_id = self.generate_uuid()
+    # chat_store = SimpleChatStore()
+        chat_store = RedisChatStore(
+            redis_client=self.redis_client
+        )  # Need to be configured
+
+        memory = ChatMemoryBuffer.from_defaults(
+            token_limit=3000, chat_store=chat_store, chat_store_key=session_id
+        )
+
+        return memory
+    
+    def get_messages(self, session_id: str) -> List[dict]:
+        """Get messages for a session_id."""
+        items = self.redis_client.lrange(session_id, 0, -1)
+        if len(items) == 0:
+            return []
+        
+        # Decode and parse each item into a dictionary
+        return [json.loads(m.decode("utf-8")) for m in items]
+
+    def delete_last_message(self, session_id: str) -> Optional[ChatMessage]:
+        """Delete last message for a session_id."""
+        return self.redis_client.rpop(session_id)
+    
+    def delete_messages(self, key: str) -> Optional[List[ChatMessage]]:
+        """Delete messages for a key."""
+        self.redis_client.delete(key)
+        return None
+    
+    def clean_message(self, session_id: str) -> Optional[ChatMessage]:
+        """Delete specific message for a session_id."""
+        current_list = self.redis_client.lrange(session_id, 0, -1)
+        
+        indices_to_delete = []
+        for index, item in enumerate(current_list):
+            data = json.loads(item)  # Parse JSON string to dict
+
+            # Logic to determine if item should be removed
+            if (data.get("role") == "assistant" and data.get("content") is None) or (data.get("role") == "tool"):
+                indices_to_delete.append(index)
+
+        # Remove elements by their indices in reverse order
+        for index in reversed(indices_to_delete):
+            self.redis_client.lrem(session_id, 1, current_list[index])  # Remove the element from the list in Redis
+
+    def get_keys(self) -> List[str]:
+        """Get all keys."""
+        try :
+            print(self.redis_client.keys("*"))
+            return [key.decode("utf-8") for key in self.redis_client.keys("*")]
+        
+        except Exception as e:
+            # Log the error and raise HTTPException for FastAPI
+            print(f"An error occurred in update data.: {e}")
+            raise HTTPException(
+                status_code=400, detail="the error when get keys"
+            )
+
+    def add_message(self, session_id: str, message: ChatMessage) -> None:
+        """Add a message for a session_id."""
+        item = json.dumps(self._message_to_dict(message))
+        self.redis_client.rpush(session_id, item)
+
+    def _message_to_dict(self, message: ChatMessage) -> dict:
+        return message.model_dump()

core/chat/engine.py

@@ -1,4 +1,4 @@
-from typing import Any
+from typing import Optional, List
 from llama_index.core.vector_stores import (
     MetadataFilter,
     MetadataFilters,
@@ -10,15 +10,17 @@ from llama_index.core.tools import QueryEngineTool, ToolMetadata
 from llama_index.agent.openai import OpenAIAgent
 from llama_index.llms.openai import OpenAI
 from llama_index.storage.chat_store.redis import RedisChatStore
-from llama_index.core.storage.chat_store import SimpleChatStore
 from llama_index.core.memory import ChatMemoryBuffer
 from llama_index.core.query_engine import CitationQueryEngine
 from llama_index.core import Settings
+from core.chat.chatstore import ChatStore
 
+from service.dto import ChatMessage
 from config import GPTBOT_CONFIG
 from core.prompt import SYSTEM_BOT_TEMPLATE
 import redis
 import os
+import json
 
 
 class Engine:
@@ -29,22 +31,10 @@ class Engine:
             max_tokens=GPTBOT_CONFIG.max_tokens,
             api_key=GPTBOT_CONFIG.api_key,
         )
-        
-        Settings.llm = self.llm
 
-    def initialize_memory_bot(self, user_id = "1"):
-        redis_client = redis.Redis(
-            host="redis-10365.c244.us-east-1-2.ec2.redns.redis-cloud.com",
-            port=10365,
-            password=os.environ.get("REDIS_PASSWORD"),
-        )
-        # chat_store = SimpleChatStore()
-        chat_store = RedisChatStore(redis_client=redis_client, ttl=3600)  # Need to be configured
-        memory = ChatMemoryBuffer.from_defaults(
-            token_limit=3000, chat_store=chat_store, chat_store_key=user_id
-        )
+        self.chat_store = ChatStore()
 
-        return memory
+        Settings.llm = self.llm
 
     def _build_description_bot(self, title, category):
         try:
@@ -56,22 +46,6 @@ class Engine:
         except Exception as e:
             return f"Error generating description: {str(e)}"
 
-    def index_to_query_engine(self, title, category, index):
-        filters = MetadataFilters(
-            filters=[
-                MetadataFilter(key="title", value=title),
-                MetadataFilter(key="category", value=category),
-            ],
-            condition=FilterCondition.AND,
-        )
-        
-        # Create the QueryEngineTool with the index and filters 
-        kwargs = {"similarity_top_k": 5, "filters": filters}
-        
-        query_engine = index.as_query_engine(**kwargs)
-
-        return query_engine
-    
     def get_citation_engine(self, title, category, index):
         filters = MetadataFilters(
             filters=[
@@ -80,39 +54,33 @@ class Engine:
             ],
             condition=FilterCondition.AND,
         )
-        
-        # Create the QueryEngineTool with the index and filters 
+
+        # Create the QueryEngineTool with the index and filters
         kwargs = {"similarity_top_k": 5, "filters": filters}
-        
+
         retriever = index.as_retriever(**kwargs)
-        
+
         citation_engine = CitationQueryEngine(retriever=retriever)
 
         return citation_engine
-    
-
-    def get_chat_engine(self, index, title=None, category=None, type="general"):
-        # Define the metadata for the QueryEngineTool
 
+    def get_chat_engine(
+        self, session_id, index, title=None, category=None, type="general"
+    ):
         # Create the QueryEngineTool based on the type
         if type == "general":
             # query_engine = index.as_query_engine(similarity_top_k=3)
             citation_engine = CitationQueryEngine.from_args(index, similarity_top_k=5)
             description = "A book containing information about medicine"
         else:
-            query_engine = self.index_to_query_engine(title, category, index)
             citation_engine = self.get_citation_engine(title, category, index)
             description = self._build_description_bot()
 
-        metadata = ToolMetadata(
-            name="bot-belajar",
-            description=description
-        )
+        metadata = ToolMetadata(name="bot-belajar", description=description)
         print(metadata)
 
         vector_query_engine = QueryEngineTool(
-            query_engine=citation_engine,
-            metadata=metadata
+            query_engine=citation_engine, metadata=metadata
         )
         print(vector_query_engine)
 
@@ -120,11 +88,9 @@ class Engine:
         chat_engine = OpenAIAgent.from_tools(
             tools=[vector_query_engine],
             llm=self.llm,
-            memory=self.initialize_memory_bot(),
+            memory=self.chat_store.initialize_memory_bot(session_id),
+            # memory = self.initialize_memory_bot(session_id),
             system_prompt=SYSTEM_BOT_TEMPLATE,
         )
 
-        return chat_engine
-
-    def get_chat_history(self):
-        pass
+        return chat_engine

core/chat/messaging.py

@@ -20,7 +20,7 @@ from core.chat import schema
 from db.db import MessageSubProcessSourceEnum
 from core.chat.schema import SubProcessMetadataKeysEnum, SubProcessMetadataMap
 from core.chat.engine import Engine
-from script.build_vector import IndexManager
+from script.vector_db import IndexManager
 from service.dto import UserPromptRequest
 
 logger = logging.getLogger(__name__)

core/prompt.py

@@ -1,5 +1,5 @@
 SYSTEM_BOT_TEMPLATE = """
-Kamu adalah Medbot, seorang ahli dalam bidang kedokteran. Tugasmu adalah memberikan jawaban yang informatif dan akurat berdasarkan tools yang tersedia dan jangan menghapus referensi atau angka dalam kurung siku, contoh [1], [2] dan sebagainya.
+Kamu adalah Medbot yang gunakan tool kamu untuk menjawab pertanyaan tentang kedokteran. Tugasmu adalah memberikan jawaban yang informatif dan akurat berdasarkan tools yang tersediaserta selalu cantumkan kutipan dari teks yang anda kutip. Jika tidak ada jawaban melalui alat yang digunakan, carilah informasi lebih lanjut dengan menggunakan alat. Jika setelah itu tidak ada informasi yang ditemukan, katakan bahwa kamu tidak mengetahuinya.
 
 **Instruksi**:
 

db/get_data.py

@@ -6,9 +6,6 @@ logging.basicConfig(level=logging.INFO)
 
 
 class GetDatabase(Repository):
-    def __init__(self, db_conn):
-        super().__init__(db_conn)
-
     async def execute_query(self, query, params=None, fetch_one=False):
         """
 
@@ -54,3 +51,17 @@ class GetDatabase(Repository):
         """
         results = await self.execute_query(query)
         return results
+    
+    async def get_data_by_id(self, id):
+        query = f"""
+        SELECT * FROM Metadata WHERE id = :id
+        """
+        
+        param = {"id" : id}
+        try:
+            results = await self.execute_query(query, param)
+            print('Query successful, results: %s', results)
+            return results[0] if results else None
+        except Exception as e:
+            print('Error fetching data by ID %s: %s', id, e)
+            return None

script/get_metadata.py

@@ -3,32 +3,33 @@
 
 class Metadata:
     def __init__(self, reference):
-        self.title = reference["title"]
-        self.author = reference["author"]
-        self.category = reference["category"]
-        self.year = reference["year"]
-        self.publisher = reference["publisher"]
+        self.reference = reference
 
     def add_metadata(self, documents, metadata):
-        """Add metadata to each item (document or node)."""
-        for document in documents:
+        """Add metadata to each document and include page number."""
+        for page_number, document in enumerate(documents, start=1):
+            # Ensure the document has a metadata attribute
             if not hasattr(document, "metadata") or document.metadata is None:
                 document.metadata = {}
+            
+            # Update metadata with page number
+            document.metadata["page"] = page_number
             document.metadata.update(metadata)
-            print("metadata is added")
-            # self.logger.log_action(f"Metadata added to document {item.id_}", action_type="METADATA")
-
+            
+            print(f"Metadata added to page {page_number}")
+            # self.logger.log_action(f"Metadata added to document {document.id_}", action_type="METADATA")
+        
         return documents
 
     def _generate_metadata(self):
         """Generate metadata and return it."""
         metadata = {
-            "title": self.title,
-            "author": self.author,
-            "category": self.category,
-            "year": self.year,
-            "publisher": self.publisher,
-            "reference": f"{self.author}. ({self.year}). *{self.title}*. {self.publisher}.",  # APA style reference
+            "title": self.reference["title"],
+            "author": self.reference["author"],
+            "category": self.reference["category"],
+            "year": self.reference["year"],
+            "publisher": self.reference["publisher"],
+            "reference": f"{self.reference['author']}. ({self.reference['year']}). *{self.reference['title']}*. {self.reference['publisher']}."  # APA style reference
         }
         print("metadata is generated")
         return metadata

script/vector_db.py

@@ -0,0 +1,157 @@
+from llama_index.core import VectorStoreIndex
+from llama_index.core import StorageContext
+from pinecone import Pinecone, ServerlessSpec
+from llama_index.llms.openai import OpenAI
+from llama_index.vector_stores.pinecone import PineconeVectorStore
+from fastapi import HTTPException, status
+from config import PINECONE_CONFIG
+from math import ceil
+import numpy as np
+import os
+import json
+
+
+class IndexManager:
+    def __init__(self, index_name: str = "summarizer-semantic-index"):
+        self.vector_index = None
+        self.index_name = index_name
+        self.client = self._get_pinecone_client()
+        self.pinecone_index = self._create_pinecone_index()
+
+    def _get_pinecone_client(self):
+        """Initialize and return the Pinecone client."""
+        # api_key = os.getenv("PINECONE_API_KEY")
+        api_key = PINECONE_CONFIG.PINECONE_API_KEY
+        if not api_key:
+            raise ValueError(
+                "Pinecone API key is missing. Please set it in environment variables."
+            )
+        return Pinecone(api_key=api_key)
+
+    def _create_pinecone_index(self):
+        """Create Pinecone index if it doesn't already exist."""
+        if self.index_name not in self.client.list_indexes().names():
+            self.client.create_index(
+                name=self.index_name,
+                dimension=1536,
+                metric="cosine",
+                spec=ServerlessSpec(cloud="aws", region="us-east-1"),
+            )
+        return self.client.Index(self.index_name)
+
+    def _initialize_vector_store(self) -> StorageContext:
+        """Initialize and return the vector store with the Pinecone index."""
+        vector_store = PineconeVectorStore(pinecone_index=self.pinecone_index)
+        return StorageContext.from_defaults(vector_store=vector_store)
+
+
+    def build_indexes(self, nodes):
+        """Build vector and tree indexes from nodes."""
+        try:
+            storage_context = self._initialize_vector_store()
+            self.vector_index = VectorStoreIndex(nodes, storage_context=storage_context)
+            self.vector_index.set_index_id("vector")
+
+            print(f"Vector Index ID: {self.vector_index.index_id}")
+            print("Vector Index created successfully.")
+
+            return json.dumps({"status": "success", "message": "Vector Index loaded successfully."})
+        
+        except HTTPException as http_exc:
+            raise http_exc  # Re-raise HTTPExceptions to ensure FastAPI handles them
+        except Exception as e:
+            raise HTTPException(
+                status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+                detail=f"Error loading existing indexes: {str(e)}"
+            )
+            
+    def get_ids_from_query(self, input_vector, title):
+        print("Searching Pinecone...")
+        print(title)
+
+        new_ids = set()  # Initialize new_ids outside the loop
+
+        while True:
+            results = self.pinecone_index.query(
+                vector=input_vector,
+                top_k=10000,
+                filter={
+                    "title": {"$eq": f"{title}"},
+                },
+            )
+            
+            ids = set()
+            for result in results['matches']:
+                ids.add(result['id'])
+            # Check if there's any overlap between ids and new_ids
+            if ids.issubset(new_ids):
+                break
+            else:
+                new_ids.update(ids)  # Add all new ids to new_ids
+
+        return new_ids
+
+           
+    def get_all_ids_from_index(self, title):
+        num_dimensions = 1536
+
+        num_vectors = self.pinecone_index.describe_index_stats(
+        )["total_vector_count"]
+        
+        print("Length of ids list is shorter than the number of total vectors...")
+        input_vector = np.random.rand(num_dimensions).tolist()
+        print("creating random vector...")
+        ids = self.get_ids_from_query(input_vector, title)
+        print("getting ids from a vector query...")
+
+        print("updating ids set...")
+        print(f"Collected {len(ids)} ids out of {num_vectors}.")
+
+        return ids
+    
+    def delete_vector_database(self, old_reference):
+        try :
+            batch_size = 1000
+            all_ids = self.get_all_ids_from_index(old_reference['title'])
+            all_ids = list(all_ids)
+            
+            # Split ids into chunks of batch_size
+            num_batches = ceil(len(all_ids) / batch_size)
+            
+            for i in range(num_batches):
+                # Fetch a batch of IDs
+                batch_ids = all_ids[i * batch_size: (i + 1) * batch_size]
+                self.pinecone_index.delete(ids=batch_ids)
+                print(f"delete from id {i * batch_size} to {(i + 1) * batch_size} successful")
+        except Exception as e:
+            print(e)
+            raise HTTPException(status_code=500, detail="An error occurred while delete metadata")    
+
+    def update_vector_database(self, old_reference, new_reference):
+        
+        reference = new_reference.model_dump()
+        
+        all_ids = self.get_all_ids_from_index(old_reference['title'])
+        all_ids = list(all_ids)
+        
+        for id in all_ids:
+            self.pinecone_index.update(
+                id=id,
+                set_metadata=reference
+            )
+            
+
+    def load_existing_indexes(self):
+        """Load existing indexes from Pinecone."""
+        try:
+            client = self._get_pinecone_client()
+            pinecone_index = client.Index(self.index_name)
+            
+            vector_store = PineconeVectorStore(pinecone_index=pinecone_index)
+            retriever = VectorStoreIndex.from_vector_store(vector_store)
+
+            print("Existing Vector Index loaded successfully.")
+            return retriever
+        except Exception as e:
+            print(f"Error loading existing indexes: {e}")
+            raise

service/aws_loader.py

@@ -18,22 +18,55 @@ class Loader:
             region_name="us-west-2",
         )
 
+    # def upload_to_s3(self, file, object_name, folder_name="summarizer"):
+    #     try:
+    #         # If folder_name is provided, prepend it to the object_name
+    #         if folder_name:
+    #             object_name = f"{folder_name}/{object_name}"
+
+    #         # Create an in-memory file-like object
+    #         with BytesIO() as file_stream:
+    #             # Write the contents of the uploaded file to the stream
+    #             file_stream.write(file.file.read())
+    #             file_stream.seek(0)  # Move to the beginning of the stream
+
+    #             # Upload file to S3
+    #             self.s3_client.upload_fileobj(file_stream, self.bucket_name, object_name)
+
+    #         print(f"File '{object_name}' successfully uploaded to bucket '{self.bucket_name}'.")
+    #     except Exception as e:
+    #         raise HTTPException(status_code=400, detail=f"Error uploading to AWS: {e}")
+    
     def upload_to_s3(self, file, object_name, folder_name="summarizer"):
         try:
             # If folder_name is provided, prepend it to the object_name
             if folder_name:
                 object_name = f"{folder_name}/{object_name}"
 
-            # Create an in-memory file-like object
-            with BytesIO() as file_stream:
-                # Write the contents of the uploaded file to the stream
-                file_stream.write(file.file.read())
-                file_stream.seek(0)  # Move to the beginning of the stream
+            # Open the PDF with PyMuPDF (fitz)
+            pdf_document = fitz.open(stream=file.file.read(), filetype="pdf")
+
+            # Loop through each page of the PDF
+            for page_num in range(pdf_document.page_count):
+
+                # Convert the page to bytes (as a separate PDF)
+                page_stream = BytesIO()
+                single_page_pdf = fitz.open()  # Create a new PDF
+                single_page_pdf.insert_pdf(pdf_document, from_page=page_num, to_page=page_num)
+                single_page_pdf.save(page_stream)
+                single_page_pdf.close()
+
+                # Reset the stream position to the start
+                page_stream.seek(0)
+
+                # Define the object name for each page (e.g., 'summarizer/object_name/page_1.pdf')
+                page_object_name = f"{object_name}/{page_num + 1}.pdf"
+
+                # Upload each page to S3
+                self.s3_client.upload_fileobj(page_stream, self.bucket_name, page_object_name)
 
-                # Upload file to S3
-                self.s3_client.upload_fileobj(file_stream, self.bucket_name, object_name)
+                print(f"Page {page_num + 1} of '{object_name}' successfully uploaded as '{page_object_name}' to bucket '{self.bucket_name}'.")
 
-            print(f"File '{object_name}' successfully uploaded to bucket '{self.bucket_name}'.")
         except Exception as e:
             raise HTTPException(status_code=400, detail=f"Error uploading to AWS: {e}")
 

service/dto.py

@@ -1,5 +1,6 @@
 from pydantic import BaseModel, Field
-from typing import List, Optional, Dict
+from typing import List, Optional, Dict, Any
+from llama_index.core.llms import MessageRole
 
 class MetadataRequest(BaseModel):
     title: str
@@ -12,6 +13,7 @@ class DeleteById(BaseModel):
     id : str
 
 class UserPromptRequest(BaseModel): 
+    session_id : str
     prompt : str
     streaming : bool
     
@@ -33,4 +35,14 @@ class BotResponseStreaming(BaseModel):
     
 class TestStreaming(BaseModel):
     role : str = "assistant"
-    content : str
+    content : str
+    
+class ChatMessage(BaseModel):
+    """Chat message."""
+
+    role: MessageRole = MessageRole.ASSISTANT
+    content: Optional[Any] = ""
+    metadata: List
+
+    def __str__(self) -> str:
+        return f"{self.role.value}: {self.content}"

utils/utils.py

@@ -1,3 +1,5 @@
+from uuid import uuid4
+
 def pipe(data, *funcs):
     """ Pipe a value through a sequence of functions
 
@@ -19,4 +21,10 @@ def pipe(data, *funcs):
     """
     for func in funcs:
         data = func(data)
-    return data
+    return data
+
+def generate_uuid(use_hex=False):
+    if use_hex:
+        return str(uuid4().hex)
+    else:
+        return str(uuid4())