Update to Qdrant db

.gitignore

@@ -2,4 +2,7 @@
 Api/venv
 
 # Other versions 
-Api/out1
+Api/out1
+
+# env 
+.env

Api/app/db_local_storage/documents_db.py

@@ -1,2 +0,0 @@
-documents_db = []
-documents_text = []

Api/app/db_local_storage/files_db.py

@@ -1,4 +0,0 @@
-FILES_NAMES_DATABASE = {}
-FILES_DIRECTORY = "src/db_local_storage/files"
-TEXT_FILES_DIRECTORY = "src/db_local_storage/text_files"
-VECTOR_FILES_DIRECTORY = "src/db_local_storage/vector_files/vec_db.json"

Api/app/db_local_storage/vector_files_db.py

@@ -1 +0,0 @@
-vector_files_db = {}

Api/app/infrastructure/models/my_models.py

@@ -0,0 +1,28 @@
+from pydantic import BaseModel
+from typing import Dict, List, Optional
+
+
+class ModelResponse(BaseModel, extra="forbid"):
+    text: str
+    isSender: bool
+    message: Optional[str] = None
+
+
+class EmbeddingCreation(BaseModel, extra="forbid"):
+    success: bool
+    message: Optional[str] = None
+
+
+class HybridSearchResponse(BaseModel):
+    success: bool
+    data: Optional[List[Dict]] = None
+    message: Optional[str] = None
+
+
+class Chunk(BaseModel):
+    index: int
+    text: str
+
+
+class ChunksResponse(BaseModel):
+    data: Dict[str, List[Chunk]]

Api/app/infrastructure/repository/document_handeler_repository.py

@@ -0,0 +1,48 @@
+from typing import Any, List, Tuple
+from app.qdrant import QdrantConnectionDb
+from qdrant_client import models
+
+
+class DocumentHandelerRepository:
+    def __init__(self, qdrant_connection_db: QdrantConnectionDb):
+        self.client = qdrant_connection_db.get_client()
+        self.collection_name = qdrant_connection_db.get_collection_name()
+
+    def find_points_by_document_name(self, document_name) -> List[int]:
+        result = self.client.scroll(
+            collection_name=self.collection_name,
+            scroll_filter=models.Filter(
+                must=[
+                    models.FieldCondition(
+                        key="document_id", match=models.MatchValue(value=document_name)
+                    )
+                ]
+            ),
+        )
+
+        if result[0]:
+            return [point.id for point in result[0]]
+
+        return
+
+    def delete_document_by_id(self, documents_id: List[int]) -> None:
+        return self.client.delete(
+            collection_name=self.collection_name,
+            points_selector=models.PointIdsList(points=documents_id),
+        )
+
+    def insert_points(self, points: List[models.PointStruct]) -> models.UpdateResult:
+        return self.client.upsert(
+            collection_name=self.collection_name,
+            wait=True,
+            points=points,
+        )
+
+    def get_all_documents(
+        self,
+    ) -> Tuple[List[models.Record], Any]:  # models.ScrollResult
+        return self.client.scroll(
+            collection_name=self.collection_name,
+            with_payload=True,
+            with_vectors=False,
+        )

Api/app/infrastructure/repository/query_search_repository.py

@@ -0,0 +1,20 @@
+from app.qdrant import QdrantConnectionDb
+from qdrant_client.models import QueryResponse, Prefetch, NamedVector
+
+
+class QuerySearchRepository:
+    def __init__(self, qdrant_connection_db: QdrantConnectionDb):
+        self.client = qdrant_connection_db.get_client()
+        self.collection_name = qdrant_connection_db.get_collection_name()
+
+    def find_text_by_hybrid_search(
+        self, prefetch_context: Prefetch, dense_vector: NamedVector
+    ) -> QueryResponse:
+        return self.client.query_points(
+            collection_name=self.collection_name,
+            prefetch=prefetch_context,
+            query=dense_vector.vector,
+            using="text-dense",
+            with_payload=True,
+            limit=10,
+        )

Api/app/main.py

@@ -1,22 +1,45 @@
 import logging
+from contextlib import asynccontextmanager
 
 import uvicorn
 from fastapi import APIRouter, FastAPI
 from fastapi.middleware.cors import CORSMiddleware
 from fastapi.responses import FileResponse
 from fastapi.staticfiles import StaticFiles
+from transformers import AutoModel, AutoModelForMaskedLM, AutoTokenizer, pipeline
+
+from app.modules.clearVariables.routes.clearVariables_route import \
+    router as clear_variables_routes
+from app.modules.documentHandeler.routes.document_handeler_route import \
+    router as upload_file_routes
+from app.modules.querySearch.routes.querySearch_route import \
+    router as query_search_routes
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    dense_model_name = "sentence-transformers/all-MiniLM-L6-v2"
+    sparse_model_name = "prithivida/Splade_PP_en_v1"
+    qa_model_name = "deepset/roberta-base-squad2"
+
+    dense_tokenizer = AutoTokenizer.from_pretrained(dense_model_name)
+    dense_model = AutoModel.from_pretrained(dense_model_name)
+
+    sparse_tokenizer = AutoTokenizer.from_pretrained(sparse_model_name)
+    sparse_model = AutoModelForMaskedLM.from_pretrained(sparse_model_name)
+
+    qa_pipeline = pipeline("question-answering", model=qa_model_name)
+
+    yield {
+        "dense_tokenizer": dense_tokenizer,
+        "dense_model": dense_model,
+        "sparse_tokenizer": sparse_tokenizer,
+        "sparse_model": sparse_model,
+        "qa_pipeline": qa_pipeline,
+    }
 
-from app.modules.querySearch.routes.querySearch_route import (
-    router as query_search_routes,
-)
-from app.modules.uploadDocument.routes.uploadDocument_route import (
-    router as upload_file_routes,
-)
-from app.modules.clearVariables.routes.clearVariables_route import (
-    router as clear_variables_routes,
-)
 
-app = FastAPI()
+app = FastAPI(lifespan=lifespan)
 
 origins = [
     "http://localhost:8000",
@@ -36,7 +59,7 @@ app.add_middleware(
 )
 
 app_router = APIRouter(prefix="/api")
-app_router.include_router(upload_file_routes, prefix="/upload", tags=["upload"])
+app_router.include_router(upload_file_routes, prefix="/document", tags=["document"])
 app_router.include_router(query_search_routes, prefix="/query", tags=["query"])
 app_router.include_router(clear_variables_routes, prefix="/clear", tags=["clear"])
 

Api/app/modules/clearVariables/routes/clearVariables_route.py

@@ -1,20 +1,12 @@
-from app.db_local_storage.documents_db import documents_db, documents_text
 from app.db_local_storage.in_memory_db import query_response_storage
-from app.db_local_storage.vector_files_db import vector_files_db
-from app.modules.uploadDocument.controllers.file_upload_controller import \
-    FileUploadController
-from fastapi import APIRouter, File, HTTPException, UploadFile
+from fastapi import APIRouter
 from fastapi.responses import JSONResponse
 
 router = APIRouter()
-fileUploadController = FileUploadController()
+
 
 @router.delete("/clear_variables/")
 async def clear_variables():
-    vector_files_db.clear()
-    documents_db.clear()
     query_response_storage.clear()
-    documents_db.clear()
-    documents_text.clear()
-    
-    return JSONResponse(status_code=200, content={"message": "All variables cleared"})
+
+    return JSONResponse(status_code=200, content={"message": "All variables cleared"})

Api/app/modules/denseEmbeddings/denseEmbeddings.py

@@ -0,0 +1,69 @@
+import torch
+from qdrant_client import models
+from qdrant_client.models import NamedVector
+from transformers import AutoModel, AutoModelForMaskedLM, AutoTokenizer
+
+
+class DenseEmbeddings:
+
+    def __init__(
+        self,
+        dense_model: AutoModel,
+        dense_tokenizer: AutoTokenizer,
+        sparse_model: AutoModelForMaskedLM,
+        sparse_tokenizer: AutoTokenizer,
+    ):
+
+        self.dense_model = dense_model
+        self.dense_tokenizer = dense_tokenizer
+        self.sparse_model = sparse_model
+        self.sparse_tokenizer = sparse_tokenizer
+
+    def get_dense_vector(self, text: str) -> NamedVector:
+        """
+        Get dense vector from the dense model
+
+        :param text: str
+        :return: NamedVector
+        """
+        inputs = self.dense_tokenizer(
+            text, return_tensors="pt", padding=True, truncation=True
+        )
+        with torch.no_grad():
+            outputs = self.dense_model(**inputs)
+
+        dense_vector = NamedVector(
+            name="text-dense",
+            vector=torch.mean(outputs.last_hidden_state, dim=1).squeeze().numpy(),
+        )
+        return dense_vector
+
+    def get_sparse_vector(self, text: str) -> models.SparseVector:
+        """
+        Get sparse vector from the sparse model
+
+        :param text: str
+        :return: SparseVector
+        """
+
+        inputs = self.sparse_tokenizer(
+            text, return_tensors="pt", padding=True, truncation=True
+        )
+        with torch.no_grad():
+            outputs = self.sparse_model(**inputs)
+
+        token_scores = outputs.logits.squeeze().max(dim=0)[0]
+        token_ids = inputs["input_ids"].squeeze()
+
+        sparse_vector = {
+            int(token_id): float(score)
+            for token_id, score in zip(token_ids, token_scores)
+            if score > -5.0
+        }
+
+        sparse_vector = models.SparseVector(
+            indices=list(sparse_vector.keys()),
+            values=list(sparse_vector.values()),
+        )
+
+        return sparse_vector

Api/app/modules/documentHandeler/controllers/document_handeler_controller.py

@@ -0,0 +1,66 @@
+from fastapi import HTTPException, UploadFile
+from fastapi.responses import JSONResponse
+
+from app.infrastructure.repository.document_handeler_repository import (
+    DocumentHandelerRepository,
+)
+from app.modules.documentHandeler.features.createEmbeddings_feature import (
+    CreateEmbeddingsFeature,
+)
+from app.modules.documentHandeler.features.deleteDocument_feature import (
+    DeleteDocumentFeature,
+)
+from app.modules.documentHandeler.features.extractText_feature import ExtractTextFeature
+from app.modules.documentHandeler.features.getAllChunkedText_feature import (
+    GetAllChunkedTextFeature,
+)
+
+
+class DocumentHandelerController:
+
+    def __init__(
+        self,
+        delete_document_feature: DeleteDocumentFeature,
+        create_embeddings_feature: CreateEmbeddingsFeature,
+        get_all_chunked_text_feature: GetAllChunkedTextFeature,
+    ):
+        self.create_embeddings_feature = create_embeddings_feature
+        self.delete_document_feature = delete_document_feature
+        self.get_all_chunked_text_feature = get_all_chunked_text_feature
+
+    async def handle_file_upload(self, file: UploadFile) -> JSONResponse:
+        try:
+
+            text_file = await ExtractTextFeature.extract_text_from_pdf(file)
+            result = await self.create_embeddings_feature.create_embeddings(
+                text_file, file.filename
+            )
+
+            return JSONResponse(status_code=200, content=result.model_dump())
+
+        except Exception as e:
+            raise HTTPException(status_code=500, detail="Probelm on controller")
+
+    async def delete_document(self, text: str) -> JSONResponse:
+        try:
+            result = await self.delete_document_feature.delete_document_by_filename(
+                text
+            )
+            if result:
+                return JSONResponse(
+                    status_code=200, content={"message": "Document deleted"}
+                )
+            return JSONResponse(
+                status_code=404, content={"message": "Document not found"}
+            )
+            
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))
+
+    async def get_all_chunks(self) -> JSONResponse:
+        try:
+            result = await self.get_all_chunked_text_feature.get_all_chunked_text()
+            return JSONResponse(status_code=200, content=result.model_dump())
+
+        except Exception as e:
+            raise HTTPException(status_code=500, detail=str(e))

Api/app/modules/documentHandeler/dependencies/dependencies.py

@@ -0,0 +1,112 @@
+from fastapi import Depends, Request
+from transformers import AutoModel, AutoModelForMaskedLM, AutoTokenizer
+
+from app.infrastructure.repository.document_handeler_repository import (
+    DocumentHandelerRepository,
+)
+from app.modules.denseEmbeddings.denseEmbeddings import DenseEmbeddings
+from app.modules.documentHandeler.controllers.document_handeler_controller import (
+    DocumentHandelerController,
+)
+from app.modules.documentHandeler.features.createEmbeddings_feature import (
+    CreateEmbeddingsFeature,
+)
+from app.modules.documentHandeler.features.deleteDocument_feature import (
+    DeleteDocumentFeature,
+)
+from app.modules.documentHandeler.features.getAllChunkedText_feature import (
+    GetAllChunkedTextFeature,
+)
+from app.qdrant import QdrantConnectionDb
+
+
+def get_qdrant_connection_db() -> QdrantConnectionDb:
+    return QdrantConnectionDb()
+
+
+def get_document_handeler_repository(
+    qdrant_connection_db: QdrantConnectionDb = Depends(get_qdrant_connection_db),
+):
+    return DocumentHandelerRepository(qdrant_connection_db)
+
+
+def get_dense_model(request: Request) -> AutoModel:
+    return request.scope["state"]["dense_model"]
+
+
+def get_sparse_model(request: Request) -> AutoModelForMaskedLM:
+    return request.scope["state"]["sparse_model"]
+
+
+def get_dense_tokenizer(request: Request) -> AutoTokenizer:
+    return request.scope["state"]["dense_tokenizer"]
+
+
+def get_sparse_tokenizer(request: Request) -> AutoTokenizer:
+    return request.scope["state"]["sparse_tokenizer"]
+
+
+def get_dense_embeddings(
+    dense_model: AutoModel = Depends(get_dense_model),
+    dense_tokenizer: AutoTokenizer = Depends(get_dense_tokenizer),
+    sparse_model: AutoModelForMaskedLM = Depends(get_sparse_model),
+    sparse_tokenizer: AutoTokenizer = Depends(get_sparse_tokenizer),
+):
+    return DenseEmbeddings(
+        dense_model=dense_model,
+        dense_tokenizer=dense_tokenizer,
+        sparse_model=sparse_model,
+        sparse_tokenizer=sparse_tokenizer,
+    )
+
+
+def get_all_chunked_text_feature(
+    document_handeler_repository: DocumentHandelerRepository = Depends(
+        get_document_handeler_repository
+    ),
+):
+    return GetAllChunkedTextFeature(document_handeler_repository)
+
+
+def get_create_embeddings_feature(
+    dense_embeddings: DenseEmbeddings = Depends(get_dense_embeddings),
+    document_handeler_repository: DocumentHandelerRepository = Depends(
+        get_document_handeler_repository
+    ),
+):
+    return CreateEmbeddingsFeature(dense_embeddings, document_handeler_repository)
+
+
+def get_delete_document_feature(
+    document_handeler_repository: DocumentHandelerRepository = Depends(
+        get_document_handeler_repository
+    ),
+):
+    return DeleteDocumentFeature(document_handeler_repository)
+
+
+def get_document_handeler_controller(
+    delete_document_feature: DeleteDocumentFeature = Depends(
+        get_delete_document_feature
+    ),
+    create_embeddings_feature: CreateEmbeddingsFeature = Depends(
+        get_create_embeddings_feature
+    ),
+    get_all_chunked_text_feature: GetAllChunkedTextFeature = Depends(
+        get_all_chunked_text_feature
+    ),
+):
+    return DocumentHandelerController(
+        delete_document_feature=delete_document_feature,
+        create_embeddings_feature=create_embeddings_feature,
+        get_all_chunked_text_feature=get_all_chunked_text_feature,
+    )
+
+
+def get_create_embeddings_feature(
+    dense_embeddings: DenseEmbeddings = Depends(get_dense_embeddings),
+    document_handeler_repository: DocumentHandelerRepository = Depends(
+        get_document_handeler_repository
+    ),
+):
+    return CreateEmbeddingsFeature(dense_embeddings, document_handeler_repository)

Api/app/modules/documentHandeler/features/createEmbeddings_feature.py

@@ -0,0 +1,69 @@
+from typing import List
+import uuid
+
+from qdrant_client.models import PointStruct
+
+from app.infrastructure.models.my_models import EmbeddingCreation
+from app.infrastructure.repository.document_handeler_repository import (
+    DocumentHandelerRepository,
+)
+from app.modules.denseEmbeddings.denseEmbeddings import DenseEmbeddings
+
+
+class CreateEmbeddingsFeature:
+
+    def __init__(
+        self,
+        dense_embeddings: DenseEmbeddings,
+        document_handeler_repository: DocumentHandelerRepository,
+    ):
+        self.dense_embeddings = dense_embeddings
+        self.document_handeler_repository = document_handeler_repository
+
+    def chunk_text(self, text: str, chunk_size: int = 512) -> List[str]:
+        """
+        Chunk text into smaller pieces
+
+        :param text: str
+        :param chunk_size: int
+        :return: List[str]
+        """
+        chunks = [text[i : i + chunk_size] for i in range(0, len(text), chunk_size)]
+        return chunks
+
+    async def create_embeddings(self, text: str, filename: str) -> EmbeddingCreation:
+        """
+        Create embeddings for the text
+
+        :param text: str
+        :param filename: str
+        :return: EmbeddingCreation
+        """
+
+        chunks = self.chunk_text(text)
+
+        document_id = filename.split(".")[0]
+
+        points = [
+            PointStruct(
+                id=str(uuid.uuid4()),
+                vector={
+                    "text-dense": self.dense_embeddings.get_dense_vector(chunk).vector,
+                    "text-sparse": self.dense_embeddings.get_sparse_vector(chunk),
+                },
+                payload={
+                    "document_id": document_id,
+                    "chunk_index": i,
+                    "filename": filename,
+                    "chunk-text": chunk,
+                },
+            )
+            for i, chunk in enumerate(chunks)
+        ]
+
+        result = self.document_handeler_repository.insert_points(points)
+        if result.status:
+            return EmbeddingCreation(
+                success=True, message="Embeddings created successfully"
+            )
+        return EmbeddingCreation(success=False, message="Embeddings creation failed")

Api/app/modules/documentHandeler/features/deleteDocument_feature.py

@@ -0,0 +1,18 @@
+from app.infrastructure.repository.document_handeler_repository import (
+    DocumentHandelerRepository,
+)
+
+
+class DeleteDocumentFeature:
+
+    def __init__(self, update_document_repository: DocumentHandelerRepository):
+        self.update_document_repository = update_document_repository
+
+    async def delete_document_by_filename(self, document_name: str) -> bool:
+        document = self.update_document_repository.find_points_by_document_name(
+            document_name
+        )
+        if document is None:
+            return False
+        self.update_document_repository.delete_document_by_id(document)
+        return True

Api/app/modules/documentHandeler/features/extractText_feature.py

@@ -0,0 +1,19 @@
+import io
+import os
+
+from fastapi import UploadFile
+import pdfplumber
+
+
+class ExtractTextFeature:
+
+    @staticmethod
+    async def extract_text_from_pdf(file: UploadFile) -> str:
+
+        content = await file.read()
+        with pdfplumber.open(io.BytesIO(content)) as pdf:
+            text = ""
+            for page in pdf.pages:
+                text += page.extract_text()
+
+        return text

Api/app/modules/documentHandeler/features/getAllChunkedText_feature.py

@@ -0,0 +1,32 @@
+from app.infrastructure.models.my_models import ChunksResponse
+from app.infrastructure.repository.document_handeler_repository import (
+    DocumentHandelerRepository,
+)
+
+
+class GetAllChunkedTextFeature:
+    def __init__(self, document_handeler_repository: DocumentHandelerRepository):
+        self.document_handeler_repository = document_handeler_repository
+
+    async def get_all_chunked_text(self):
+
+        qdrant_response = self.document_handeler_repository.get_all_documents()
+
+        transformed_data = {}
+
+        for document in qdrant_response[0]:
+            document_id = document.payload["document_id"]
+            chunk_index = document.payload["chunk_index"]
+            text = document.payload["chunk-text"]
+
+            if document_id not in transformed_data:
+                transformed_data[document_id] = []
+
+            transformed_data[document_id].append({"index": chunk_index, "text": text})
+
+        for doc in transformed_data:
+            transformed_data[doc] = sorted(
+                transformed_data[doc], key=lambda x: x["index"]
+            )
+
+        return ChunksResponse(data=transformed_data)

Api/app/modules/documentHandeler/features/uploadDocument_feature.py

@@ -0,0 +1,26 @@
+# import os
+# from typing import Dict
+
+# from fastapi import UploadFile
+# from app.db_local_storage.files_db import FILES_DIRECTORY, FILES_NAMES_DATABASE
+# from app.db_local_storage.documents_db import documents_db
+
+
+# class UploadDocumentFeature:
+
+#     @staticmethod
+#     async def uploadFile(document: UploadFile) -> Dict[str, str]:
+#         """
+#         Upload a file to the server
+#         :param document: the file to upload
+#         :return: a message to confirm the upload
+#         """
+
+#         data = {
+#             "id": len(documents_db) + 1,
+#             "filename": document.filename,
+#         }
+
+#         documents_db.append(data)
+
+#         return {"message": "Document Updated"}

Api/app/modules/documentHandeler/routes/document_handeler_route.py

@@ -0,0 +1,31 @@
+from fastapi import APIRouter, Depends, File, UploadFile
+
+from app.modules.documentHandeler.controllers.document_handeler_controller import \
+    DocumentHandelerController
+from app.modules.documentHandeler.dependencies.dependencies import \
+    get_document_handeler_controller
+
+router = APIRouter()
+
+
+@router.get("/get_chunks")
+async def get_all_documents(
+    controller: DocumentHandelerController = Depends(get_document_handeler_controller),
+):
+    return await controller.get_all_chunks()
+
+
+@router.delete("/delete_document/{filename}")
+async def delete_document(
+    filename: str,
+    controller: DocumentHandelerController = Depends(get_document_handeler_controller),
+):
+    return await controller.delete_document(filename)
+
+
+@router.post("/upload_file")
+async def upload_file(
+    file: UploadFile = File(...),
+    controller: DocumentHandelerController = Depends(get_document_handeler_controller),
+):
+    return await controller.handle_file_upload(file)

Api/app/modules/hybridSearcher/hybridSearcher.py

@@ -0,0 +1,71 @@
+from qdrant_client import models
+from qdrant_client.conversions import common_types as types
+from qdrant_client.models import NamedVector, SparseVector
+
+from app.infrastructure.models.my_models import HybridSearchResponse
+from app.infrastructure.repository.query_search_repository import QuerySearchRepository
+from app.modules.denseEmbeddings.denseEmbeddings import DenseEmbeddings
+from app.qdrant import QdrantConnectionDb
+
+
+class HybridSearcher:
+
+    def __init__(
+        self,
+        dense_embeddings: DenseEmbeddings,
+        query_search_repository: QuerySearchRepository,
+    ):
+        self.dense_embeddings = dense_embeddings
+        self.query_search_repository = query_search_repository
+
+    def sparse_dense_rrf_prefetch(
+        self, sparse_vector: SparseVector, dense_vector: NamedVector
+    ) -> models.Prefetch:
+        result = models.Prefetch(
+            prefetch=[
+                models.Prefetch(
+                    query=dense_vector.vector,
+                    using="text-dense",
+                    limit=10,
+                ),
+                models.Prefetch(
+                    query=sparse_vector,
+                    using="text-sparse",
+                    limit=10,
+                ),
+            ],
+            query=models.FusionQuery(
+                fusion=models.Fusion.RRF,
+            ),
+        )
+
+        return result
+
+    def hybrid_search(self, user_query: str) -> types.QueryResponse:
+        """
+        Hybrid search
+
+        :param user_query: str
+        :return: types.QueryResponse
+        """
+        try:
+            sparse_vector = self.dense_embeddings.get_sparse_vector(user_query)
+            dense_vector = self.dense_embeddings.get_dense_vector(user_query)
+
+            prefetch_context = self.sparse_dense_rrf_prefetch(
+                sparse_vector, dense_vector
+            )
+
+            result = self.query_search_repository.find_text_by_hybrid_search(
+                prefetch_context, dense_vector
+            )
+
+            response_data = [
+                {"chunk-text": point.payload["chunk-text"]} for point in result.points
+            ]
+            return HybridSearchResponse(success=True, data=response_data)
+
+        except Exception as e:
+            return HybridSearchResponse(
+                success=False, message=f"Database operation failed: {str(e)}"
+            )

Api/app/modules/model.py

@@ -1,6 +0,0 @@
-from sentence_transformers import SentenceTransformer
-from transformers import pipeline
-
-
-model = SentenceTransformer("paraphrase-MiniLM-L6-v2")
-qa_pipeline = pipeline("question-answering", model="deepset/roberta-base-squad2")

Api/app/modules/querySearch/controllers/querySearch_controller.py

@@ -1,4 +1,3 @@
-from typing import Any
 from fastapi import HTTPException
 from fastapi.responses import JSONResponse
 
@@ -10,13 +9,12 @@ class QuerySearchController:
     def __init__(self, query_search_feature: QuerySearchFeature):
         self.query_search_feature = query_search_feature
 
-    async def handle_query_search(self, q: str) -> Any:
+    async def handle_query_search(self, query: str) -> JSONResponse:
         try:
 
-            result = await self.query_search_feature.query_search(q)
-            message = result.get("message", "No message provided")
+            result = await self.query_search_feature.query_search(query)
 
-            return JSONResponse(status_code=200, content={"message": message})
+            return JSONResponse(status_code=200, content=result.model_dump())
 
         except Exception as e:
             raise HTTPException(status_code=500, detail=str(e))

Api/app/modules/querySearch/dependecies.py

@@ -0,0 +1,85 @@
+from fastapi import Depends, Request
+from transformers import (AutoModel, AutoModelForMaskedLM, AutoTokenizer,
+                          pipeline)
+
+from app.infrastructure.repository.query_search_repository import \
+    QuerySearchRepository
+from app.modules.denseEmbeddings.denseEmbeddings import DenseEmbeddings
+from app.modules.hybridSearcher.hybridSearcher import HybridSearcher
+from app.modules.querySearch.controllers.querySearch_controller import \
+    QuerySearchController
+from app.modules.querySearch.features.querySearch_feature import \
+    QuerySearchFeature
+from app.modules.questionAnswer.questionAnswer import QuestionAnswering
+from app.qdrant import QdrantConnectionDb
+
+
+def get_qdrant_connection_db() -> QdrantConnectionDb:
+    return QdrantConnectionDb()
+
+
+def get_query_search_repository(
+    qdrant_connection_db: QdrantConnectionDb = Depends(get_qdrant_connection_db),
+):
+    return QuerySearchRepository(qdrant_connection_db)
+
+
+def get_dense_model(request: Request) -> AutoModel:
+    return request.scope["state"]["dense_model"]
+
+
+def get_sparse_model(request: Request) -> AutoModelForMaskedLM:
+    return request.scope["state"]["sparse_model"]
+
+
+def get_dense_tokenizer(request: Request) -> AutoTokenizer:
+    return request.scope["state"]["dense_tokenizer"]
+
+
+def get_sparse_tokenizer(request: Request) -> AutoTokenizer:
+    return request.scope["state"]["sparse_tokenizer"]
+
+
+def get_dense_embeddings(
+    dense_model: AutoModel = Depends(get_dense_model),
+    dense_tokenizer: AutoTokenizer = Depends(get_dense_tokenizer),
+    sparse_model: AutoModelForMaskedLM = Depends(get_sparse_model),
+    sparse_tokenizer: AutoTokenizer = Depends(get_sparse_tokenizer),
+):
+    return DenseEmbeddings(
+        dense_model=dense_model,
+        dense_tokenizer=dense_tokenizer,
+        sparse_model=sparse_model,
+        sparse_tokenizer=sparse_tokenizer,
+    )
+
+
+def get_qa_pipeline(request: Request):
+    return request.scope["state"]["qa_pipeline"]
+
+
+def get_question_ansering(qa_pipline: pipeline = Depends(get_qa_pipeline)):
+    return QuestionAnswering(qa_pipline)
+
+
+def get_hybrid_searcher(
+    dense_embeddings: DenseEmbeddings = Depends(get_dense_embeddings),
+    query_search_repository: QuerySearchRepository = Depends(
+        get_query_search_repository
+    ),
+):
+    return HybridSearcher(dense_embeddings, query_search_repository)
+
+
+def get_query_search_feature(
+    qa_pipeline: pipeline = Depends(get_qa_pipeline),
+    hybrid_searcher: HybridSearcher = Depends(get_hybrid_searcher),
+    question_answering: QuestionAnswering = Depends(get_question_ansering),
+):
+    return QuerySearchFeature(qa_pipeline, hybrid_searcher, question_answering)
+
+
+def get_query_search_controller(
+    query_search_feature: QuerySearchFeature = Depends(get_query_search_feature),
+):
+    return QuerySearchController(query_search_feature)

Api/app/modules/querySearch/features/querySearch_feature.py

@@ -1,89 +1,51 @@
-import json
-from typing import List, Tuple
+from qdrant_client.conversions import common_types as types
 
-import numpy as np
-# from fastapi.responses import JSONResponse
-# from sentence_transformers import SentenceTransformer
-# from transformers import pipeline
-
-from app.db_local_storage.vector_files_db import vector_files_db
-from app.db_local_storage.files_db import VECTOR_FILES_DIRECTORY
 from app.db_local_storage.in_memory_db import query_response_storage
-
+from app.infrastructure.models.my_models import HybridSearchResponse, ModelResponse
+from app.modules.hybridSearcher.hybridSearcher import HybridSearcher
+from app.modules.questionAnswer.questionAnswer import QuestionAnswering
+from transformers import pipeline
 
 
 class QuerySearchFeature:
 
-    def __init__(self, model, qa_pipeline):
-        self.model = model
+    def __init__(
+        self,
+        qa_pipeline: pipeline,
+        hybrid_searcher: HybridSearcher,
+        question_answering: QuestionAnswering,
+    ):
         self.qa_pipeline = qa_pipeline
+        self.hybrid_searcher = hybrid_searcher
+        self.question_answering = question_answering
 
-    async def query_search(self, query: str) -> dict:
-
-        user_query = {
-            "text": query,
-            "isSender": True,
-        }
-
-        query_response_storage.append(user_query)
+    async def query_search(self, query: str) -> ModelResponse:
 
-        # dataBase = await QuerySearchFeature.load_data()
-        dataBase = vector_files_db
-        text_data, embeddings = await QuerySearchFeature.split_dataBase(dataBase)
-
-        lexical_results = await QuerySearchFeature.lexical_search(query, text_data)
-        semantic_results = await QuerySearchFeature.semantic_search(
-            query, text_data, embeddings, self.model
+        query_response_storage.append(
+            {
+                "text": query,
+                "isSender": True,
+            }
         )
 
-        combined_results = list(set(lexical_results + semantic_results))
-        context = await QuerySearchFeature.get_context(combined_results)
-
-        response = self.qa_pipeline(question=query, context=context)
-
-        response_query = {
-            "text": response["answer"],
-            "isSender": False,
-        }
+        result = self.hybrid_searcher.hybrid_search(query)
+        context = self.get_and_join_context(result)
+        model_response = self.question_answering.answer_question(query, context)
 
-        query_response_storage.append(response_query)
-
-        return {
-            "message": response["answer"],
-            "context_used": context,
-            "chunks": context,
-        }
-
-    @staticmethod
-    async def semantic_search(
-        query: str, chunks: List[str], embeddings: np.ndarray, model
-    ) -> List[str]:
-        query_embedding = model.encode([query])
-        similarities = np.dot(embeddings, query_embedding.T).flatten()
-        top_indices = np.argsort(-similarities)[:3]
-        return [chunks[i] for i in top_indices]
-
-    @staticmethod
-    async def lexical_search(query: str, chunks: List[str]) -> List[str]:
-        return [chunk for chunk in chunks if query.lower() in chunk.lower()]
-
-    @staticmethod
-    async def load_data():
-        with open(VECTOR_FILES_DIRECTORY, "r") as file:
-            dataBase = json.load(file)
-        return dataBase
-
-    @staticmethod
-    async def split_dataBase(db) -> Tuple[List[str], np.ndarray]:
-        text_data = []
-        embeddings = []
+        # TODO: Manage memory for display messages
+        query_response_storage.append(
+            {
+                "text": model_response,
+                "isSender": False,
+            }
+        )
 
-        for document in db.values():
-            for page in document["data"]:
-                text_data.append(page["metadata"]["original_text"])
-                embeddings.append(page["embedding"])
-        return text_data, embeddings
+        return ModelResponse(
+            text=model_response,
+            isSender=False,
+            message="success",
+        )
 
-    @staticmethod
-    async def get_context(chunks: List[str]) -> str:
-        return " ".join(chunks)
+    def get_and_join_context(self, search_result: HybridSearchResponse) -> str:
+        contexts = [point["chunk-text"] for point in search_result.data]
+        return ", ".join(contexts)