Upload 2 files

Dataset-10k.zip

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a2aaba7baab33f913f9e984d8a247052c1203f71cdb2a05eb2a98708044fbfa4
+size 5198341

app.py

@@ -0,0 +1,74 @@
+
+
+## Setup
+# Import the necessary Libraries
+import json
+import tiktoken
+import pandas as pd
+from openai import OpenAI
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_community.document_loaders import PyPDFDirectoryLoader
+from langchain_community.embeddings.sentence_transformer import (
+    SentenceTransformerEmbeddings
+)
+from langchain_community.vectorstores import Chroma
+
+import os
+import uuid
+import joblib
+import json
+
+import gradio as gr
+
+
+from huggingface_hub import CommitScheduler
+from pathlib import Path
+
+# Create Client
+client = OpenAI(
+    base_url="https://api.endpoints.anyscale.com/v1",
+    api_key=secret_key
+)
+
+# Define the embedding model and the vectorstore
+embedding_model = SentenceTransformerEmbeddings(model_name='thenlper/gte-large')
+
+# Load the persisted vectorDB
+persisted_vectordb_location = './proj3_db'
+
+# Prepare the logging functionality
+
+log_file = Path("logs/") / f"data_{uuid.uuid4()}.json"
+log_folder = log_file.parent
+
+scheduler = CommitScheduler(
+    repo_id="---------",
+    repo_type="dataset",
+    folder_path=log_folder,
+    path_in_repo="data",
+    every=2
+)
+
+# Define the Q&A system message
+qna_system_message = """
+               
+        User input will have the context required by you to answer user questions.
+        This context will begin with the token: ###Context
+        The context contains references to specific portions of a document relevant to the user query.
+        
+        User questions will begin with the token: ###Question
+        
+        Please answer only using the context provided in the input. Do not mention anything about the context in your final answer.
+        
+        If the answer is not found in the context, respond "I don't know".
+"""
+
+# Define the user message template
+qna_user_message_template = """
+###Context
+Here are some documents that are relevant to the question mentioned below.
+{context}
+
+###Question
+{question}
+"""