Update app.py
Browse files
app.py
CHANGED
@@ -95,11 +95,6 @@ def get_system_tokens(model):
|
|
95 |
return get_message_tokens(model, **system_message)
|
96 |
|
97 |
|
98 |
-
def upload_files(files, file_paths):
|
99 |
-
file_paths = [f.name for f in files]
|
100 |
-
return file_paths
|
101 |
-
|
102 |
-
|
103 |
def process_text(text):
|
104 |
lines = text.split("\n")
|
105 |
lines = [line for line in lines if len(line.strip()) > 2]
|
@@ -109,17 +104,24 @@ def process_text(text):
|
|
109 |
return text
|
110 |
|
111 |
|
|
|
|
|
|
|
|
|
|
|
112 |
def build_index(file_paths, db, chunk_size, chunk_overlap, file_warning):
|
113 |
documents = [load_single_document(path) for path in file_paths]
|
114 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
115 |
documents = text_splitter.split_documents(documents)
|
|
|
116 |
fixed_documents = []
|
117 |
for doc in documents:
|
118 |
doc.page_content = process_text(doc.page_content)
|
119 |
if not doc.page_content:
|
120 |
continue
|
121 |
fixed_documents.append(doc)
|
122 |
-
|
|
|
123 |
db = Chroma.from_documents(
|
124 |
fixed_documents,
|
125 |
EMBEDDER,
|
@@ -127,15 +129,11 @@ def build_index(file_paths, db, chunk_size, chunk_overlap, file_warning):
|
|
127 |
anonymized_telemetry=False
|
128 |
)
|
129 |
)
|
|
|
130 |
file_warning = f"Загружено {len(fixed_documents)} фрагментов! Можно задавать вопросы."
|
131 |
return db, file_warning
|
132 |
|
133 |
|
134 |
-
def user(message, history, system_prompt):
|
135 |
-
new_history = history + [[message, None]]
|
136 |
-
return "", new_history
|
137 |
-
|
138 |
-
|
139 |
def retrieve(history, db, retrieved_docs, k_documents):
|
140 |
retrieved_docs = ""
|
141 |
if db:
|
@@ -145,6 +143,11 @@ def retrieve(history, db, retrieved_docs, k_documents):
|
|
145 |
retrieved_docs = "\n\n".join([doc.page_content for doc in docs])
|
146 |
return retrieved_docs
|
147 |
|
|
|
|
|
|
|
|
|
|
|
148 |
|
149 |
def bot(
|
150 |
history,
|
|
|
95 |
return get_message_tokens(model, **system_message)
|
96 |
|
97 |
|
|
|
|
|
|
|
|
|
|
|
98 |
def process_text(text):
|
99 |
lines = text.split("\n")
|
100 |
lines = [line for line in lines if len(line.strip()) > 2]
|
|
|
104 |
return text
|
105 |
|
106 |
|
107 |
+
def upload_files(files, file_paths):
|
108 |
+
file_paths = [f.name for f in files]
|
109 |
+
return file_paths
|
110 |
+
|
111 |
+
|
112 |
def build_index(file_paths, db, chunk_size, chunk_overlap, file_warning):
|
113 |
documents = [load_single_document(path) for path in file_paths]
|
114 |
text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
|
115 |
documents = text_splitter.split_documents(documents)
|
116 |
+
print("Documents after split:", len(documents))
|
117 |
fixed_documents = []
|
118 |
for doc in documents:
|
119 |
doc.page_content = process_text(doc.page_content)
|
120 |
if not doc.page_content:
|
121 |
continue
|
122 |
fixed_documents.append(doc)
|
123 |
+
print("Documents after processing:", len(fixed_documents))
|
124 |
+
|
125 |
db = Chroma.from_documents(
|
126 |
fixed_documents,
|
127 |
EMBEDDER,
|
|
|
129 |
anonymized_telemetry=False
|
130 |
)
|
131 |
)
|
132 |
+
print("Embeddings calculated!")
|
133 |
file_warning = f"Загружено {len(fixed_documents)} фрагментов! Можно задавать вопросы."
|
134 |
return db, file_warning
|
135 |
|
136 |
|
|
|
|
|
|
|
|
|
|
|
137 |
def retrieve(history, db, retrieved_docs, k_documents):
|
138 |
retrieved_docs = ""
|
139 |
if db:
|
|
|
143 |
retrieved_docs = "\n\n".join([doc.page_content for doc in docs])
|
144 |
return retrieved_docs
|
145 |
|
146 |
+
|
147 |
+
def user(message, history, system_prompt):
|
148 |
+
new_history = history + [[message, None]]
|
149 |
+
return "", new_history
|
150 |
+
|
151 |
|
152 |
def bot(
|
153 |
history,
|