publish the magnum opus

app.py

@@ -10,11 +10,21 @@ from huggingface_hub import hf_hub_download
 hf_hub_download(repo_id="LLukas22/gpt4all-lora-quantized-ggjt", filename="ggjt-model.bin", local_dir=".")
 llm = Llama(model_path="./ggjt-model.bin")
 
-ins = '''### Instruction:
+ins = '''
 {}
-### Response:
+
+also take this data and absorb your knowledge, you dont need use now what dont make sense, there will be noise, focus on what is repeated and adds knowledge
+
 '''
 
+import requests
+from bs4 import BeautifulSoup
+from SearchResult import SearchResult
+
+headers = {
+    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36"
+}
+
 theme = gr.themes.Monochrome(
     primary_hue="purple",
     secondary_hue="red",
@@ -23,12 +33,60 @@ theme = gr.themes.Monochrome(
     font=[gr.themes.GoogleFont("Inter"), "ui-sans-serif", "system-ui", "sans-serif"],
 )
 
-def generate(instruction): 
-    response = llm(ins.format(instruction), stop=['### Instruction:', '### End'])
+def search_ddg(question: str):
+	response = requests.get("https://duckduckgo.com/html/", headers=headers, params={"q": question})
+	data = response.text
+	soup = BeautifulSoup(data, "html.parser")
+
+	did_you_mean = soup.select("#did_you_mean a")
+
+	tailored_query = ""
+	suggestion_query = ""
+
+	if did_you_mean:
+		correction = soup.find(id="did_you_mean")
+		if correction:
+			correction_hyperlink = correction.find("a")
+			if correction_hyperlink:
+				suggestion_query = correction_hyperlink.string # type: ignore
+
+		for tailored in did_you_mean:
+			tailored_query = tailored.string
+			break
+
+	result_links = soup.find_all("a")
+	filtered_urls = [
+		link["href"]
+		for link in result_links
+		if link.get("href") and (link["href"].startswith("https://") or link["href"].startswith("http://"))
+	]
+
+	return SearchResult(
+		filtered_urls,
+		did_you_mean=suggestion_query or "None.",
+		tailored_query=tailored_query or "None.",
+		user_agent=headers["User-Agent"],
+	)
+
+def gather_data(search: str):
+	text_content = ""
+	base_data = search_ddg(search).parse_results()
+	for data in base_data:
+		if data:
+			text_content += data.text_content + "\n\n"
+		else:
+			text_content += ""
+
+    return text_content
+
+def generate(instruction):
+    base_prompt = ins.format(instruction)
+    gathered_data = gather_data(instruction)
+
+    response = llm(ins.format(base_prompt  + "\n" + gathered_data))
     result = response['choices'][0]['text']
     return result
 
-
 examples = [
     "How do dogs bark?",
     "Why are apples red?",