import tiktoken import os from bs4 import BeautifulSoup import gradio as gr from langchain import OpenAI, ConversationChain, LLMChain, PromptTemplate from langchain.memory import ConversationBufferWindowMemory import openai import requests from langchain.chat_models import ChatOpenAI import ast import re import json import tempfile import collectionstions OPENAI_API_KEY = os.environ['OPENAI_API_KEY'] def save_webpage_as_html(url): headers = { 'authority': 'ms-mt--api-web.spain.advgo.net', 'sec-ch-ua': '" Not;A Brand";v="99", "Google Chrome";v="91", "Chromium";v="91"', 'accept': 'application/json, text/plain, */*', 'x-adevinta-channel': 'web-desktop', 'x-schibsted-tenant': 'coches', 'sec-ch-ua-mobile': '?0', 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36', 'content-type': 'application/json;charset=UTF-8', 'origin': 'https://www.coches.net', 'sec-fetch-site': 'cross-site', 'sec-fetch-mode': 'cors', 'sec-fetch-dest': 'empty', 'referer': 'https://www.coches.net/', 'accept-language': 'en-US,en;q=0.9,es;q=0.8', } response = requests.get(url, headers=headers) # Check if the request was successful if response.status_code != 200: print(f"Failed to get the webpage: {url}") return # Create a BeautifulSoup object and specify the parser soup = BeautifulSoup(response.text, 'html.parser') # Create a dictionary to hold the result result = collections.defaultdict(list) # Find all tags that contain text (you may need to add more tags to this list) for tag in soup.find_all(['li', 'ol']): result[tag.name].append(tag.get_text(strip=True)) return result output_json_format = ''' { "category": "root_category", "subcategories": [ { "category": "node_category", "subcategories": [ { "category": "node_category", "subcategories": [category1, category2, ...] }, { "category": "node_category", "subcategories": [category1, category2, ...] } ] }, { "category": "node_category", "subcategories": [category1, category2, ...] } ] } ''' empty_json = { "category": "root_category", "subcategories": [ ] } def get_taxanomy_from_url(url): url_dict = save_webpage_as_html(url) json_input = str(url_dict) template = ''' {history} {human_input} ''' prompt = PromptTemplate( input_variables=["history", "human_input"], template=template ) chatgpt_chain = LLMChain( llm=ChatOpenAI(model="gpt-4", temperature=0,openai_api_key=OPENAI_API_KEY), prompt=prompt, verbose=True, memory=ConversationBufferWindowMemory(k=10), ) prompt_input2 = f''' You are an expert ecommerce product taxanomy analyst. You are equiped with vast knowledge of taxanomy, ontology and everything related to it. You fit have deep expertise in the domain of: "An ontology identifies and distinguishes concepts and their relationships; it describes content and relationships. A taxonomy formalizes the hierarchical relationships among concepts and specifies the term to be used to refer to each; it prescribes structure and terminology." You have a task to extract taxanomy from a python dictionary of an extracted html page of an ecommerce website. Here is the input python dictionary: {json_input} Here is the output json format: {output_json_format} From the input python dictionary, extract all available products under the li and ol key and create the output json taxanomy. Think step by step. Place the products in categories and subcategories accordingly. Organize all the products to fit the output json format. The output should follow a python dictionary.. Do not declare a new variable, output the python dictionary json object only. Do not output "The taxonomy extracted from the given python list can be represented as follows:" Do not provide extra information. Directly output the python dictionary only. Do not insert any string before or after the python dictionary. Output python dictionary only. ''' encoding = tiktoken.encoding_for_model("gpt-4") encoded_prompt2 = encoding.encode(prompt_input2)[:8000] prompt_input2 = encoding.decode(encoded_prompt2) json_dict = "" while type(json_dict) != dict: json_taxanomy_output=chatgpt_chain.predict(human_input=prompt_input2) json_dict = ast.literal_eval(json_taxanomy_output) file_name = "url_temp.json" # Save the modified data back to the file with open(file_name, 'w') as json_file: json.dump(json_dict, json_file, indent=4) # 'indent' parameter makes the output more readable return(file_name) def expand_taxanomy(json_dict, num_layers, num_items, category_type): num_layers = str(int(num_layers)) num_items = str(int(num_items)) json_input = str(json_dict) template = ''' {history} {human_input} ''' prompt = PromptTemplate( input_variables=["history", "human_input"], template=template ) chatgpt_chain = LLMChain( llm=ChatOpenAI(model="gpt-4", temperature=0,openai_api_key=OPENAI_API_KEY), prompt=prompt, verbose=True, memory=ConversationBufferWindowMemory(k=10), ) prompt_input1 = f''' You are an expert ecommerce product taxanomy analyst. You are equiped with vast knowledge of taxanomy, ontology and everything related to it. You fit have deep expertise in the domain of: "An ontology identifies and distinguishes concepts and their relationships; it describes content and relationships. A taxonomy formalizes the hierarchical relationships among concepts and specifies the term to be used to refer to each; it prescribes structure and terminology." You have a task to expand a taxanomy that is formatted in a json file. The taxanomy tree should be {num_layers} layer deep with a total of {num_items} items. The category type is {category_type}. Here is the input json file: {json_input} Here is the output json format: {output_json_format} Expand the taxanomy of the input json file. Find subcategories that fits each category. Expand the leafs of the taxanomy tree. Go deeper. Think step by step. Find all subcategories and output it as a json object. The output should follow a python dictionary.. Do not declare a new variable, output the python dictionary json object only. Do not provide extra information. Directly output the python dictionary only. ''' encoding = tiktoken.encoding_for_model("gpt-4") encoded_prompt1 = encoding.encode(prompt_input1)[:8000] prompt_input1 = encoding.decode(encoded_prompt1) json_taxanomy_output=chatgpt_chain.predict(human_input=prompt_input1) json_dict = ast.literal_eval(json_taxanomy_output) return(json_dict) def add_nodes_edges(graph, data, parent=None): new_name = data['category'] # create node graph.node(new_name) if parent: # create an edge between parent and child graph.edge(parent, new_name) # iterate over subcategories (if they exist) for subcat in data.get('subcategories', []): # subcategories can be either strings or new dicts if isinstance(subcat, str): # create node for the string subcategory graph.node(subcat) # create edge between the parent category and this subcategory graph.edge(new_name, subcat) else: # if subcat is a dict, repeat the process with subcat as the parent add_nodes_edges(graph, subcat, new_name) def visualize_json(data): graph = graphviz.Digraph(graph_attr={'rankdir': 'LR'}) # Added 'LR' for left to right graph # Add nodes and edges add_nodes_edges(graph, data) # Visualize the graph #graph.view() return graph def get_file(json_file): try: print("loading json file") print("temp_file", json_file.name) file_path = json_file.name with open(file_path, 'r') as json_file: data = json.load(json_file) except: print("using temp json") file_path = 'temp.json' with open(file_path, 'r') as json_file: data = json.load(json_file) try: os.remove('graph.png') print("graph removed") except: print("no existing graph") graph = visualize_json(data) # Render the graph as a PNG file graph.format = 'png' graph = graph.render(filename='graph', cleanup=True) return graph def modify_json(json_input, num_layers, num_items, category_type): print("json_input first", json_input) if json_input is not None: file_path = json_input.name # Open the file and load the JSON data with open(file_path, 'r') as json_file: data = json.load(json_file) else: data = empty_json data["category"] = category_type # Directly from dictionary file_path = 'temp.json' with open(file_path, 'w') as outfile: json.dump(data, outfile) json_dict = expand_taxanomy(data, num_layers, num_items,category_type) print("json_dict", json_dict) # Save the modified data back to the file with open(file_path, 'w') as json_file: json.dump(json_dict, json_file, indent=4) # 'indent' parameter makes the output more readable return(file_path) def print_num(a,b): return(int(a), int(b)) with gr.Blocks() as demo: gr.Markdown( """ # Auto Taxanomy App Upload a JSON taxanomy file or generate from scratch. """) with gr.Row(): with gr.Column(): json_file = gr.File(label="Upload JSON here.") num_layers = gr.Number(label="Number of layers") num_items = gr.Number(label="Number of items") category_type = gr.Text(label="Category type") modify_btn = gr.Button(value="Generate") render_btn = gr.Button(value="Render") print_btn = gr.Button(value="Print") with gr.Column(): input_url = gr.Text(label="Insert URL") geturl_btn = gr.Button(value="Get JSON Taxanomy") #url_json_file = gr.File(label="URL JSON file.") rendered_tree = gr.Image(label="Taxanomy Tree.") output_file = gr.File(label="Ouput JSON file.") #print_text = gr.Text(label="Printing") modify_btn.click(modify_json, inputs=[json_file, num_layers, num_items, category_type], outputs=output_file) render_btn.click(get_file, inputs=json_file, outputs=rendered_tree) #print_btn.click(print_num, inputs=[num_layers,num_items], outputs=print_text) geturl_btn.click(get_taxanomy_from_url, inputs=input_url, outputs=output_file) demo.launch()