import os import json import requests import feedparser from typing import List import pynecone as pc from pygooglenews import GoogleNews from newspaper import Article, Config import nltk import time import openai nltk.download('punkt') class State(pc.State): text: str = "Type something..." titles: List[List] = [] img_src: str = "" resource_href: List = [] src_meta: List[List] = [] summary: str = "" middle_summary_state:str = "" openai_key_show: bool = False _valid_state = ["info", "error", "success"] is_valid_code: str = _valid_state[0] _engine = GoogleNews(lang="en", country="US") _USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36' _article_config = Config() _article_config.request_timeout = 10 # _article_config.follow_meta_refresh=True _article_config.browser_user_agent = _USER_AGENT _prompt = ''' Please help me gather data from various media sources above and analyze it across multiple articles to recognize similarities and differences. For instance, if several articles report on the launch of a new Tesla car, one source might state the retail price as $10, while another mentions it as $15. The similarities between the articles would be that they all cover the new car launch, while the differences would be the varying retail prices reported. The final output should include a summary paragraph followed by a list of similarities and differences, where the differences are presented in the format of source A reporting a price of $10, while source B reports a price of $15. response should be formed organized and neat in html layout. ''' show_progress = False tmp_openai_key_text = "" OPENAI_API_KEY = "" # os.environ["my_openai_key"] _ENDPOINT_URL = 'https://api.openai.com/v1/chat/completions' _OPENAI_HEADER: dict = {} def toggle_progress(self): self.show_progress = not self.show_progress def search(self): self.summary = "" self.middle_summary_state = "" self.titles = [] self.src_meta = [] if self.text != "": src_response = self._engine.search(self.text, when="3d") # title, src, date, url for t in src_response["entries"]: self.titles.append(t["title"].split(" - ")[0]) self.src_meta.append( [t["source"]["title"], t["published"], t["link"]]) def set_text(self, text): self.text = f"intext:{text}" def set_openai_key_text(self, text): self.tmp_openai_key_text = text def submit_openai_key(self): # Define the API endpoint URL endpoint_url = "https://api.openai.com/v1/models" # Set the headers for the API request headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.tmp_openai_key_text}" } # Send the API request response = requests.get(endpoint_url, headers=headers) self.is_valid_code = self._valid_state[int( response.status_code == 200) + 1] if self.is_valid_code == "success": self.OPENAI_API_KEY = self.tmp_openai_key_text self._OPENAI_HEADER = {"Content-Type": "application/json", "Authorization": f"Bearer {self.OPENAI_API_KEY}" } def summarize(self, cur_title): cur_index = self.titles.index(cur_title) cur_src_meta = self.fetch_info(self.src_meta[cur_index][2]) print(self.is_valid_code) self.middle_summary_state += "Start related content ........\n" if cur_src_meta is not None and self.is_valid_code == "success": self.img_src = cur_src_meta["image"] self.middle_summary_state += "{}\n".format(f"Processing Main Article: {cur_src_meta['title']}") related = self._engine.search(cur_src_meta['title'], when="7d") # print("----------- Finish finding related news") summary_list = [cur_src_meta["body"]] self.resource_href.append(cur_src_meta["url"]) r_i = 1 for r in related["entries"][1:6]: # print(f"+++++++++++++: {r['title']}") r_meta = self.fetch_info(r["link"]) if r_meta is not None: summary_list.append(r_meta["body"]) self.resource_href.append(r_meta["url"]) # print(f"-------------: {r['title']}") self.middle_summary_state += "{}\n".format(f"Processing Related Article {r_i}: {r['title']}") r_i += 1 time.sleep(3) # self.summary = "\n".join(summary_list) self.call_openai(summary_list) # print("SUMMARY: ") # print(self.summary) elif cur_src_meta is not None and self.is_valid_code != "success": self.summary = "Please set up your openai api-key before running NewsGPT" else: self.summary = "Please set up your openai api-key before running NewsGPT" @pc.var def get_summary(self): if self.summary == "": return self.middle_summary_state else: return self.summary def fetch_info(self, rss_feed): news_feed = feedparser.parse(rss_feed) article = Article(news_feed["href"]) # , config=self._article_config) try: article.download() article.parse() article.nlp() return \ { "title": article.title, "body": article.text, "summary": article.summary, "image": article.top_image, "authors": article.authors, "keywords": article.keywords, "url": article.url } except: return None def call_openai(self, article_list): summary_list = [] error_list = [] # print("All Articles:") # print(". ".join(article_list)) s_i = 1 for idx, r in enumerate(article_list): messages = [ {"role": "system", "content": "You are a very professional news artlce summization and analysis agent."}, {"role": "user", "content": r + " " + "summarize the article above and preserve information on \ the following concepts: Personnel/Human Resources, Time and Place, Object/Thing. "}, ] data = { "model": "gpt-3.5-turbo", "messages": messages, "temperature": 0.7 } try: response = requests.post( url=self._ENDPOINT_URL, headers=self._OPENAI_HEADER, data=json.dumps(data)) response_json = response.json() # print(response_json) answer = response_json["choices"][0]["message"]["content"].strip( ) summary_list.append(answer) error_list.append("") self._middle_summary_state += "{}".format(f"Summarizing Article {s_i} ......") except Exception as e: error_list.append(e) if len(summary_list): summary_ = ", ".join([f"article {si} summary: {s}" for si, s in enumerate( summary_list)]) + self._prompt messages = [ {"role": "system", "content": "You are a very professional news artlce summization and analysis agent."}, {"role": "user", "content": summary_}, ] data = { "model": "gpt-3.5-turbo", "messages": messages, "temperature": 0.7 } # print(messages) self._middle_summary_state += "{}".format(f"Analyzing All Articles .....") try: response = requests.post( url=self._ENDPOINT_URL, headers=self._OPENAI_HEADER, data=json.dumps(data)) response_json = response.json() # print(response_json) self.summary = \ f""" Reference article number: {len(summary_list)} {response_json["choices"][0]["message"]["content"].strip()} """ # print("Finish Calling OPENAI") except Exception as e: self.summary = e else: # print(error_list) self.summary = "Something went wrong" def clear(self): self.summary = "" self.titles = [] self.src_meta = [] def openai_setup_window(self): self.openai_key_show = not (self.openai_key_show) @pc.var def check_openai_setup(self): return self.OPENAI_API_KEY != "" article_box_style = { "bg": "rgba(255,255,255, 0.5)", "box_shadow": "3px -3px 7px #cccecf, -3px 3px 7px #ffffff", "border_radius": "10px", "height": "80px", "width": "100%", "margin_top": "0.75em", "align_items": "left" } title_style = { "padding": "1em", "font_weight": "bold", "font_size": "0.9em", } def article_card(data): return \ pc.container( pc.box( pc.text( data, style=title_style ), on_click=State.summarize(data), style=article_box_style, _hover={"cursor": "pointer"} ), ) def home(): """The home page.""" # return pc.center( return \ pc.vstack( pc.box( pc.flex( pc.link( pc.text( "NewsGPT", font_size="1.5em", font_weight=600, background_image="linear-gradient(271.68deg, #EE756A 0.75%, #756AEE 88.52%)", background_clip="text", margin_left="20px" ), href="/", on_click=State.clear ), pc.spacer(), pc.hstack( pc.input( placeholder="New Topic?", on_blur=State.set_text, border_radius="10px", width="450px" ), pc.button( "Search", bg="lightgreen", color="black", is_active=True, on_click=State.search, size="sm", border_radius="1em", variant="outline", ), ), pc.spacer(), pc.button( "OpenAI-KEY", bg="lightgreen", color="black", is_active=True, on_click=State.openai_setup_window, size="sm", border_radius="1em", variant="outline", margin_right="20px" ), pc.alert_dialog( pc.alert_dialog_overlay( pc.alert_dialog_content( pc.alert_dialog_header( "OpenAI API-Key Setup"), pc.alert_dialog_body( pc.container( pc.input( placeholder="OpenAI Key", on_blur=State.set_openai_key_text, border_radius="10px", width="100%", type_="password", ), ), ), pc.alert_dialog_footer( pc.vstack( pc.alert( pc.alert_icon(), pc.alert_title( "set api-key [" + State.is_valid_code + "]" ), status=State.is_valid_code ), pc.hstack( pc.button( "Submit", on_click=State.submit_openai_key, ), pc.button( "Close", on_click=State.openai_setup_window, ), ), ), ), align_items="center" ), ), is_open=State.openai_key_show, ), align_items="center" ), bg="rgba(255,255,255, 0.7)", backdrop_filter="blur(10px)", padding_y=["0.8em", "0.8em", "0.5em"], border_bottom="0.08em solid rgba(32, 32, 32, .3)", position="sticky", width="80%", top="20px", z_index="99", justify="center", border_radius="20px", ), pc.grid( pc.grid_item(pc.spacer(), row_span=5, col_span=1), pc.grid_item( pc.vstack( pc.foreach( State.titles, article_card ), overflow="auto", height="875px" ), row_span=5, col_span=3, # bg="rgba(255,255,255, 0.9)", margin_top="3em", border_radius="20px", box_shadow="7px -7px 14px #cccecf, -7px 7px 14px #ffffff" ), # pc.grid_item(pc.spacer(), row_span=5, col_span=1), pc.grid_item( pc.box( pc.html(State.get_summary, padding="10px"), height="875px", width="100%", overflow="auto", ), row_span=5, col_span=3, margin_top="3em", border_radius="20px", margin_left="50px", box_shadow="7px -7px 14px #cccecf, -7px 7px 14px #ffffff" ), template_rows="repeat(5, 1fr)", template_columns="repeat(8, 1fr)", width="100%", ), pc.spacer(), background="radial-gradient(circle at 22% 11%,rgba(62, 180, 137,.20),hsla(0,0%,100%,0) 19%),radial-gradient(circle at 82% 25%,rgba(33,150,243,.18),hsla(0,0%,100%,0) 35%),radial-gradient(circle at 25% 61%,rgba(250, 128, 114, .28),hsla(0,0%,100%,0) 55%)", )