Spaces:
Runtime error
Runtime error
File size: 8,060 Bytes
db1003c 34d89c7 a5a25a9 db1003c 76d488b db1003c 6b6f1c9 db1003c 6b6f1c9 db1003c 76d488b db1003c 6b6f1c9 db1003c 76d488b db1003c 6b6f1c9 db1003c 34d89c7 db1003c 6b6f1c9 db1003c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 |
from llama_index import GPTSimpleVectorIndex, SimpleDirectoryReader, LLMPredictor, Document,ServiceContext
from langchain.llms import OpenAIChat
from llama_index import download_loader
from langchain.chains import LLMChain, TransformChain, SimpleSequentialChain
from langchain.prompts import PromptTemplate
from langchain.agents import initialize_agent, Tool,load_tools
from langchain.chat_models import ChatOpenAI
import gradio as gr
import pandas as pd
import openai
import re
from bs4 import BeautifulSoup
import pandas as pd
import datetime
from datetime import datetime, date, time, timedelta
import os
import regex
import requests
import json
from sec_edgar_downloader._utils import get_filing_urls_to_download
listofcategories=["10-K", "10-Q","8-K"]
def getstuff(openapikey,category_selector,ticker_input):
dateforfilesave=datetime.today().strftime("%d-%m-%Y %I:%M%p")
print(ticker_input)
print(dateforfilesave)
if openapikey=='':
return pd.DataFrame(["Please provide OpenAPI Key"],columns=['ERROR']),pd.DataFrame(["Please provide OpenAPI Key"],columns=['ERROR']),'Error: Please provide OpenAPI key','Error: Please provide OpenAPI key'
os.environ['OPENAI_API_KEY'] = str(openapikey)
if category_selector=='10-K':
num_filings_needed=1
elif category_selector=='8-K':
num_filings_needed=8
elif category_selector=='10-Q':
num_filings_needed=4
else:
num_filings_needed=1
filings_temp=get_filing_urls_to_download(category_selector, ticker_input,num_filings_to_download=num_filings_needed,include_amends=False,before_date='2023-04-01',after_date='2022-01-01')
files=[filings_temp[0].full_submission_url]
print('Came here1')
filetextcontentlist=[]
for each in files:
headers = {
"User-Agent": '[email protected]',
"Accept-Encoding": "gzip, deflate",
"Host": "www.sec.gov",
}
resp=requests.get(each,headers=headers)
raw_10k = resp.text
print('Came here2')
# Regex to find <DOCUMENT> tags
doc_start_pattern = re.compile(r'<DOCUMENT>')
doc_end_pattern = re.compile(r'</DOCUMENT>')
# Regex to find <TYPE> tag prceeding any characters, terminating at new line
type_pattern = re.compile(r'<TYPE>[^\n]+')
# Create 3 lists with the span idices for each regex
### There are many <Document> Tags in this text file, each as specific exhibit like 10-K, EX-10.17 etc
### First filter will give us document tag start <end> and document tag end's <start>
### We will use this to later grab content in between these tags
doc_start_is = [x.end() for x in doc_start_pattern.finditer(raw_10k)]
doc_end_is = [x.start() for x in doc_end_pattern.finditer(raw_10k)]
### Type filter is interesting, it looks for <TYPE> with Not flag as new line, ie terminare there, with + sign
### to look for any char afterwards until new line \n. This will give us <TYPE> followed Section Name like '10-K'
### Once we have have this, it returns String Array, below line will with find content after <TYPE> ie, '10-K'
### as section names
doc_types = [x[len('<TYPE>'):] for x in type_pattern.findall(raw_10k)]
document = {}
# Create a loop to go through each section type and save only the 10-K section in the dictionary
for doc_type, doc_start, doc_end in zip(doc_types, doc_start_is, doc_end_is):
if doc_type == category_selector:
document[doc_type] = raw_10k[doc_start:doc_end]
item_content = BeautifulSoup(document[category_selector], 'lxml')
filetextcontentlist.append(str(item_content.text.encode('ascii','ignore')))
print('Came here3')
temp=". ".join(filetextcontentlist).replace('\xa024',' ')
temp=temp.replace('\n',' ').strip()
temp=temp.split('.')
newlist=[]
for each in temp:
if len(each.split())>10: ###eliminate sentences with less words
newlist.append(each)
documents=[Document(t) for t in newlist]
index = GPTSimpleVectorIndex.from_documents(documents)
print('Came here4')
querylist=['What are the main products/ services mentioned?','What are the major risks?',"What are the top investment focus areas?","What is the financial outlook of the company?","What key technologies like AI, blockchain etc are mentioned?","What other company names/ competitors are mentioned?"]
llm = ChatOpenAI(temperature=0)
llm_predictor = LLMPredictor(llm=OpenAIChat(temperature=0, model_name="gpt-3.5-turbo"))
service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor)
answerlist=[]
for i in range(len(querylist)):
print(i,"Query: ",querylist[i])
response = index.query(
querylist[i],
service_context=service_context,
response_mode="tree_summarize",
similarity_top_k=min(int(len(documents)/3),20)
)
print(response.response)
if 'dataframe' in querylist[i]:
try:
pattern = regex.compile(r'\{(?:[^{}]|(?R))*\}')
jsonextract=pattern.findall(response.response)[0]
#print("json extract\n",jsonextract)
df_tmp=pd.read_json(jsonextract)
if len(df_tmp.columns)<=1:
df=pd.DataFrame(df_tmp[df_tmp.columns[0]].tolist())
else:
df=df_tmp
except:
df=pd.DataFrame()
df['message']=['Data insufficient to decipher']
df['action']=['try again in a few hours']
answerlist.append(df)
else:
answerlist.append(querylist[i]+'\n'+response.response)
print('Came to return statement')
return answerlist
with gr.Blocks() as demo:
gr.Markdown("<h1><center>ChatGPT SEC Filings Question Answers</center></h1>")
gr.Markdown(
"""What are the products & services? What are the risks? What is the outlook? and much more. \n\nThis is a demo & showcases ChatGPT integrated with real data. It shows how to get real-time data and marry it with ChatGPT capabilities. This demonstrates 'Chain of Thought' thinking using ChatGPT.\n\n4 snapshots are provided for illustration (trends, sector outlook, news summary email, macro trends email)\n\nNote: llama-index & gpt-3.5-turbo are used. The analysis takes roughly 120 secs & may not always be consistent. If ChatGPT API is overloaded you will get an error\n """
)
with gr.Row() as row:
with gr.Column():
category_selector=gr.Dropdown(
listofcategories, label="Filing Categories", info="Select the filing you want..."
)
input1 = gr.Textbox(placeholder='Enter ticker (USA only)', lines=1,label='Ticker')
with gr.Column():
textboxopenapi = gr.Textbox(placeholder="Enter OpenAPI Key...", lines=1,label='OpenAPI Key')
with gr.Column():
btn = gr.Button("Generate \nAnswers")
with gr.Row() as row:
with gr.Column():
output1 = gr.Textbox(placeholder='', lines=4,label='Snapshot 1')
with gr.Column():
output2 = gr.Textbox(placeholder='', lines=4,label='Snapshot 2')
with gr.Row() as row:
with gr.Column():
output3 = gr.Textbox(placeholder='', lines=4,label='Snapshot 3')
with gr.Column():
output4 = gr.Textbox(placeholder='', lines=4,label='Snapshot 4')
with gr.Row() as row:
with gr.Column():
output5 = gr.Textbox(placeholder='', lines=4,label='Snapshot 5')
with gr.Column():
output6 = gr.Textbox(placeholder='', lines=4,label='Snapshot 6')
btn.click(getstuff, inputs=[textboxopenapi,category_selector,input1],outputs=[output1,output2,output3,output4,output5,output6])
demo.launch(debug=True) |