Spaces:
Runtime error
Runtime error
File size: 8,420 Bytes
db1003c 34d89c7 a5a25a9 db1003c a0a3021 db1003c a0a3021 db1003c e837bca db1003c e837bca db1003c a0a3021 db1003c 76d488b db1003c 6b6f1c9 db1003c 6b6f1c9 db1003c 76d488b db1003c 6b6f1c9 a0a3021 db1003c 76d488b db1003c e837bca db1003c 8989162 db1003c a0a3021 db1003c 6b6f1c9 a0a3021 db1003c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 |
from llama_index import GPTSimpleVectorIndex, SimpleDirectoryReader, LLMPredictor, Document,ServiceContext
from langchain.llms import OpenAIChat
from llama_index import download_loader
from langchain.chains import LLMChain, TransformChain, SimpleSequentialChain
from langchain.prompts import PromptTemplate
from langchain.agents import initialize_agent, Tool,load_tools
from langchain.chat_models import ChatOpenAI
import gradio as gr
import pandas as pd
import openai
import re
from bs4 import BeautifulSoup
import pandas as pd
import datetime
from datetime import datetime, date, time, timedelta
import os
import regex
import requests
import json
from sec_edgar_downloader._utils import get_filing_urls_to_download
listofcategories=["10-K", "10-Q","8-K"]
def getstuff(openapikey,category_selector,ticker_input,user_question):
dateforfilesave=datetime.today().strftime("%d-%m-%Y %I:%M%p")
print(ticker_input)
print(user_question)
print(dateforfilesave)
if openapikey=='':
return ["Please provide OpenAPI Key","Please provide OpenAPI Key","Please provide OpenAPI Key","Please provide OpenAPI Key","Please provide OpenAPI Key","Please provide OpenAPI Key","Please provide OpenAPI Key",]
os.environ['OPENAI_API_KEY'] = str(openapikey)
if category_selector=='10-K':
num_filings_needed=1
elif category_selector=='8-K':
num_filings_needed=4
elif category_selector=='10-Q':
num_filings_needed=2
else:
num_filings_needed=1
filings_temp=get_filing_urls_to_download(category_selector, ticker_input,num_filings_to_download=num_filings_needed,include_amends=False,before_date='2023-04-01',after_date='2022-01-01')
files=[filings_temp[i].full_submission_url for i in range(len(filings_temp))]
print('Came here1')
filetextcontentlist=[]
for each in files:
headers = {
"User-Agent": '[email protected]',
"Accept-Encoding": "gzip, deflate",
"Host": "www.sec.gov",
}
resp=requests.get(each,headers=headers)
raw_10k = resp.text
print('Came here2')
# Regex to find <DOCUMENT> tags
doc_start_pattern = re.compile(r'<DOCUMENT>')
doc_end_pattern = re.compile(r'</DOCUMENT>')
# Regex to find <TYPE> tag prceeding any characters, terminating at new line
type_pattern = re.compile(r'<TYPE>[^\n]+')
# Create 3 lists with the span idices for each regex
### There are many <Document> Tags in this text file, each as specific exhibit like 10-K, EX-10.17 etc
### First filter will give us document tag start <end> and document tag end's <start>
### We will use this to later grab content in between these tags
doc_start_is = [x.end() for x in doc_start_pattern.finditer(raw_10k)]
doc_end_is = [x.start() for x in doc_end_pattern.finditer(raw_10k)]
### Type filter is interesting, it looks for <TYPE> with Not flag as new line, ie terminare there, with + sign
### to look for any char afterwards until new line \n. This will give us <TYPE> followed Section Name like '10-K'
### Once we have have this, it returns String Array, below line will with find content after <TYPE> ie, '10-K'
### as section names
doc_types = [x[len('<TYPE>'):] for x in type_pattern.findall(raw_10k)]
document = {}
# Create a loop to go through each section type and save only the 10-K section in the dictionary
for doc_type, doc_start, doc_end in zip(doc_types, doc_start_is, doc_end_is):
if doc_type == category_selector:
document[doc_type] = raw_10k[doc_start:doc_end]
item_content = BeautifulSoup(document[category_selector], 'lxml')
filetextcontentlist.append(str(item_content.text.encode('ascii','ignore')))
print('Came here3')
temp=". ".join(filetextcontentlist).replace('\xa024',' ')
temp=temp.replace('\n',' ').strip()
temp=temp.split('.')
newlist=[]
for each in temp:
if len(each.split())>10: ###eliminate sentences with less words
newlist.append(each)
documents=[Document(t) for t in newlist]
index = GPTSimpleVectorIndex.from_documents(documents)
print('Came here4')
querylist=['What are the main products/ services mentioned?','What are the major risks?',"What are the top investment focus areas?","What is the financial outlook of the company?","What key technologies like AI, blockchain etc are mentioned?","What other company names/ competitors are mentioned?"]
if user_question=='':
querylist.append('What is the key summary?')
else:
querylist.append(user_question)
llm = ChatOpenAI(temperature=0)
llm_predictor = LLMPredictor(llm=OpenAIChat(temperature=0, model_name="gpt-3.5-turbo"))
service_context = ServiceContext.from_defaults(llm_predictor=llm_predictor)
answerlist=[]
for i in range(len(querylist)):
print(i,"Query: ",querylist[i])
response = index.query(
querylist[i],
service_context=service_context,
response_mode="tree_summarize",
similarity_top_k=min(int(len(documents)/3),20)
)
print(response.response)
if 'dataframe' in querylist[i]:
try:
pattern = regex.compile(r'\{(?:[^{}]|(?R))*\}')
jsonextract=pattern.findall(response.response)[0]
#print("json extract\n",jsonextract)
df_tmp=pd.read_json(jsonextract)
if len(df_tmp.columns)<=1:
df=pd.DataFrame(df_tmp[df_tmp.columns[0]].tolist())
else:
df=df_tmp
except:
df=pd.DataFrame()
df['message']=['Data insufficient to decipher']
df['action']=['try again in a few hours']
answerlist.append(df)
else:
answerlist.append(querylist[i]+'\n\n'+response.response)
print('Came to return statement')
return answerlist
with gr.Blocks() as demo:
gr.Markdown("<h1><center>ChatGPT SEC Filings Question Answers</center></h1>")
gr.Markdown(
"""What are the products & services? What are the risks? What is the outlook? and much more. \n\nThis is a demo & showcases ChatGPT integrated with real data. It shows how to get real-time data and marry it with ChatGPT capabilities.\n\nMultiple snapshots/ Question-Answers are provided for illustration (products, risks, focus areas, etc). The latest one 10-K/ two 10-Q/ four 8-K filings are pulled\n\nNote: llama-index & gpt-3.5-turbo are used. The analysis takes more than 1-3 mins & may not always be consistent. If ChatGPT API is overloaded/ no API key is provided/ API quota is over you will get an error\n """
)
with gr.Row() as row:
with gr.Column():
category_selector=gr.Dropdown(
listofcategories, label="Filing Categories", info="Select the filing you want..."
)
input1 = gr.Textbox(placeholder='Enter ticker (USA only)', lines=1,label='Ticker')
with gr.Column():
input2 = gr.Textbox(placeholder='Enter your question', lines=1,label='User Question')
textboxopenapi = gr.Textbox(placeholder="Enter OpenAPI Key...", lines=1,label='OpenAPI Key')
with gr.Column():
btn = gr.Button("Generate \nAnswers")
with gr.Row() as row:
with gr.Column():
output1 = gr.Textbox(placeholder='', lines=4,label='Snapshot 1')
with gr.Column():
output2 = gr.Textbox(placeholder='', lines=4,label='Snapshot 2')
with gr.Row() as row:
with gr.Column():
output3 = gr.Textbox(placeholder='', lines=4,label='Snapshot 3')
with gr.Column():
output4 = gr.Textbox(placeholder='', lines=4,label='Snapshot 4')
with gr.Row() as row:
with gr.Column():
output5 = gr.Textbox(placeholder='', lines=4,label='Snapshot 5')
with gr.Column():
output6 = gr.Textbox(placeholder='', lines=4,label='Snapshot 6')
btn.click(getstuff, inputs=[textboxopenapi,category_selector,input1,input2],outputs=[output1,output2,output3,output4,output5,output6,input2])
demo.launch(debug=True) |