update cron job for create stock db
This commit is contained in:
MuslemRahimi
parent
0fa400ef10
commit
984354fd95
@ -100,17 +100,8 @@ class StockDatabase:
|
|||||||
urls = [
|
urls = [
|
||||||
f"https://financialmodelingprep.com/api/v3/profile/{symbol}?apikey={api_key}",
|
f"https://financialmodelingprep.com/api/v3/profile/{symbol}?apikey={api_key}",
|
||||||
f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}",
|
f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}",
|
||||||
f"https://financialmodelingprep.com/api/v3/income-statement/{symbol}?period=annual&apikey={api_key}",
|
|
||||||
f"https://financialmodelingprep.com/api/v3/income-statement-growth/{symbol}?period=annual&apikey={api_key}",
|
|
||||||
#f"https://financialmodelingprep.com/api/v4/esg-environmental-social-governance-data-ratings?symbol={symbol}&apikey={api_key}",
|
|
||||||
#f"https://financialmodelingprep.com/api/v4/esg-environmental-social-governance-data?symbol={symbol}&apikey={api_key}",
|
|
||||||
f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_dividend/{symbol}?limit=400&apikey={api_key}",
|
f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_dividend/{symbol}?limit=400&apikey={api_key}",
|
||||||
f"https://financialmodelingprep.com/api/v4/historical/employee_count?symbol={symbol}&apikey={api_key}",
|
f"https://financialmodelingprep.com/api/v4/historical/employee_count?symbol={symbol}&apikey={api_key}",
|
||||||
f"https://financialmodelingprep.com/api/v3/balance-sheet-statement/{symbol}?period=annual&apikey={api_key}",
|
|
||||||
f"https://financialmodelingprep.com/api/v3/balance-sheet-statement-growth/{symbol}?period=annual&apikey={api_key}",
|
|
||||||
f"https://financialmodelingprep.com/api/v3/cash-flow-statement/{symbol}?period=annual&apikey={api_key}",
|
|
||||||
f"https://financialmodelingprep.com/api/v3/cash-flow-statement-growth/{symbol}?period=annual&apikey={api_key}",
|
|
||||||
f"https://financialmodelingprep.com/api/v3/ratios/{symbol}?period=annual&apikey={api_key}",
|
|
||||||
f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_split/{symbol}?apikey={api_key}",
|
f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_split/{symbol}?apikey={api_key}",
|
||||||
f"https://financialmodelingprep.com/api/v4/stock_peers?symbol={symbol}&apikey={api_key}",
|
f"https://financialmodelingprep.com/api/v4/stock_peers?symbol={symbol}&apikey={api_key}",
|
||||||
f"https://financialmodelingprep.com/api/v4/institutional-ownership/institutional-holders/symbol-ownership-percent?date={quarter_date}&symbol={symbol}&page=0&apikey={api_key}",
|
f"https://financialmodelingprep.com/api/v4/institutional-ownership/institutional-holders/symbol-ownership-percent?date={quarter_date}&symbol={symbol}&page=0&apikey={api_key}",
|
||||||
@ -156,75 +147,11 @@ class StockDatabase:
|
|||||||
}
|
}
|
||||||
fundamental_data.update(data_dict)
|
fundamental_data.update(data_dict)
|
||||||
|
|
||||||
elif isinstance(parsed_data, list) and "income-statement/" in url:
|
|
||||||
# Handle list response, save as JSON object
|
|
||||||
fundamental_data['income'] = ujson.dumps(parsed_data)
|
|
||||||
data_dict = {'revenue': parsed_data[0]['revenue'],
|
|
||||||
'netIncome': parsed_data[0]['netIncome'],
|
|
||||||
'grossProfit': parsed_data[0]['grossProfit'],
|
|
||||||
'costOfRevenue':parsed_data[0]['costOfRevenue'],
|
|
||||||
'costAndExpenses':parsed_data[0]['costAndExpenses'],
|
|
||||||
'interestIncome':parsed_data[0]['interestIncome'],
|
|
||||||
'interestExpense':parsed_data[0]['interestExpense'],
|
|
||||||
'researchAndDevelopmentExpenses':parsed_data[0]['researchAndDevelopmentExpenses'],
|
|
||||||
'ebitda':parsed_data[0]['ebitda'],
|
|
||||||
'ebitdaratio':parsed_data[0]['ebitdaratio'],
|
|
||||||
'depreciationAndAmortization':parsed_data[0]['depreciationAndAmortization'],
|
|
||||||
'operatingIncome':parsed_data[0]['operatingIncome'],
|
|
||||||
'operatingExpenses':parsed_data[0]['operatingExpenses']
|
|
||||||
}
|
|
||||||
fundamental_data.update(data_dict)
|
|
||||||
elif isinstance(parsed_data, list) and "/v3/ratios/" in url:
|
|
||||||
fundamental_data['ratios'] = ujson.dumps(parsed_data)
|
|
||||||
data_dict = {'payoutRatio': parsed_data[0]['payoutRatio'],
|
|
||||||
'priceToBookRatio': parsed_data[0]['priceToBookRatio'],
|
|
||||||
'dividendPayoutRatio': parsed_data[0]['dividendPayoutRatio'],
|
|
||||||
'priceToSalesRatio':parsed_data[0]['priceToSalesRatio'],
|
|
||||||
'priceEarningsRatio':parsed_data[0]['priceEarningsRatio'],
|
|
||||||
'priceCashFlowRatio':parsed_data[0]['priceCashFlowRatio'],
|
|
||||||
'priceSalesRatio':parsed_data[0]['priceSalesRatio'],
|
|
||||||
'dividendYield':parsed_data[0]['dividendYield'],
|
|
||||||
'cashFlowToDebtRatio':parsed_data[0]['cashFlowToDebtRatio'],
|
|
||||||
'freeCashFlowPerShare':parsed_data[0]['freeCashFlowPerShare'],
|
|
||||||
'cashPerShare':parsed_data[0]['cashPerShare'],
|
|
||||||
}
|
|
||||||
fundamental_data.update(data_dict)
|
|
||||||
|
|
||||||
elif isinstance(parsed_data, list) and "balance-sheet-statement/" in url:
|
|
||||||
# Handle list response, save as JSON object
|
|
||||||
fundamental_data['balance'] = ujson.dumps(parsed_data)
|
|
||||||
elif isinstance(parsed_data, list) and "cash-flow-statement/" in url:
|
|
||||||
# Handle list response, save as JSON object
|
|
||||||
fundamental_data['cashflow'] = ujson.dumps(parsed_data)
|
|
||||||
|
|
||||||
elif isinstance(parsed_data, list) and "sector-benchmark" in url:
|
elif isinstance(parsed_data, list) and "sector-benchmark" in url:
|
||||||
# Handle list response, save as JSON object
|
# Handle list response, save as JSON object
|
||||||
fundamental_data['esg_sector_benchmark'] = ujson.dumps(parsed_data)
|
fundamental_data['esg_sector_benchmark'] = ujson.dumps(parsed_data)
|
||||||
elif isinstance(parsed_data, list) and "income-statement-growth/" in url:
|
|
||||||
# Handle list response, save as JSON object
|
|
||||||
fundamental_data['income_growth'] = ujson.dumps(parsed_data)
|
|
||||||
data_dict = {'growthRevenue': parsed_data[0]['growthRevenue']*100,
|
|
||||||
'growthNetIncome': parsed_data[0]['growthNetIncome']*100,
|
|
||||||
'growthGrossProfit': parsed_data[0]['growthGrossProfit']*100,
|
|
||||||
'growthCostOfRevenue':parsed_data[0]['growthCostOfRevenue']*100,
|
|
||||||
'growthCostAndExpenses':parsed_data[0]['growthCostAndExpenses']*100,
|
|
||||||
'growthInterestExpense':parsed_data[0]['growthInterestExpense']*100,
|
|
||||||
'growthResearchAndDevelopmentExpenses':parsed_data[0]['growthResearchAndDevelopmentExpenses']*100,
|
|
||||||
'growthEBITDA':parsed_data[0]['growthEBITDA']*100,
|
|
||||||
'growthEBITDARatio':parsed_data[0]['growthEBITDARatio']*100,
|
|
||||||
'growthDepreciationAndAmortization':parsed_data[0]['growthDepreciationAndAmortization']*100,
|
|
||||||
'growthEPS':parsed_data[0]['growthEPS']*100,
|
|
||||||
'growthOperatingIncome':parsed_data[0]['growthOperatingIncome']*100,
|
|
||||||
'growthOperatingExpenses':parsed_data[0]['growthOperatingExpenses']*100
|
|
||||||
}
|
|
||||||
|
|
||||||
fundamental_data.update(data_dict)
|
fundamental_data.update(data_dict)
|
||||||
elif isinstance(parsed_data, list) and "balance-sheet-statement-growth/" in url:
|
|
||||||
# Handle list response, save as JSON object
|
|
||||||
fundamental_data['balance_growth'] = ujson.dumps(parsed_data)
|
|
||||||
elif isinstance(parsed_data, list) and "cash-flow-statement-growth/" in url:
|
|
||||||
# Handle list response, save as JSON object
|
|
||||||
fundamental_data['cashflow_growth'] = ujson.dumps(parsed_data)
|
|
||||||
|
|
||||||
elif "stock_dividend" in url:
|
elif "stock_dividend" in url:
|
||||||
# Handle list response, save as JSON object
|
# Handle list response, save as JSON object
|
||||||
@ -291,9 +218,9 @@ class StockDatabase:
|
|||||||
ticker_type = stock.get('type', '')
|
ticker_type = stock.get('type', '')
|
||||||
if exchange_short_name in ['XETRA','NYSE', 'NASDAQ','AMEX', 'PNK','EURONEXT'] and ticker_type in ['stock']:
|
if exchange_short_name in ['XETRA','NYSE', 'NASDAQ','AMEX', 'PNK','EURONEXT'] and ticker_type in ['stock']:
|
||||||
symbol = stock.get('symbol', '')
|
symbol = stock.get('symbol', '')
|
||||||
if exchange_short_name == 'PNK' and symbol not in ['DRSHF','NTDOY','OTGLF','TCEHY', 'KRKNF','BYDDY','XIACY','NSRGY']:
|
if exchange_short_name == 'PNK' and symbol not in ['DRSHF','NTDOY','OTGLF','TCEHY', 'KRKNF','BYDDY','XIACY','NSRGY','TLPFY','TLPFF']:
|
||||||
pass
|
pass
|
||||||
elif exchange_short_name == 'EURONEXT' and symbol not in ['ALEUP.PA','ALNEV.PA','ALGAU.PA','ALDRV.PA','ALHYG.PA','ALVMG.PA']:
|
elif exchange_short_name == 'EURONEXT' and symbol not in ['ALEUP.PA','ALNEV.PA','ALGAU.PA','ALDRV.PA','ALHYG.PA','ALVMG.PA','TEP.PA']:
|
||||||
pass
|
pass
|
||||||
else:
|
else:
|
||||||
name = stock.get('name', '')
|
name = stock.get('name', '')
|
||||||
@ -343,7 +270,7 @@ class StockDatabase:
|
|||||||
tasks.append(self.save_fundamental_data(session, symbol))
|
tasks.append(self.save_fundamental_data(session, symbol))
|
||||||
|
|
||||||
i += 1
|
i += 1
|
||||||
if i % 60 == 0:
|
if i % 30 == 0:
|
||||||
await asyncio.gather(*tasks)
|
await asyncio.gather(*tasks)
|
||||||
tasks = []
|
tasks = []
|
||||||
print('sleeping mode: ', i)
|
print('sleeping mode: ', i)
|
||||||
@ -374,7 +301,7 @@ class StockDatabase:
|
|||||||
self._create_ticker_table(symbol) # Ensure the table exists
|
self._create_ticker_table(symbol) # Ensure the table exists
|
||||||
|
|
||||||
# Fetch OHLC data from the API
|
# Fetch OHLC data from the API
|
||||||
url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&to={end_date}&apikey={api_key}"
|
url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&apikey={api_key}"
|
||||||
async with session.get(url) as response:
|
async with session.get(url) as response:
|
||||||
data = await response.text()
|
data = await response.text()
|
||||||
|
|
||||||
|
|||||||
@ -1,9 +1,6 @@
|
|||||||
import ujson
|
import ujson
|
||||||
import asyncio
|
import asyncio
|
||||||
import aiohttp
|
|
||||||
import sqlite3
|
import sqlite3
|
||||||
from datetime import datetime
|
|
||||||
from rating import rating_model
|
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
from tqdm import tqdm
|
from tqdm import tqdm
|
||||||
|
|
||||||
@ -11,17 +8,6 @@ async def save_stockdeck(symbol, data):
|
|||||||
with open(f"json/stockdeck/{symbol}.json", 'w') as file:
|
with open(f"json/stockdeck/{symbol}.json", 'w') as file:
|
||||||
ujson.dump(data, file)
|
ujson.dump(data, file)
|
||||||
|
|
||||||
def clean_financial_data(self, list1, list2):
|
|
||||||
#combine income_statement with income_growth_statement
|
|
||||||
combined_list = []
|
|
||||||
for item1 in list1:
|
|
||||||
for item2 in list2:
|
|
||||||
if item1["date"] == item2["date"]:
|
|
||||||
combined_item = {**item1, **item2} # Combine the dictionaries
|
|
||||||
combined_list.append(combined_item)
|
|
||||||
break
|
|
||||||
return combined_list
|
|
||||||
|
|
||||||
query_template = """
|
query_template = """
|
||||||
SELECT
|
SELECT
|
||||||
profile, quote,
|
profile, quote,
|
||||||
|
|||||||
13
app/main.py
13
app/main.py
@ -328,19 +328,6 @@ def replace_nan_inf_with_none(obj):
|
|||||||
return obj
|
return obj
|
||||||
|
|
||||||
|
|
||||||
def clean_financial_data(list1, list2):
|
|
||||||
#combine income_statement with income_growth_statement
|
|
||||||
combined_list = []
|
|
||||||
for item1 in list1:
|
|
||||||
for item2 in list2:
|
|
||||||
if item1["date"] == item2["date"]:
|
|
||||||
combined_item = {**item1, **item2} # Combine the dictionaries
|
|
||||||
combined_list.append(combined_item)
|
|
||||||
break
|
|
||||||
return combined_list
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
@app.get("/")
|
@app.get("/")
|
||||||
async def hello_world(api_key: str = Security(get_api_key)):
|
async def hello_world(api_key: str = Security(get_api_key)):
|
||||||
return {"stocknear api"}
|
return {"stocknear api"}
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user