diff --git a/app/create_etf_db.py b/app/create_etf_db.py index f6adf75..310930a 100755 --- a/app/create_etf_db.py +++ b/app/create_etf_db.py @@ -255,12 +255,14 @@ class ETFDatabase: for key in fundamental_data } + ''' if len(json.loads(fundamental_data['holding'])) == 0: self.cursor.execute("DELETE FROM etfs WHERE symbol = ?", (symbol,)) #self.cursor.execute("DELETE FROM symbol WHERE symbol = ?", (symbol,)) self.conn.commit() print(f"Delete {symbol}") return + ''' for column, (column_type, value) in column_definitions.items(): if column not in columns and column_type: @@ -392,5 +394,10 @@ async def fetch_tickers(): db = ETFDatabase('backup_db/etf.db') loop = asyncio.get_event_loop() all_tickers = loop.run_until_complete(fetch_tickers()) +''' +for item in all_tickers: + if item['symbol'] == 'GLD': + print(item) +''' loop.run_until_complete(db.save_etfs(all_tickers)) db.close_connection() \ No newline at end of file diff --git a/app/cron_analyst_estimate.py b/app/cron_analyst_estimate.py index 4f1f6ea..b36978d 100644 --- a/app/cron_analyst_estimate.py +++ b/app/cron_analyst_estimate.py @@ -27,7 +27,6 @@ async def get_data(ticker, con): analyst_estimates = ujson.loads(data['analyst_estimates'].iloc[0]) income = ujson.loads(data['income'].iloc[0]) combined_data = defaultdict(dict) - #Sum up quarter results eps_sums = {} revenue_sums = {} diff --git a/app/cron_industry.py b/app/cron_industry.py index c6af514..7e908fa 100644 --- a/app/cron_industry.py +++ b/app/cron_industry.py @@ -5,8 +5,14 @@ import asyncio import pandas as pd from tqdm import tqdm import orjson +from GetStartEndDate import GetStartEndDate from collections import defaultdict +import os +from dotenv import load_dotenv +load_dotenv() +api_key = os.getenv('FMP_API_KEY') + with open(f"json/stock-screener/data.json", 'rb') as file: stock_screener_data = orjson.loads(file.read()) @@ -14,20 +20,30 @@ with open(f"json/stock-screener/data.json", 'rb') as file: stock_screener_data_dict = {item['symbol']: item for item in stock_screener_data} -def save_as_json(data): - with open(f"json/industry/overview.json", 'w') as file: +date, _ = GetStartEndDate().run() +date = date.strftime('%Y-%m-%d') + +def save_as_json(data, filename): + with open(f"json/industry/{filename}.json", 'w') as file: ujson.dump(data, file) -#async def get_data(): +# Function to fetch data from the API +async def get_data(session, class_type='sector'): + if class_type == 'sector': + url = f"https://financialmodelingprep.com/api/v4/sector_price_earning_ratio?date={date}&exchange=NYSE&apikey={api_key}" + else: + url = f"https://financialmodelingprep.com/api/v4/industry_price_earning_ratio?date={date}&exchange=NYSE&apikey={api_key}" + async with session.get(url) as response: + data = await response.json() + return data -def run(): +async def run(): # Initialize a dictionary to store stock count, market cap, and other totals for each industry sector_industry_data = defaultdict(lambda: defaultdict(lambda: { 'numStocks': 0, 'totalMarketCap': 0.0, - 'totalPE': 0.0, 'totalDividendYield': 0.0, 'totalNetIncome': 0.0, 'totalRevenue': 0.0, @@ -44,7 +60,6 @@ def run(): sector = stock.get('sector') industry = stock.get('industry') market_cap = stock.get('marketCap') - pe = stock.get('pe') dividend_yield = stock.get('dividendYield') net_income = stock.get('netIncome') revenue = stock.get('revenue') @@ -57,11 +72,6 @@ def run(): sector_industry_data[sector][industry]['numStocks'] += 1 sector_industry_data[sector][industry]['totalMarketCap'] += float(market_cap) - # Accumulate PE ratio if available - if pe is not None: - sector_industry_data[sector][industry]['totalPE'] += float(pe) - sector_industry_data[sector][industry]['peCount'] += 1 - # Accumulate dividend yield if available if dividend_yield is not None: sector_industry_data[sector][industry]['totalDividendYield'] += float(dividend_yield) @@ -83,19 +93,18 @@ def run(): sector_industry_data[sector][industry]['change1YCount'] += 1 # Prepare the final data in the requested format - result = {} + overview = {} for sector, industries in sector_industry_data.items(): # Sort industries by stock count in descending order sorted_industries = sorted(industries.items(), key=lambda x: x[1]['numStocks'], reverse=True) - # Add sorted industries with averages to the result for each sector - result[sector] = [ + # Add sorted industries with averages to the overview for each sector + overview[sector] = [ { 'industry': industry, 'numStocks': data['numStocks'], 'totalMarketCap': data['totalMarketCap'], - 'pe': round((data['totalMarketCap'] / data['totalNetIncome']),2) if data['totalNetIncome'] > 0 else None, 'avgDividendYield': round((data['totalDividendYield'] / data['dividendCount']),2) if data['dividendCount'] > 0 else None, 'profitMargin': round((data['totalNetIncome'] / data['totalRevenue'])*100,2) if data['totalRevenue'] > 0 else None, 'avgChange1M': round((data['totalChange1M'] / data['change1MCount']),2) if data['change1MCount'] > 0 else None, @@ -103,9 +112,87 @@ def run(): } for industry, data in sorted_industries ] - print(result) + # Assign the P/E values from pe_industry to the overview + async with aiohttp.ClientSession() as session: + pe_industry = await get_data(session, class_type='industry') + for sector, industries in overview.items(): + for industry_data in industries: + industry_name = industry_data['industry'] - save_as_json(result) + # Look for a matching industry in pe_industry to assign the P/E ratio + matching_pe = next((item['pe'] for item in pe_industry if item['industry'] == industry_name), None) + + if matching_pe is not None: + industry_data['pe'] = round(float(matching_pe), 2) + + save_as_json(overview, filename = 'overview') + + industry_overview = [] + + for key in overview: + industry_overview.extend(overview[key]) + + industry_overview = sorted(industry_overview, key= lambda x: x['numStocks'], reverse=True) + + save_as_json(industry_overview, filename='industry-overview') -run() \ No newline at end of file + sector_overview = [] + + for sector, industries in sector_industry_data.items(): + total_market_cap = 0 + total_stocks = 0 + total_dividend_yield = 0 + total_net_income = 0 + total_revenue = 0 + total_change_1m = 0 + total_change_1y = 0 + + dividend_count = 0 + change_1m_count = 0 + change_1y_count = 0 + + for industry, data in industries.items(): + # Sum up values across industries for the sector summary + total_market_cap += data['totalMarketCap'] + total_stocks += data['numStocks'] + total_net_income += data['totalNetIncome'] + total_revenue += data['totalRevenue'] + total_change_1m += data['totalChange1M'] + total_change_1y += data['totalChange1Y'] + + dividend_count += data['dividendCount'] + change_1m_count += data['change1MCount'] + change_1y_count += data['change1YCount'] + total_dividend_yield += data['totalDividendYield'] + + # Calculate averages and profit margin for the sector + sector_overview.append({ + 'sector': sector, + 'numStocks': total_stocks, + 'totalMarketCap': total_market_cap, + 'avgDividendYield': round((total_dividend_yield / dividend_count), 2) if dividend_count > 0 else None, + 'profitMargin': round((total_net_income / total_revenue) * 100, 2) if total_revenue > 0 else None, + 'avgChange1M': round((total_change_1m / change_1m_count), 2) if change_1m_count > 0 else None, + 'avgChange1Y': round((total_change_1y / change_1y_count), 2) if change_1y_count > 0 else None + }) + + + # Assign the P/E values from pe_industry to the overview + async with aiohttp.ClientSession() as session: + pe_sector = await get_data(session, class_type='sector') + # Loop through sector_overview to update P/E ratios from pe_sector + for sector_data in sector_overview: + sector_name = sector_data['sector'] + + # Find the matching sector in pe_sector and assign the P/E ratio + matching_pe = next((item['pe'] for item in pe_sector if item['sector'] == sector_name), None) + + if matching_pe is not None: + sector_data['pe'] = round(float(matching_pe), 2) + sector_overview = sorted(sector_overview, key= lambda x: x['numStocks'], reverse=True) + + save_as_json(sector_overview, filename='sector-overview') + +loop = asyncio.get_event_loop() +sector_results = loop.run_until_complete(run()) \ No newline at end of file diff --git a/app/cron_share_statistics.py b/app/cron_share_statistics.py index 79b2ed9..9f7d43f 100644 --- a/app/cron_share_statistics.py +++ b/app/cron_share_statistics.py @@ -82,11 +82,13 @@ async def get_data(ticker, con): async def run(): con = sqlite3.connect('stocks.db') - + etf_con = sqlite3.connect('etf.db') + cursor = con.cursor() cursor.execute("PRAGMA journal_mode = wal") cursor.execute("SELECT DISTINCT symbol FROM stocks") stock_symbols = [row[0] for row in cursor.fetchall()] + counter = 0 diff --git a/app/main.py b/app/main.py index 03bd1e6..2c22780 100755 --- a/app/main.py +++ b/app/main.py @@ -3607,9 +3607,9 @@ async def get_economic_indicator(api_key: str = Security(get_api_key)): headers={"Content-Encoding": "gzip"} ) -@app.get("/industry-overview") +@app.get("/sector-industry-overview") async def get_industry_overview(api_key: str = Security(get_api_key)): - cache_key = f"industry_overview" + cache_key = f"sector-industry-overview" cached_result = redis_client.get(cache_key) if cached_result: return StreamingResponse( @@ -3635,6 +3635,62 @@ async def get_industry_overview(api_key: str = Security(get_api_key)): headers={"Content-Encoding": "gzip"} ) +@app.get("/sector-overview") +async def get_sector_overview(api_key: str = Security(get_api_key)): + cache_key = f"sector-overview" + cached_result = redis_client.get(cache_key) + if cached_result: + return StreamingResponse( + io.BytesIO(cached_result), + media_type="application/json", + headers={"Content-Encoding": "gzip"} + ) + try: + with open(f"json/industry/sector-overview.json", 'rb') as file: + res = orjson.loads(file.read()) + except: + res = [] + + data = orjson.dumps(res) + compressed_data = gzip.compress(data) + + redis_client.set(cache_key, compressed_data) + redis_client.expire(cache_key,3600*3600) + + return StreamingResponse( + io.BytesIO(compressed_data), + media_type="application/json", + headers={"Content-Encoding": "gzip"} + ) + +@app.get("/industry-overview") +async def get_industry_overview(api_key: str = Security(get_api_key)): + cache_key = f"industry-overview" + cached_result = redis_client.get(cache_key) + if cached_result: + return StreamingResponse( + io.BytesIO(cached_result), + media_type="application/json", + headers={"Content-Encoding": "gzip"} + ) + try: + with open(f"json/industry/industry-overview.json", 'rb') as file: + res = orjson.loads(file.read()) + except: + res = [] + + data = orjson.dumps(res) + compressed_data = gzip.compress(data) + + redis_client.set(cache_key, compressed_data) + redis_client.expire(cache_key,3600*3600) + + return StreamingResponse( + io.BytesIO(compressed_data), + media_type="application/json", + headers={"Content-Encoding": "gzip"} + ) + @app.post("/next-earnings") async def get_next_earnings(data:TickerData, api_key: str = Security(get_api_key)): ticker = data.ticker