From b9b69f039177e9b1a1316bf97d42ae7585443378 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Tue, 12 Nov 2024 17:52:55 +0100 Subject: [PATCH] update cron list --- app/cron_list.py | 118 ++++++++++++++++++++++++++++++++++---------- app/main.py | 96 +---------------------------------- app/restart_json.py | 50 +------------------ 3 files changed, 95 insertions(+), 169 deletions(-) diff --git a/app/cron_list.py b/app/cron_list.py index 61c6ef7..bd2b520 100644 --- a/app/cron_list.py +++ b/app/cron_list.py @@ -4,6 +4,11 @@ import asyncio import aiohttp import pandas as pd from tqdm import tqdm +from dotenv import load_dotenv +import os + +load_dotenv() +api_key = os.getenv('FMP_API_KEY') # Load stock screener data @@ -43,7 +48,7 @@ async def process_category(cursor, category, condition, category_type='market-ca cursor.execute(full_query) raw_data = cursor.fetchall() - result_list = [] + res_list = [] for row in raw_data: symbol = row[0] quote_data = await get_quote_data(symbol) @@ -51,10 +56,9 @@ async def process_category(cursor, category, condition, category_type='market-ca item = { 'symbol': symbol, 'name': row[1], - 'price': round(quote_data.get('price'),2), - 'changesPercentage': round(quote_data.get('changesPercentage'),2), - 'marketCap': quote_data.get('marketCap'), - 'sector': row[4], # Include sector information + 'price': round(quote_data.get('price'), 2) if quote_data.get('price') is not None else None, + 'changesPercentage': round(quote_data.get('changesPercentage'), 2) if quote_data.get('changesPercentage') is not None else None, + 'marketCap': quote_data.get('marketCap', None), 'revenue': None, } @@ -63,10 +67,10 @@ async def process_category(cursor, category, condition, category_type='market-ca item['revenue'] = stock_screener_data_dict[symbol].get('revenue') if item['marketCap'] > 0: - result_list.append(item) + res_list.append(item) # Sort by market cap and save - sorted_result = sorted(result_list, key=lambda x: x['marketCap'] if x['marketCap'] else 0, reverse=True) + sorted_result = sorted(res_list, key=lambda x: x['marketCap'] if x['marketCap'] else 0, reverse=True) # Add rank to each item for rank, item in enumerate(sorted_result, 1): item['rank'] = rank @@ -204,7 +208,7 @@ async def etf_bitcoin_list(): etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") etf_symbols = [row[0] for row in etf_cursor.fetchall()] - result_list = [] + res_list = [] query_template = """ SELECT symbol, name, expenseRatio, totalAssets @@ -232,7 +236,7 @@ async def etf_bitcoin_list(): price = round(quote_data.get('price'), 2) if quote_data else None changesPercentage = round(quote_data.get('changesPercentage'), 2) if quote_data else None if total_assets > 0: - result_list.append({ + res_list.append({ 'symbol': symbol, 'name': name, 'expenseRatio': expense_ratio, @@ -243,13 +247,13 @@ async def etf_bitcoin_list(): except Exception as e: print(f"Error processing symbol {symbol}: {e}") - if result_list: - result_list = sorted(result_list, key=lambda x: x['totalAssets'], reverse=True) - for rank, item in enumerate(result_list, start=1): + if res_list: + res_list = sorted(res_list, key=lambda x: x['totalAssets'], reverse=True) + for rank, item in enumerate(res_list, start=1): item['rank'] = rank with open("json/etf-bitcoin-list/data.json", 'wb') as file: - file.write(orjson.dumps(result_list)) + file.write(orjson.dumps(res_list)) except Exception as e: print(f"Database error: {e}") @@ -269,7 +273,7 @@ async def get_all_reits_list(cursor): cursor.execute(full_query) # Assuming cursor is async raw_data = cursor.fetchall() - result_list = [] + res_list = [] for row in raw_data: symbol = row[0] @@ -294,22 +298,78 @@ async def get_all_reits_list(cursor): # Append item if conditions are met if item['marketCap'] > 0 and item['dividendYield'] is not None: - result_list.append(item) + res_list.append(item) - if result_list: - result_list = sorted(result_list, key=lambda x: x['marketCap'] or 0, reverse=True) + if res_list: + res_list = sorted(res_list, key=lambda x: x['marketCap'] or 0, reverse=True) # Add rank to each item - for rank, item in enumerate(result_list, 1): + for rank, item in enumerate(res_list, 1): item['rank'] = rank with open("json/industry/list/reits.json", 'wb') as file: - file.write(orjson.dumps(result_list)) + file.write(orjson.dumps(res_list)) + + +async def get_index_list(): + with sqlite3.connect('stocks.db') as con: + cursor = con.cursor() + cursor.execute("PRAGMA journal_mode = wal") + cursor.execute("SELECT DISTINCT symbol FROM stocks") + symbols = [row[0] for row in cursor.fetchall()] + + async with aiohttp.ClientSession() as session: + + for index_list in ['nasdaq','dowjones','sp500']: + url = f"https://financialmodelingprep.com/api/v3/{index_list}_constituent?apikey={api_key}" + async with session.get(url) as response: + data = await response.json() + data = [{k: v for k, v in stock.items() if stock['symbol'] in symbols} for stock in data] + data = [entry for entry in data if entry] + + res_list = [] + for item in data: + try: + symbol = item['symbol'] + quote_data = await get_quote_data(symbol) + + if quote_data: + item = { + 'symbol': symbol, + 'price': round(quote_data.get('price', 0), 2), + 'changesPercentage': round(quote_data.get('changesPercentage', 0), 2), + 'marketCap': quote_data.get('marketCap', 0), + 'revenue': None, + } + item['revenue'] = stock_screener_data_dict[symbol].get('revenue') + + if item['marketCap'] > 0: + res_list.append(item) + except Exception as e: + print(e) + + if res_list: + res_list = sorted(res_list, key=lambda x: x['marketCap'] or 0, reverse=True) + + # Add rank to each item + for rank, item in enumerate(res_list, 1): + item['rank'] = rank + + if index_list == 'nasdaq': + extension = '100' + else: + extension = '' + with open(f"json/stocks-list/list/{index_list+extension}.json", 'wb') as file: + file.write(orjson.dumps(res_list)) async def run(): - await etf_bitcoin_list() - await get_magnificent_seven() + await asyncio.gather( + get_index_list(), + etf_bitcoin_list(), + get_magnificent_seven() + ) + """Main function to run the analysis for all categories""" market_cap_conditions = { @@ -345,6 +405,11 @@ async def run(): 'jp': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' or exchangeShortName = 'AMEX') AND country = 'JP'", } + exchange_conditions = { + 'nasdaq': "exchangeShortName = 'NASDAQ'", + 'nyse': "exchangeShortName = 'NYSE'", + 'amex': "exchangeShortName = 'AMEX'", + } try: con = sqlite3.connect('stocks.db') @@ -359,22 +424,25 @@ async def run(): await get_all_reits_list(cursor) + for category, condition in exchange_conditions.items(): + await process_category(cursor, category, condition, 'stocks-list') + #await asyncio.sleep(1) # Small delay between categories + for category, condition in country_conditions.items(): await process_category(cursor, category, condition, 'stocks-list') - await asyncio.sleep(1) # Small delay between categories + #await asyncio.sleep(1) # Small delay between categories for category, condition in market_cap_conditions.items(): await process_category(cursor, category, condition, 'market-cap') - await asyncio.sleep(1) # Small delay between categories + #await asyncio.sleep(1) # Small delay between categories # Process sector categories for category, condition in sector_conditions.items(): await process_category(cursor, category, condition, 'sector') - await asyncio.sleep(1) # Small delay between categories + #await asyncio.sleep(1) # Small delay between categories await get_etf_holding(etf_symbols, etf_con) - await get_etf_provider(etf_con) diff --git a/app/main.py b/app/main.py index 0541113..8c19c85 100755 --- a/app/main.py +++ b/app/main.py @@ -1789,35 +1789,6 @@ async def etf_holdings(data: TickerData, api_key: str = Security(get_api_key)): -@app.post("/exchange-constituents") -async def top_ai_signals(data:FilterStockList, api_key: str = Security(get_api_key)): - data = data.dict() - filter_list = data['filterList'] - - cache_key = f"filter-list-{filter_list}" - cached_result = redis_client.get(cache_key) - if cached_result: - return orjson.loads(cached_result) - - if filter_list == 'nasdaqConstituent': - path = f"nasdaq_constituent.json" - elif filter_list == 'dowjonesConstituent': - path = f"dowjones_constituent.json" - elif filter_list == 'sp500Constituent': - path = f"sp500_constituent.json" - - try: - with open(f"json/stocks-list/{path}", 'rb') as file: - res = orjson.loads(file.read()) - except: - res = [] - - redis_client.set(cache_key, orjson.dumps(res)) - redis_client.expire(cache_key, 3600 * 24) # Set cache expiration time to 1 day - - return res - - @app.get("/all-stock-tickers") async def get_all_stock_tickers(api_key: str = Security(get_api_key)): cache_key = f"all_stock_tickers" @@ -1957,71 +1928,6 @@ async def historical_sector_price(data:FilterStockList, api_key: str = Security( ) -@app.post("/filter-stock-list") -async def filter_stock_list(data: FilterStockList, api_key: str = Security(get_api_key)): - data = data.dict() - filter_list = data['filterList'] - cache_key = f"filter-list-{filter_list}" - cached_result = redis_client.get(cache_key) - - if cached_result: - return orjson.loads(cached_result) - - cursor = con.cursor() - cursor.execute("PRAGMA journal_mode = wal") - - base_query = """ - SELECT symbol, name, price, changesPercentage, marketCap - FROM stocks - WHERE (price IS NOT NULL OR changesPercentage IS NOT NULL) - AND {} - """ - - conditions = { - 'nasdaq': "exchangeShortName = 'NASDAQ'", - 'nyse': "exchangeShortName = 'NYSE'", - 'xetra': "exchangeShortName = 'XETRA'", - 'amex': "exchangeShortName = 'AMEX'", - } - - # Execute the query with the relevant country - if filter_list in conditions: - full_query = base_query.format(conditions[filter_list]) - cursor.execute(full_query) - - # Fetch the results - raw_data = cursor.fetchall() - - res_list = [{ - 'symbol': symbol, - 'name': name, - 'price': price, - 'changesPercentage': changesPercentage, - 'marketCap': marketCap, - 'revenue': None, # Placeholder for revenue - 'netIncome': None # Placeholder for netIncome - } for (symbol, name, price, changesPercentage, marketCap) in raw_data] - - # Create the dictionary keyed by symbol for revenue and netIncome - stock_screener_data_dict = {item['symbol']: item for item in stock_screener_data} - - # Update revenue and netIncome for each item in res_list - for item in res_list: - symbol = item['symbol'] - if symbol in stock_screener_data_dict: - item['revenue'] = stock_screener_data_dict[symbol].get('revenue', None) - item['netIncome'] = stock_screener_data_dict[symbol].get('netIncome', None) - - - sorted_res_list = sorted(res_list, key=lambda x: x['marketCap'], reverse=True) - - # Cache the result - redis_client.set(cache_key, orjson.dumps(sorted_res_list)) - redis_client.expire(cache_key, 3600 * 24) # Set cache expiration time to 1 day - - return sorted_res_list - - def remove_text_before_operator(text): # Find the index of the first occurrence of "Operator" @@ -4026,7 +3932,7 @@ async def get_statistics(data: FilterStockList, api_key: str = Security(get_api_ category_type = 'sector' elif filter_list == 'reits': category_type = 'industry' - elif filter_list in ['ca','cn','de','gb','il','in','jp']: + elif filter_list in ['ca','cn','de','gb','il','in','jp','nyse','nasdaq','amex','dowjones','sp500','nasdaq100']: category_type = 'stocks-list' else: category_type = 'market-cap' diff --git a/app/restart_json.py b/app/restart_json.py index b4f66b8..54851b0 100755 --- a/app/restart_json.py +++ b/app/restart_json.py @@ -1350,43 +1350,6 @@ async def get_economic_calendar(): return filtered_data -async def get_index_list(con,symbols, index_list): - - async with aiohttp.ClientSession() as session: - - query_template = """ - SELECT - price, changesPercentage, marketCap - FROM - stocks - WHERE - symbol = ? - """ - - url = f"https://financialmodelingprep.com/api/v3/{index_list}?apikey={api_key}" - async with session.get(url) as response: - data = await response.json() - filtered_data = [{k: v for k, v in stock.items() if stock['symbol'] in symbols} for stock in data] - filtered_data = [entry for entry in filtered_data if entry] - - res_list = [] - for entry in filtered_data: - symbol = entry['symbol'] - - query_data = pd.read_sql_query(query_template, con, params=(entry['symbol'],)) - - if query_data['marketCap'].iloc[0] != None and query_data['price'].iloc[0] != None and query_data['changesPercentage'].iloc[0] != None: - entry['marketCap'] = int(query_data['marketCap'].iloc[0]) - entry['price'] = round(float(query_data['price'].iloc[0]),2) - entry['changesPercentage'] = round(float(query_data['changesPercentage'].iloc[0]),2) - res_list.append(entry) - - sorted_res_list = sorted(res_list, key=lambda x: x['marketCap'], reverse=True) - return sorted_res_list - - - - def replace_representative(office): replacements = { 'Carper, Thomas R. (Senator)': 'Tom Carper', @@ -2021,18 +1984,7 @@ async def save_json_files(): with open(f"json/stock-splits-calendar/calendar.json", 'w') as file: ujson.dump(stock_splits_data, file) - #Stocks Lists - data = await get_index_list(con,symbols,'nasdaq_constituent') - with open(f"json/stocks-list/nasdaq_constituent.json", 'w') as file: - ujson.dump(data, file) - - data = await get_index_list(con,symbols,'dowjones_constituent') - with open(f"json/stocks-list/dowjones_constituent.json", 'w') as file: - ujson.dump(data, file) - - data = await get_index_list(con,symbols,'sp500_constituent') - with open(f"json/stocks-list/sp500_constituent.json", 'w') as file: - ujson.dump(data, file) +