diff --git a/app/cron_list.py b/app/cron_list.py index 228e309..f5a36cf 100644 --- a/app/cron_list.py +++ b/app/cron_list.py @@ -1087,9 +1087,61 @@ async def get_all_stock_tickers(): except Exception as e: print(f"Database error: {e}") +async def get_all_etf_tickers(): + try: + ''' + with sqlite3.connect('etf.db') as etf_con: + etf_cursor = etf_con.cursor() + etf_cursor.execute("PRAGMA journal_mode = wal") + etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") + etf_symbols = [row[0] for row in etf_cursor.fetchall()] + ''' + with sqlite3.connect('etf.db') as con: + cursor = con.cursor() + cursor.execute("PRAGMA journal_mode = wal") + cursor.execute("SELECT DISTINCT symbol FROM etfs") + etf_symbols = [row[0] for row in cursor.fetchall()] + + res_list = [] + for symbol in etf_symbols: + try: + + try: + with open(f"json/quote/{symbol}.json", "rb") as file: + quote_data = orjson.loads(file.read()) + except (FileNotFoundError, orjson.JSONDecodeError): + quote_data = None + + if quote_data: + item = { + 'symbol': symbol, + 'name': quote_data.get('name',None), + 'price': round(quote_data.get('price'), 2) if quote_data.get('price') is not None else None, + 'changesPercentage': round(quote_data.get('changesPercentage'), 2) if quote_data.get('changesPercentage') is not None else None, + 'marketCap': quote_data.get('marketCap', None), + } + + if item['marketCap'] > 0: + res_list.append(item) + + + except Exception as e: + print(f"Error processing symbol {symbol}: {e}") + + if res_list: + res_list = sorted(res_list, key=lambda x: x['symbol'], reverse=False) + + with open("json/stocks-list/list/all-etf-tickers.json", 'wb') as file: + file.write(orjson.dumps(res_list)) + + except Exception as e: + print(f"Database error: {e}") + + async def run(): await asyncio.gather( get_all_stock_tickers(), + get_all_etf_tickers(), get_index_list(), etf_bitcoin_list(), get_magnificent_seven(), diff --git a/app/cron_one_day_price.py b/app/cron_one_day_price.py index a80d75c..6d187ef 100755 --- a/app/cron_one_day_price.py +++ b/app/cron_one_day_price.py @@ -7,13 +7,11 @@ import pandas as pd from GetStartEndDate import GetStartEndDate from dotenv import load_dotenv import os -from utils.helper import check_market_hours load_dotenv() api_key = os.getenv('FMP_API_KEY') - async def save_price_data(symbol, data): with open(f"json/one-day-price/{symbol}.json", 'w') as file: ujson.dump(data, file) @@ -27,7 +25,9 @@ async def fetch_and_save_symbols_data(symbols): responses = await asyncio.gather(*tasks) for symbol, response in zip(symbols, responses): - await save_price_data(symbol, response) + if len(response) > 0: + print(response[0]) + await save_price_data(symbol, response) async def get_todays_data(ticker): @@ -109,19 +109,14 @@ async def run(): total_symbols = stocks_symbols + etf_symbols total_symbols = sorted(total_symbols, key=lambda x: '.' in x) - - - market_open = check_market_hours() - if market_open: - chunk_size = 1000 - for i in range(0, len(total_symbols), chunk_size): - symbols_chunk = total_symbols[i:i+chunk_size] - await fetch_and_save_symbols_data(symbols_chunk) - print('sleeping for 45 sec') - await asyncio.sleep(45) # Wait for 60 seconds between chunks - else: - print('Market Closed') + chunk_size = 1000 + for i in range(0, len(total_symbols), chunk_size): + symbols_chunk = total_symbols[i:i+chunk_size] + await fetch_and_save_symbols_data(symbols_chunk) + print('sleeping...') + await asyncio.sleep(60) # Wait for 60 seconds between chunks + try: asyncio.run(run()) diff --git a/app/main.py b/app/main.py index f4d7a44..b002848 100755 --- a/app/main.py +++ b/app/main.py @@ -4158,7 +4158,7 @@ async def get_statistics(data: FilterStockList, api_key: str = Security(get_api_ category_type = 'sector' elif filter_list == 'reits': category_type = 'industry' - elif filter_list in ['highest-option-premium','highest-option-iv-rank','highest-open-interest','highest-open-interest-change','most-shorted-stocks','most-ftd-shares','highest-income-tax','most-employees','highest-revenue','top-rated-dividend-stocks','penny-stocks','overbought-stocks','oversold-stocks','faang','magnificent-seven','ca','cn','de','gb','il','in','jp','nyse','nasdaq','amex','dowjones','sp500','nasdaq100','all-stock-tickers']: + elif filter_list in ['highest-option-premium','highest-option-iv-rank','highest-open-interest','highest-open-interest-change','most-shorted-stocks','most-ftd-shares','highest-income-tax','most-employees','highest-revenue','top-rated-dividend-stocks','penny-stocks','overbought-stocks','oversold-stocks','faang','magnificent-seven','ca','cn','de','gb','il','in','jp','nyse','nasdaq','amex','dowjones','sp500','nasdaq100','all-etf-tickers','all-stock-tickers']: category_type = 'stocks-list' elif filter_list in ['dividend-kings','dividend-aristocrats']: category_type = 'dividends' diff --git a/app/primary_cron_job.py b/app/primary_cron_job.py index 0e6390b..5e1441b 100755 --- a/app/primary_cron_job.py +++ b/app/primary_cron_job.py @@ -222,8 +222,12 @@ def run_historical_price(): run_command(["python3", "cron_historical_price.py"]) def run_one_day_price(): + from utils.helper import check_market_hours + week = datetime.today().weekday() - if week <= 4: + market_open = check_market_hours() + + if week <= 4 and market_open: run_command(["python3", "cron_one_day_price.py"]) def run_sec_filings(): @@ -406,7 +410,7 @@ schedule.every(30).minutes.do(run_threaded, run_cron_market_news).tag('market_ne schedule.every(30).minutes.do(run_threaded, run_cron_industry).tag('industry_job') -schedule.every(7).minutes.do(run_threaded, run_one_day_price).tag('one_day_price_job') +schedule.every(8).minutes.do(run_threaded, run_one_day_price).tag('one_day_price_job') #schedule.every(15).minutes.do(run_threaded, run_cron_heatmap).tag('heatmap_job') diff --git a/app/utils/helper.py b/app/utils/helper.py index f0d9f33..a9b163e 100644 --- a/app/utils/helper.py +++ b/app/utils/helper.py @@ -23,7 +23,7 @@ def check_market_hours(): # Determine the market status if is_weekend or is_holiday: return False #"Market is closed." - elif 9 <= current_hour < 17 or (current_hour == 17 and current_minute == 0): + elif (current_hour == 16 and current_minute == 10) or 9 <= current_hour < 16: return True #"Market hours." else: return False #"Market is closed."