update

app/cron_list.py

@@ -1087,9 +1087,61 @@ async def get_all_stock_tickers():
     except Exception as e:
         print(f"Database error: {e}")
 
+async def get_all_etf_tickers():
+    try:
+        '''
+        with sqlite3.connect('etf.db') as etf_con:
+            etf_cursor = etf_con.cursor()
+            etf_cursor.execute("PRAGMA journal_mode = wal")
+            etf_cursor.execute("SELECT DISTINCT symbol FROM etfs")
+            etf_symbols = [row[0] for row in etf_cursor.fetchall()]
+        '''
+        with sqlite3.connect('etf.db') as con:
+            cursor = con.cursor()
+            cursor.execute("PRAGMA journal_mode = wal")
+            cursor.execute("SELECT DISTINCT symbol FROM etfs")
+            etf_symbols = [row[0] for row in cursor.fetchall()]
+
+        res_list = []
+        for symbol in etf_symbols:
+            try:
+                
+                try:
+                    with open(f"json/quote/{symbol}.json", "rb") as file:
+                        quote_data = orjson.loads(file.read())
+                except (FileNotFoundError, orjson.JSONDecodeError):
+                    quote_data = None
+
+                if quote_data:
+                    item = {
+                        'symbol': symbol,
+                        'name': quote_data.get('name',None),
+                        'price': round(quote_data.get('price'), 2) if quote_data.get('price') is not None else None,
+                        'changesPercentage': round(quote_data.get('changesPercentage'), 2) if quote_data.get('changesPercentage') is not None else None,
+                        'marketCap': quote_data.get('marketCap', None),
+                    }
+                    
+                    if item['marketCap'] > 0:
+                        res_list.append(item)
+
+            
+            except Exception as e:
+                print(f"Error processing symbol {symbol}: {e}")
+            
+        if res_list:
+            res_list = sorted(res_list, key=lambda x: x['symbol'], reverse=False)
+                
+            with open("json/stocks-list/list/all-etf-tickers.json", 'wb') as file:
+                file.write(orjson.dumps(res_list))
+
+    except Exception as e:
+        print(f"Database error: {e}")
+
+
 async def run():
     await asyncio.gather(
         get_all_stock_tickers(),
+        get_all_etf_tickers(),
         get_index_list(),
         etf_bitcoin_list(),
         get_magnificent_seven(),

app/cron_one_day_price.py

@@ -7,13 +7,11 @@ import pandas as pd
 from GetStartEndDate import GetStartEndDate
 from dotenv import load_dotenv
 import os
-from utils.helper import check_market_hours
 
 load_dotenv()
 api_key = os.getenv('FMP_API_KEY')
 
 
-
 async def save_price_data(symbol, data):
     with open(f"json/one-day-price/{symbol}.json", 'w') as file:
         ujson.dump(data, file)
@@ -27,7 +25,9 @@ async def fetch_and_save_symbols_data(symbols):
     responses = await asyncio.gather(*tasks)
     
     for symbol, response in zip(symbols, responses):
-        await save_price_data(symbol, response)
+        if len(response) > 0:
+            print(response[0])
+            await save_price_data(symbol, response)
 
 async def get_todays_data(ticker):
 
@@ -110,18 +110,13 @@ async def run():
     total_symbols = stocks_symbols + etf_symbols
     total_symbols = sorted(total_symbols, key=lambda x: '.' in x)
     
+    chunk_size = 1000
+    for i in range(0, len(total_symbols), chunk_size):
+        symbols_chunk = total_symbols[i:i+chunk_size]
+        await fetch_and_save_symbols_data(symbols_chunk)
+        print('sleeping...')
+        await asyncio.sleep(60)  # Wait for 60 seconds between chunks
 
-    market_open = check_market_hours()
-    
-    if market_open:
-        chunk_size = 1000
-        for i in range(0, len(total_symbols), chunk_size):
-            symbols_chunk = total_symbols[i:i+chunk_size]
-            await fetch_and_save_symbols_data(symbols_chunk)
-            print('sleeping for 45 sec')
-            await asyncio.sleep(45)  # Wait for 60 seconds between chunks
-    else:
-        print('Market Closed')
 
 try:
     asyncio.run(run())

app/main.py

@@ -4158,7 +4158,7 @@ async def get_statistics(data: FilterStockList, api_key: str = Security(get_api_
         category_type = 'sector'
     elif filter_list == 'reits':
         category_type = 'industry'
-    elif filter_list in ['highest-option-premium','highest-option-iv-rank','highest-open-interest','highest-open-interest-change','most-shorted-stocks','most-ftd-shares','highest-income-tax','most-employees','highest-revenue','top-rated-dividend-stocks','penny-stocks','overbought-stocks','oversold-stocks','faang','magnificent-seven','ca','cn','de','gb','il','in','jp','nyse','nasdaq','amex','dowjones','sp500','nasdaq100','all-stock-tickers']:
+    elif filter_list in ['highest-option-premium','highest-option-iv-rank','highest-open-interest','highest-open-interest-change','most-shorted-stocks','most-ftd-shares','highest-income-tax','most-employees','highest-revenue','top-rated-dividend-stocks','penny-stocks','overbought-stocks','oversold-stocks','faang','magnificent-seven','ca','cn','de','gb','il','in','jp','nyse','nasdaq','amex','dowjones','sp500','nasdaq100','all-etf-tickers','all-stock-tickers']:
         category_type = 'stocks-list'
     elif filter_list in ['dividend-kings','dividend-aristocrats']:
         category_type = 'dividends'

app/primary_cron_job.py

@@ -222,8 +222,12 @@ def run_historical_price():
         run_command(["python3", "cron_historical_price.py"])
 
 def run_one_day_price():
+    from utils.helper import check_market_hours
+
     week = datetime.today().weekday()
-    if week <= 4:
+    market_open = check_market_hours()
+
+    if week <= 4 and market_open:
         run_command(["python3", "cron_one_day_price.py"])
 
 def run_sec_filings():
@@ -406,7 +410,7 @@ schedule.every(30).minutes.do(run_threaded, run_cron_market_news).tag('market_ne
 
 schedule.every(30).minutes.do(run_threaded, run_cron_industry).tag('industry_job')
 
-schedule.every(7).minutes.do(run_threaded, run_one_day_price).tag('one_day_price_job')
+schedule.every(8).minutes.do(run_threaded, run_one_day_price).tag('one_day_price_job')
 #schedule.every(15).minutes.do(run_threaded, run_cron_heatmap).tag('heatmap_job')
 
 

app/utils/helper.py

@@ -23,7 +23,7 @@ def check_market_hours():
     # Determine the market status
     if is_weekend or is_holiday:
         return False #"Market is closed."
-    elif 9 <= current_hour < 17 or (current_hour == 17 and current_minute == 0):
+    elif (current_hour == 16 and current_minute == 10) or 9 <= current_hour < 16:
         return True #"Market hours."
     else:
         return False #"Market is closed."