update cron list

app/cron_list.py

@@ -4,6 +4,11 @@ import asyncio
 import aiohttp
 import pandas as pd
 from tqdm import tqdm
+from dotenv import load_dotenv
+import os
+
+load_dotenv()
+api_key = os.getenv('FMP_API_KEY')
 
 
 # Load stock screener data
@@ -43,7 +48,7 @@ async def process_category(cursor, category, condition, category_type='market-ca
     cursor.execute(full_query)
     raw_data = cursor.fetchall()
     
-    result_list = []
+    res_list = []
     for row in raw_data:
         symbol = row[0]
         quote_data = await get_quote_data(symbol)
@@ -51,10 +56,9 @@ async def process_category(cursor, category, condition, category_type='market-ca
             item = {
                 'symbol': symbol,
                 'name': row[1],
-                'price': round(quote_data.get('price'),2),
+                'price': round(quote_data.get('price'), 2) if quote_data.get('price') is not None else None,
-                'changesPercentage': round(quote_data.get('changesPercentage'),2),
+                'changesPercentage': round(quote_data.get('changesPercentage'), 2) if quote_data.get('changesPercentage') is not None else None,
-                'marketCap': quote_data.get('marketCap'),
+                'marketCap': quote_data.get('marketCap', None),
-                'sector': row[4],  # Include sector information
                 'revenue': None,
             }
             
@@ -63,10 +67,10 @@ async def process_category(cursor, category, condition, category_type='market-ca
                 item['revenue'] = stock_screener_data_dict[symbol].get('revenue')
             
             if item['marketCap'] > 0:
-                result_list.append(item)
+                res_list.append(item)
     
     # Sort by market cap and save
-    sorted_result = sorted(result_list, key=lambda x: x['marketCap'] if x['marketCap'] else 0, reverse=True)
+    sorted_result = sorted(res_list, key=lambda x: x['marketCap'] if x['marketCap'] else 0, reverse=True)
     # Add rank to each item
     for rank, item in enumerate(sorted_result, 1):
         item['rank'] = rank
@@ -204,7 +208,7 @@ async def etf_bitcoin_list():
             etf_cursor.execute("SELECT DISTINCT symbol FROM etfs")
             etf_symbols = [row[0] for row in etf_cursor.fetchall()]
 
-            result_list = []
+            res_list = []
             query_template = """
                 SELECT 
                     symbol, name, expenseRatio, totalAssets
@@ -232,7 +236,7 @@ async def etf_bitcoin_list():
                         price = round(quote_data.get('price'), 2) if quote_data else None
                         changesPercentage = round(quote_data.get('changesPercentage'), 2) if quote_data else None
                         if total_assets > 0:
-                            result_list.append({
+                            res_list.append({
                                 'symbol': symbol,
                                 'name': name,
                                 'expenseRatio': expense_ratio,
@@ -243,13 +247,13 @@ async def etf_bitcoin_list():
                 except Exception as e:
                     print(f"Error processing symbol {symbol}: {e}")
             
-            if result_list:
+            if res_list:
-                result_list = sorted(result_list, key=lambda x: x['totalAssets'], reverse=True)
+                res_list = sorted(res_list, key=lambda x: x['totalAssets'], reverse=True)
-                for rank, item in enumerate(result_list, start=1):
+                for rank, item in enumerate(res_list, start=1):
                     item['rank'] = rank
                     
                 with open("json/etf-bitcoin-list/data.json", 'wb') as file:
-                    file.write(orjson.dumps(result_list))
+                    file.write(orjson.dumps(res_list))
 
     except Exception as e:
         print(f"Database error: {e}")
@@ -269,7 +273,7 @@ async def get_all_reits_list(cursor):
     cursor.execute(full_query)  # Assuming cursor is async
     raw_data = cursor.fetchall()
     
-    result_list = []
+    res_list = []
     for row in raw_data:
         symbol = row[0]
         
@@ -294,22 +298,78 @@ async def get_all_reits_list(cursor):
             
             # Append item if conditions are met
             if item['marketCap'] > 0 and item['dividendYield'] is not None:
-                result_list.append(item)
+                res_list.append(item)
     
-    if result_list:
+    if res_list:
-        result_list = sorted(result_list, key=lambda x: x['marketCap'] or 0, reverse=True)
+        res_list = sorted(res_list, key=lambda x: x['marketCap'] or 0, reverse=True)
         
         # Add rank to each item
-        for rank, item in enumerate(result_list, 1):
+        for rank, item in enumerate(res_list, 1):
             item['rank'] = rank
 
         with open("json/industry/list/reits.json", 'wb') as file:
-            file.write(orjson.dumps(result_list))
+            file.write(orjson.dumps(res_list))
+
+
+async def get_index_list():
+    with sqlite3.connect('stocks.db') as con:
+        cursor = con.cursor()
+        cursor.execute("PRAGMA journal_mode = wal")
+        cursor.execute("SELECT DISTINCT symbol FROM stocks")
+        symbols = [row[0] for row in cursor.fetchall()]
+
+    async with aiohttp.ClientSession() as session:
+
+        for index_list in ['nasdaq','dowjones','sp500']:
+            url = f"https://financialmodelingprep.com/api/v3/{index_list}_constituent?apikey={api_key}"
+            async with session.get(url) as response:
+                data = await response.json()
+                data = [{k: v for k, v in stock.items() if stock['symbol'] in symbols} for stock in data]
+                data = [entry for entry in data if entry]
+
+                res_list = []
+                for item in data:
+                    try:
+                        symbol = item['symbol']
+                        quote_data = await get_quote_data(symbol)
+
+                        if quote_data:
+                            item = {
+                                'symbol': symbol,
+                                'price': round(quote_data.get('price', 0), 2),
+                                'changesPercentage': round(quote_data.get('changesPercentage', 0), 2),
+                                'marketCap': quote_data.get('marketCap', 0),
+                                'revenue': None,
+                            }
+                            item['revenue'] = stock_screener_data_dict[symbol].get('revenue')
+
+                        if item['marketCap'] > 0:
+                            res_list.append(item)
+                    except Exception as e:
+                        print(e)
+
+            if res_list:
+                res_list = sorted(res_list, key=lambda x: x['marketCap'] or 0, reverse=True)
+                
+                # Add rank to each item
+                for rank, item in enumerate(res_list, 1):
+                    item['rank'] = rank
+
+                if index_list == 'nasdaq':
+                    extension = '100'
+                else:
+                    extension = ''
+                with open(f"json/stocks-list/list/{index_list+extension}.json", 'wb') as file:
+                    file.write(orjson.dumps(res_list))
 
 
 async def run():
-    await etf_bitcoin_list()
+    await asyncio.gather(
-    await get_magnificent_seven()
+        get_index_list(),
+        etf_bitcoin_list(),
+        get_magnificent_seven()
+    )
+
 
     """Main function to run the analysis for all categories"""
     market_cap_conditions = {
@@ -345,6 +405,11 @@ async def run():
         'jp': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' or exchangeShortName = 'AMEX') AND country = 'JP'",
     }
 
+    exchange_conditions = {
+        'nasdaq': "exchangeShortName = 'NASDAQ'",
+        'nyse': "exchangeShortName = 'NYSE'",
+        'amex': "exchangeShortName = 'AMEX'",
+    }
 
     try:
         con = sqlite3.connect('stocks.db')
@@ -359,22 +424,25 @@ async def run():
 
         await get_all_reits_list(cursor)
 
+        for category, condition in exchange_conditions.items():
+            await process_category(cursor, category, condition, 'stocks-list')
+            #await asyncio.sleep(1)  # Small delay between categories
+
         for category, condition in country_conditions.items():
             await process_category(cursor, category, condition, 'stocks-list')
-            await asyncio.sleep(1)  # Small delay between categories
+            #await asyncio.sleep(1)  # Small delay between categories
 
         for category, condition in market_cap_conditions.items():
             await process_category(cursor, category, condition, 'market-cap')
-            await asyncio.sleep(1)  # Small delay between categories
+            #await asyncio.sleep(1)  # Small delay between categories
 
         # Process sector categories
         for category, condition in sector_conditions.items():
             await process_category(cursor, category, condition, 'sector')
-            await asyncio.sleep(1)  # Small delay between categories
+            #await asyncio.sleep(1)  # Small delay between categories
         
         
         await get_etf_holding(etf_symbols, etf_con)
-        
         await get_etf_provider(etf_con)
 
 

app/main.py

@@ -1789,35 +1789,6 @@ async def etf_holdings(data: TickerData, api_key: str = Security(get_api_key)):
 
 
 
-@app.post("/exchange-constituents")
-async def top_ai_signals(data:FilterStockList, api_key: str = Security(get_api_key)):
-    data = data.dict()
-    filter_list = data['filterList']
-
-    cache_key = f"filter-list-{filter_list}"
-    cached_result = redis_client.get(cache_key)
-    if cached_result:
-        return orjson.loads(cached_result)
-
-    if filter_list == 'nasdaqConstituent':
-        path = f"nasdaq_constituent.json"
-    elif filter_list == 'dowjonesConstituent':
-        path = f"dowjones_constituent.json"
-    elif filter_list == 'sp500Constituent':
-        path = f"sp500_constituent.json"
-
-    try:
-        with open(f"json/stocks-list/{path}", 'rb') as file:
-            res = orjson.loads(file.read())
-    except:
-        res = []
-
-    redis_client.set(cache_key, orjson.dumps(res))
-    redis_client.expire(cache_key, 3600 * 24)  # Set cache expiration time to 1 day
-
-    return res
-
-
 @app.get("/all-stock-tickers")
 async def get_all_stock_tickers(api_key: str = Security(get_api_key)):
     cache_key = f"all_stock_tickers"
@@ -1957,71 +1928,6 @@ async def historical_sector_price(data:FilterStockList, api_key: str = Security(
     )
 
 
-@app.post("/filter-stock-list")
-async def filter_stock_list(data: FilterStockList, api_key: str = Security(get_api_key)):
-    data = data.dict()
-    filter_list = data['filterList']
-    cache_key = f"filter-list-{filter_list}"
-    cached_result = redis_client.get(cache_key)
-    
-    if cached_result:
-        return orjson.loads(cached_result)
-
-    cursor = con.cursor()
-    cursor.execute("PRAGMA journal_mode = wal")
-
-    base_query = """
-        SELECT symbol, name, price, changesPercentage, marketCap
-        FROM stocks 
-        WHERE (price IS NOT NULL OR changesPercentage IS NOT NULL) 
-        AND {}
-    """
-
-    conditions = {
-        'nasdaq': "exchangeShortName = 'NASDAQ'",
-        'nyse': "exchangeShortName = 'NYSE'",
-        'xetra': "exchangeShortName = 'XETRA'",
-        'amex': "exchangeShortName = 'AMEX'",
-    }
-
-    # Execute the query with the relevant country
-    if filter_list in conditions:
-        full_query = base_query.format(conditions[filter_list])
-        cursor.execute(full_query)
-
-    # Fetch the results
-    raw_data = cursor.fetchall()
-
-    res_list = [{
-            'symbol': symbol,
-            'name': name,
-            'price': price,
-            'changesPercentage': changesPercentage,
-            'marketCap': marketCap,
-            'revenue': None,    # Placeholder for revenue
-            'netIncome': None   # Placeholder for netIncome
-        } for (symbol, name, price, changesPercentage, marketCap) in raw_data]
-
-    # Create the dictionary keyed by symbol for revenue and netIncome
-    stock_screener_data_dict = {item['symbol']: item for item in stock_screener_data}
-    
-    # Update revenue and netIncome for each item in res_list
-    for item in res_list:
-        symbol = item['symbol']
-        if symbol in stock_screener_data_dict:
-            item['revenue'] = stock_screener_data_dict[symbol].get('revenue', None)
-            item['netIncome'] = stock_screener_data_dict[symbol].get('netIncome', None)
-    
-    
-    sorted_res_list = sorted(res_list, key=lambda x: x['marketCap'], reverse=True)
-
-    # Cache the result
-    redis_client.set(cache_key, orjson.dumps(sorted_res_list))
-    redis_client.expire(cache_key, 3600 * 24)  # Set cache expiration time to 1 day
-
-    return sorted_res_list
-
-
 
 def remove_text_before_operator(text):
     # Find the index of the first occurrence of "Operator"
@@ -4026,7 +3932,7 @@ async def get_statistics(data: FilterStockList, api_key: str = Security(get_api_
         category_type = 'sector'
     elif filter_list == 'reits':
         category_type = 'industry'
-    elif filter_list in ['ca','cn','de','gb','il','in','jp']:
+    elif filter_list in ['ca','cn','de','gb','il','in','jp','nyse','nasdaq','amex','dowjones','sp500','nasdaq100']:
         category_type = 'stocks-list'
     else:
         category_type = 'market-cap'

app/restart_json.py

@@ -1350,43 +1350,6 @@ async def get_economic_calendar():
     return filtered_data
 
 
-async def get_index_list(con,symbols, index_list):
-    
-    async with aiohttp.ClientSession() as session:
-
-        query_template = """
-            SELECT 
-                price, changesPercentage, marketCap
-            FROM 
-                stocks 
-            WHERE
-                symbol = ?
-        """
-        
-        url = f"https://financialmodelingprep.com/api/v3/{index_list}?apikey={api_key}"
-        async with session.get(url) as response:
-            data = await response.json()
-            filtered_data = [{k: v for k, v in stock.items() if stock['symbol'] in symbols} for stock in data]
-            filtered_data = [entry for entry in filtered_data if entry]
-
-            res_list = []
-            for entry in filtered_data:
-                symbol = entry['symbol']
-
-                query_data = pd.read_sql_query(query_template, con, params=(entry['symbol'],))
-
-                if query_data['marketCap'].iloc[0] != None and query_data['price'].iloc[0] != None and query_data['changesPercentage'].iloc[0] != None:
-                    entry['marketCap'] = int(query_data['marketCap'].iloc[0])
-                    entry['price'] = round(float(query_data['price'].iloc[0]),2)
-                    entry['changesPercentage'] = round(float(query_data['changesPercentage'].iloc[0]),2)
-                    res_list.append(entry)
-
-    sorted_res_list = sorted(res_list, key=lambda x: x['marketCap'], reverse=True)
-    return sorted_res_list
-
-
-
-
 def replace_representative(office):
     replacements = {
         'Carper, Thomas R. (Senator)': 'Tom Carper',
@@ -2021,18 +1984,7 @@ async def save_json_files():
     with open(f"json/stock-splits-calendar/calendar.json", 'w') as file:
         ujson.dump(stock_splits_data, file)
 
-    #Stocks Lists
-    data = await get_index_list(con,symbols,'nasdaq_constituent')
-    with open(f"json/stocks-list/nasdaq_constituent.json", 'w') as file:
-        ujson.dump(data, file)
     
-    data = await get_index_list(con,symbols,'dowjones_constituent')
-    with open(f"json/stocks-list/dowjones_constituent.json", 'w') as file:
-        ujson.dump(data, file)
-
-    data = await get_index_list(con,symbols,'sp500_constituent')
-    with open(f"json/stocks-list/sp500_constituent.json", 'w') as file:
-        ujson.dump(data, file)