refactor code

app/create_stock_db.py

@@ -273,7 +273,7 @@ class StockDatabase:
                 if i % 60 == 0:
                     await asyncio.gather(*tasks)
                     tasks = []
-                    print('sleeping mode: ', i)
+                    print('sleeping mode 30 seconds')
                     await asyncio.sleep(30)  # Pause for 60 seconds
 
             

app/main.py

@@ -2012,7 +2012,7 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap
     cursor.execute("PRAGMA journal_mode = wal")
 
     base_query = """
-        SELECT symbol, name, price, changesPercentage, marketCap, revenue, netIncome
+        SELECT symbol, name, price, changesPercentage, marketCap
         FROM stocks 
         WHERE (price IS NOT NULL OR changesPercentage IS NOT NULL) 
         AND {}
@@ -2064,16 +2064,25 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap
             'price': price,
             'changesPercentage': changesPercentage,
             'marketCap': marketCap,
-            'revenue': revenue,
+            'revenue': None,    # Placeholder for revenue
-            'netIncome': netIncome
+            'netIncome': None   # Placeholder for netIncome
-        } for (symbol, name, price, changesPercentage, marketCap, revenue, netIncome) in raw_data]
+        } for (symbol, name, price, changesPercentage, marketCap) in raw_data]
 
-    # Update res_list with dividendYield
+    # Create the dictionary keyed by symbol for revenue and netIncome
-    if filter_list == 'reit':
-        # Create the dictionary keyed by symbol
     stock_screener_data_dict = {item['symbol']: item for item in stock_screener_data}
     
-        # Update dividendYield for each item in res_list
+    # Update revenue and netIncome for each item in res_list
+    for item in res_list:
+        symbol = item['symbol']
+        if symbol in stock_screener_data_dict:
+            item['revenue'] = stock_screener_data_dict[symbol].get('revenue', None)
+            item['netIncome'] = stock_screener_data_dict[symbol].get('netIncome', None)
+    
+    # Optional: Filter or process the list further, e.g., for REITs as in your original code.
+    if filter_list == 'reit':
+        # No filtering based on dividendYield
+        # This includes all REITs in the list regardless of their dividendYield
+        # Simply check if the item is in the REIT condition
         for item in res_list:
             symbol = item['symbol']
             if symbol in stock_screener_data_dict:
@@ -2082,8 +2091,6 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap
         # Remove elements where dividendYield is None
         res_list = [item for item in res_list if item.get('dividendYield') is not None]
         
-
-
     sorted_res_list = sorted(res_list, key=lambda x: x['marketCap'], reverse=True)
 
     # Cache the result
@@ -2093,6 +2100,7 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap
     return sorted_res_list
 
 
+
 def remove_text_before_operator(text):
     # Find the index of the first occurrence of "Operator"
     operator_index = text.find("Operator")

app/restart_json.py

@@ -683,7 +683,7 @@ async def get_dividends_calendar(con,symbols):
         #Database read 1y and 3y data
         query_template = """
             SELECT 
-                name, marketCap, revenue
+                name, marketCap
             FROM 
                 stocks 
             WHERE
@@ -699,10 +699,15 @@ async def get_dividends_calendar(con,symbols):
             for entry in filtered_data:
                 try:
                     symbol = entry['symbol']
+                    try:
+                        with open(f"json/financial-statements/income-statement/annual/{symbol}.json", 'rb') as file:
+                            entry['revenue'] = orjson.loads(file.read())[0]['revenue']
+                    except:
+                        entry['revenue'] = None
+
                     data = pd.read_sql_query(query_template, con, params=(symbol,))
                     entry['name'] = data['name'].iloc[0]
                     entry['marketCap'] = int(data['marketCap'].iloc[0])
-                    entry['revenue'] = int(data['revenue'].iloc[0])
                 except:
                     entry['name'] = 'n/a'
                     entry['marketCap'] = None
@@ -734,7 +739,7 @@ async def get_earnings_calendar(con, symbols):
 
         query_template = """
             SELECT 
-                name,marketCap,revenue,eps
+                name,marketCap,eps
             FROM 
                 stocks 
             WHERE
@@ -749,10 +754,15 @@ async def get_earnings_calendar(con, symbols):
             for entry in filtered_data:
                 try:
                     symbol = entry['symbol']
+                    try:
+                        with open(f"json/financial-statements/income-statement/annual/{symbol}.json", 'rb') as file:
+                            entry['revenue'] = orjson.loads(file.read())[0]['revenue']
+                    except:
+                        entry['revenue'] = None
+
                     fundamental_data = pd.read_sql_query(query_template, con, params=(symbol,))
                     entry['name'] = fundamental_data['name'].iloc[0]
                     entry['marketCap'] = int(fundamental_data['marketCap'].iloc[0])
-                    entry['revenue'] = int(fundamental_data['revenue'].iloc[0])
                     entry['eps'] = float(fundamental_data['eps'].iloc[0])
                 except:
                     entry['marketCap'] = 'n/a'
@@ -810,7 +820,7 @@ async def get_stock_splits_calendar(con,symbols):
         #Database read 1y and 3y data
         query_template = """
             SELECT 
-                name, marketCap,eps, revenue, netIncome
+                name, marketCap,eps
             FROM 
                 stocks 
             WHERE
@@ -826,17 +836,14 @@ async def get_stock_splits_calendar(con,symbols):
             for entry in filtered_data:
                 try:
                     symbol = entry['symbol']
+
                     data = pd.read_sql_query(query_template, con, params=(symbol,))
                     entry['name'] = data['name'].iloc[0]
                     entry['marketCap'] = int(data['marketCap'].iloc[0])
-                    entry['revenue'] = int(data['revenue'].iloc[0])
-                    entry['netIncome'] = int(data['netIncome'].iloc[0])
                     entry['eps'] = float(data['eps'].iloc[0])
                 except:
                     entry['name'] = 'n/a'
                     entry['marketCap'] = None
-                    entry['revenue'] = None
-                    entry['netIncome'] = None
                     entry['eps'] = None
 
     filtered_data = [d for d in filtered_data if d['symbol'] in symbols]
@@ -1157,7 +1164,7 @@ async def get_index_list(con,symbols, index_list):
 
         query_template = """
             SELECT 
-                price, changesPercentage, marketCap, revenue, netIncome
+                price, changesPercentage, marketCap
             FROM 
                 stocks 
             WHERE
@@ -1172,12 +1179,12 @@ async def get_index_list(con,symbols, index_list):
 
             res_list = []
             for entry in filtered_data:
+                symbol = entry['symbol']
+
                 query_data = pd.read_sql_query(query_template, con, params=(entry['symbol'],))
 
-                if query_data['marketCap'].iloc[0] != None and query_data['revenue'].iloc[0] !=None and query_data['price'].iloc[0] != None and query_data['changesPercentage'].iloc[0] != None:
+                if query_data['marketCap'].iloc[0] != None and query_data['price'].iloc[0] != None and query_data['changesPercentage'].iloc[0] != None:
                     entry['marketCap'] = int(query_data['marketCap'].iloc[0])
-                    entry['revenue'] = int(query_data['revenue'].iloc[0])
-                    entry['netIncome'] = int(query_data['netIncome'].iloc[0])
                     entry['price'] = round(float(query_data['price'].iloc[0]),2)
                     entry['changesPercentage'] = round(float(query_data['changesPercentage'].iloc[0]),2)
                     res_list.append(entry)
@@ -1539,15 +1546,25 @@ async def get_magnificent_seven(con):
     
     query_template = """
         SELECT 
-            symbol, name, price, changesPercentage, revenue, netIncome, marketCap,pe
+            symbol, name, price, changesPercentage, marketCap,pe
         FROM 
             stocks 
         WHERE
             symbol = ?
     """
+    
     res_list = []
     for symbol in symbol_list:
         try:
+            try:
+                with open(f"json/financial-statements/income-statement/annual/{symbol}.json", 'rb') as file:
+                    json_data = orjson.loads(file.read())[0]
+                    revenue = json_data['revenue']
+                    netIncome = json_data['netIncome']
+            except:
+                revenue = None
+                netIncome = None
+
             data = pd.read_sql_query(query_template, con, params=(symbol,))
             
             name = data['name'].iloc[0]
@@ -1555,8 +1572,6 @@ async def get_magnificent_seven(con):
             price = round(float(data['price'].iloc[0]),2)
             changesPercentage = round(float(data['changesPercentage'].iloc[0]),2)
             marketCap = int(data['marketCap'].iloc[0])
-            revenue = int(data['revenue'].iloc[0])
-            netIncome = int(data['netIncome'].iloc[0])
             pe = round(float(data['pe'].iloc[0]),2)
 
             res_list.append({'symbol': symbol, 'name': name, 'price': price, \
@@ -1851,6 +1866,7 @@ async def save_json_files():
 
 
 
+    
     stock_screener_data = await get_stock_screener(con)
     with open(f"json/stock-screener/data.json", 'w') as file:
         ujson.dump(stock_screener_data, file)
@@ -1860,6 +1876,10 @@ async def save_json_files():
         with open(f"json/economic-calendar/calendar.json", 'w') as file:
             ujson.dump(economic_list, file)
 
+    dividends_list = await get_dividends_calendar(con,symbols)
+    with open(f"json/dividends-calendar/calendar.json", 'w') as file:
+        ujson.dump(dividends_list, file)
+
 
     earnings_list = await get_earnings_calendar(con,symbols)
     with open(f"json/earnings-calendar/calendar.json", 'w') as file:
@@ -1875,11 +1895,6 @@ async def save_json_files():
     with open(f"json/congress-trading/rss-feed/data.json", 'w') as file:
         ujson.dump(data, file)
     
-    
-    data = await get_magnificent_seven(con)
-    with open(f"json/magnificent-seven/data.json", 'w') as file:
-        ujson.dump(data, file)
-    
     data = await get_ipo_calendar(con, symbols)
     with open(f"json/ipo-calendar/data.json", 'w') as file:
         ujson.dump(data, file)
@@ -1910,9 +1925,6 @@ async def save_json_files():
     with open(f"json/delisted-companies/data.json", 'w') as file:
         ujson.dump(delisted_data, file)
 
-    dividends_list = await get_dividends_calendar(con,symbols)
-    with open(f"json/dividends-calendar/calendar.json", 'w') as file:
-        ujson.dump(dividends_list, file)
             
     stock_splits_data = await get_stock_splits_calendar(con,symbols)
     with open(f"json/stock-splits-calendar/calendar.json", 'w') as file:
@@ -1931,7 +1943,9 @@ async def save_json_files():
     with open(f"json/stocks-list/sp500_constituent.json", 'w') as file:
         ujson.dump(data, file)
     
-    
+    data = await get_magnificent_seven(con)
+    with open(f"json/magnificent-seven/data.json", 'w') as file:
+        ujson.dump(data, file)
 
     con.close()
     etf_con.close()