bugfixing

app/cron_list.py

@@ -80,8 +80,18 @@ async def process_category(cursor, category, condition, category_type='market-ca
     return sorted_result
 
 
-async def get_etf_holding(etf_symbols, etf_con):
+
-    for ticker in ['SPY']: #tqdm(etf_symbols):
+async def get_etf_holding():
+    # Create a connection to the ETF database
+    etf_con = sqlite3.connect('etf.db')
+    etf_cursor = etf_con.cursor()
+    etf_cursor.execute("PRAGMA journal_mode = wal")
+    
+    # Fetch distinct ETF symbols
+    etf_cursor.execute("SELECT DISTINCT symbol FROM etfs")
+    etf_symbols = [row[0] for row in etf_cursor.fetchall()]
+    
+    for ticker in etf_symbols:
         res = []
         df = pd.read_sql_query(query_etf_holding, etf_con, params=(ticker,))
         try:
@@ -115,7 +125,6 @@ async def get_etf_holding(etf_symbols, etf_con):
                 except:
                     pass
                 
-                
                 # Assign price and changesPercentage if available, otherwise set to None
                 item['weightPercentage'] = round(item.get('weightPercentage'), 2) if item['weightPercentage'] else None
 
@@ -129,22 +138,26 @@ async def get_etf_holding(etf_symbols, etf_con):
             with open(f"json/etf/holding/{ticker}.json", 'wb') as file:
                 final_res = {'lastUpdate': last_update, 'holdings': res}
                 file.write(orjson.dumps(final_res))
+    
+    # Close the database connection
+    etf_con.close()
 
 
 
-async def get_etf_provider(etf_con):
+async def get_etf_provider():
-
+    # Create a connection to the ETF database
+    etf_con = sqlite3.connect('etf.db')
     cursor = etf_con.cursor()
     cursor.execute("SELECT DISTINCT etfProvider FROM etfs")
     etf_provider = [row[0] for row in cursor.fetchall()]
+    
     query = "SELECT symbol, name, expenseRatio, totalAssets, numberOfHoldings FROM etfs WHERE etfProvider = ?"
     
     for provider in etf_provider:
         try:
             cursor.execute(query, (provider,))
             raw_data = cursor.fetchall()
-            # Extract only relevant data and sort it
+            # Extract only relevant data and filter only integer or float totalAssets
-            # Extract only relevant data and filter only integer totalAssets
             res = [
                 {'symbol': row[0], 'name': row[1], 'expenseRatio': row[2], 'totalAssets': row[3], 'numberOfHoldings': row[4]}
                 for row in raw_data if isinstance(row[3], float) or isinstance(row[3], int)
@@ -158,12 +171,11 @@ async def get_etf_provider(etf_con):
                     item['price'] = round(quote_data.get('price'), 2) if quote_data else None
                     item['changesPercentage'] = round(quote_data.get('changesPercentage'), 2) if quote_data else None
                     item['name'] = quote_data.get('name') if quote_data else None
-                except:
+                except Exception:
                     pass
 
             sorted_res = sorted(res, key=lambda x: x['totalAssets'], reverse=True)
 
-
             # Save results to a file if there's data to write
             if sorted_res:
                 with open(f"json/etf/provider/{provider}.json", 'wb') as file:
@@ -171,7 +183,11 @@ async def get_etf_provider(etf_con):
         except Exception as e:
             print(e)
             pass
+    
+    # Close the cursor and connection
     cursor.close()
+    etf_con.close()
+
 
 
 async def get_magnificent_seven():
@@ -912,50 +928,53 @@ async def get_all_reits_list(cursor):
         WHERE {}
     """
     
-    # Use the specific condition within the dictionary
     condition = "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND industry LIKE '%REIT%' AND symbol NOT LIKE '%-%'"
     full_query = base_query.format(condition)
     
-    # Execute the query and fetch all rows
+    cursor.execute(full_query)
-    cursor.execute(full_query)  # Assuming cursor is async
     raw_data = cursor.fetchall()
     
     res_list = []
     for row in raw_data:
         symbol = row[0]
         
-        # Fetch quote data asynchronously
         try:
             quote_data = await get_quote_data(symbol)
-        except Exception as e:
+            if not quote_data:
-            print(f"Error fetching quote data for {symbol}: {e}")
+                continue
-            continue
+                
-        
+            price = quote_data.get('price')
-        if quote_data:
+            changes_percentage = quote_data.get('changesPercentage')
+            
             item = {
                 'symbol': symbol,
                 'name': row[1],
-                'price': round(quote_data.get('price', 0), 2),
+                'price': round(float(price) if price is not None else 0, 2),
-                'changesPercentage': round(quote_data.get('changesPercentage', 0), 2),
+                'changesPercentage': round(float(changes_percentage) if changes_percentage is not None else 0, 2),
                 'marketCap': quote_data.get('marketCap', 0),
             }
             
-            # Get dividend yield if available
+            dividend_yield = stock_screener_data_dict.get(symbol, {}).get('dividendYield')
-            item['dividendYield'] = stock_screener_data_dict.get(symbol, {}).get('dividendYield', None)
+            item['dividendYield'] = dividend_yield
             
-            # Append item if conditions are met
+            if item['marketCap'] > 0 and dividend_yield is not None:
-            if item['marketCap'] > 0 and item['dividendYield'] is not None:
                 res_list.append(item)
+                
+        except Exception as e:
+            print(f"Error processing {symbol}: {e}")
+            continue
     
     if res_list:
         res_list = sorted(res_list, key=lambda x: x['marketCap'] or 0, reverse=True)
         
-        # Add rank to each item
         for rank, item in enumerate(res_list, 1):
             item['rank'] = rank
 
+        import orjson
         with open("json/industry/list/reits.json", 'wb') as file:
             file.write(orjson.dumps(res_list))
+    
+    return res_list
 
 
 async def get_index_list():
@@ -1084,6 +1103,8 @@ async def run():
         get_highest_oi_change(),
         get_highest_option_iv_rank(),
         get_highest_option_premium(),
+        get_etf_holding(),
+        get_etf_provider(),
     )
 
 
@@ -1138,8 +1159,10 @@ async def run():
         etf_cursor.execute("SELECT DISTINCT symbol FROM etfs")
         etf_symbols = [row[0] for row in etf_cursor.fetchall()]
 
-        await get_all_reits_list(cursor)
 
+        
+        await get_all_reits_list(cursor)
+        
         for category, condition in exchange_conditions.items():
             await process_category(cursor, category, condition, 'stocks-list')
             #await asyncio.sleep(1)  # Small delay between categories
@@ -1158,9 +1181,6 @@ async def run():
             #await asyncio.sleep(1)  # Small delay between categories
         
         
-        await get_etf_holding(etf_symbols, etf_con)
-        await get_etf_provider(etf_con)
-
 
     except Exception as e:
         print(e)