diff --git a/app/cron_list.py b/app/cron_list.py index b5bc0dd..8eb646f 100644 --- a/app/cron_list.py +++ b/app/cron_list.py @@ -80,8 +80,18 @@ async def process_category(cursor, category, condition, category_type='market-ca return sorted_result -async def get_etf_holding(etf_symbols, etf_con): - for ticker in ['SPY']: #tqdm(etf_symbols): + +async def get_etf_holding(): + # Create a connection to the ETF database + etf_con = sqlite3.connect('etf.db') + etf_cursor = etf_con.cursor() + etf_cursor.execute("PRAGMA journal_mode = wal") + + # Fetch distinct ETF symbols + etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") + etf_symbols = [row[0] for row in etf_cursor.fetchall()] + + for ticker in etf_symbols: res = [] df = pd.read_sql_query(query_etf_holding, etf_con, params=(ticker,)) try: @@ -115,7 +125,6 @@ async def get_etf_holding(etf_symbols, etf_con): except: pass - # Assign price and changesPercentage if available, otherwise set to None item['weightPercentage'] = round(item.get('weightPercentage'), 2) if item['weightPercentage'] else None @@ -129,22 +138,26 @@ async def get_etf_holding(etf_symbols, etf_con): with open(f"json/etf/holding/{ticker}.json", 'wb') as file: final_res = {'lastUpdate': last_update, 'holdings': res} file.write(orjson.dumps(final_res)) + + # Close the database connection + etf_con.close() -async def get_etf_provider(etf_con): - +async def get_etf_provider(): + # Create a connection to the ETF database + etf_con = sqlite3.connect('etf.db') cursor = etf_con.cursor() cursor.execute("SELECT DISTINCT etfProvider FROM etfs") etf_provider = [row[0] for row in cursor.fetchall()] + query = "SELECT symbol, name, expenseRatio, totalAssets, numberOfHoldings FROM etfs WHERE etfProvider = ?" for provider in etf_provider: try: cursor.execute(query, (provider,)) raw_data = cursor.fetchall() - # Extract only relevant data and sort it - # Extract only relevant data and filter only integer totalAssets + # Extract only relevant data and filter only integer or float totalAssets res = [ {'symbol': row[0], 'name': row[1], 'expenseRatio': row[2], 'totalAssets': row[3], 'numberOfHoldings': row[4]} for row in raw_data if isinstance(row[3], float) or isinstance(row[3], int) @@ -158,12 +171,11 @@ async def get_etf_provider(etf_con): item['price'] = round(quote_data.get('price'), 2) if quote_data else None item['changesPercentage'] = round(quote_data.get('changesPercentage'), 2) if quote_data else None item['name'] = quote_data.get('name') if quote_data else None - except: + except Exception: pass sorted_res = sorted(res, key=lambda x: x['totalAssets'], reverse=True) - # Save results to a file if there's data to write if sorted_res: with open(f"json/etf/provider/{provider}.json", 'wb') as file: @@ -171,7 +183,11 @@ async def get_etf_provider(etf_con): except Exception as e: print(e) pass + + # Close the cursor and connection cursor.close() + etf_con.close() + async def get_magnificent_seven(): @@ -912,50 +928,53 @@ async def get_all_reits_list(cursor): WHERE {} """ - # Use the specific condition within the dictionary condition = "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND industry LIKE '%REIT%' AND symbol NOT LIKE '%-%'" full_query = base_query.format(condition) - # Execute the query and fetch all rows - cursor.execute(full_query) # Assuming cursor is async + cursor.execute(full_query) raw_data = cursor.fetchall() res_list = [] for row in raw_data: symbol = row[0] - # Fetch quote data asynchronously try: quote_data = await get_quote_data(symbol) - except Exception as e: - print(f"Error fetching quote data for {symbol}: {e}") - continue - - if quote_data: + if not quote_data: + continue + + price = quote_data.get('price') + changes_percentage = quote_data.get('changesPercentage') + item = { 'symbol': symbol, 'name': row[1], - 'price': round(quote_data.get('price', 0), 2), - 'changesPercentage': round(quote_data.get('changesPercentage', 0), 2), + 'price': round(float(price) if price is not None else 0, 2), + 'changesPercentage': round(float(changes_percentage) if changes_percentage is not None else 0, 2), 'marketCap': quote_data.get('marketCap', 0), } - # Get dividend yield if available - item['dividendYield'] = stock_screener_data_dict.get(symbol, {}).get('dividendYield', None) + dividend_yield = stock_screener_data_dict.get(symbol, {}).get('dividendYield') + item['dividendYield'] = dividend_yield - # Append item if conditions are met - if item['marketCap'] > 0 and item['dividendYield'] is not None: + if item['marketCap'] > 0 and dividend_yield is not None: res_list.append(item) + + except Exception as e: + print(f"Error processing {symbol}: {e}") + continue if res_list: res_list = sorted(res_list, key=lambda x: x['marketCap'] or 0, reverse=True) - # Add rank to each item for rank, item in enumerate(res_list, 1): item['rank'] = rank + import orjson with open("json/industry/list/reits.json", 'wb') as file: file.write(orjson.dumps(res_list)) + + return res_list async def get_index_list(): @@ -1084,6 +1103,8 @@ async def run(): get_highest_oi_change(), get_highest_option_iv_rank(), get_highest_option_premium(), + get_etf_holding(), + get_etf_provider(), ) @@ -1138,8 +1159,10 @@ async def run(): etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") etf_symbols = [row[0] for row in etf_cursor.fetchall()] - await get_all_reits_list(cursor) + + await get_all_reits_list(cursor) + for category, condition in exchange_conditions.items(): await process_category(cursor, category, condition, 'stocks-list') #await asyncio.sleep(1) # Small delay between categories @@ -1158,9 +1181,6 @@ async def run(): #await asyncio.sleep(1) # Small delay between categories - await get_etf_holding(etf_symbols, etf_con) - await get_etf_provider(etf_con) - except Exception as e: print(e)