diff --git a/app/cron_options_bubble.py b/app/cron_options_bubble.py index f00e7c4..681df8b 100755 --- a/app/cron_options_bubble.py +++ b/app/cron_options_bubble.py @@ -5,8 +5,8 @@ import asyncio import os from dotenv import load_dotenv from benzinga import financial_data -import time - +from tqdm import tqdm +from concurrent.futures import ThreadPoolExecutor load_dotenv() api_key = os.getenv('BENZINGA_API_KEY') @@ -59,7 +59,6 @@ def options_bubble_data(chunk): if item['ticker'] in ['BRK.A', 'BRK.B']: item['ticker'] = f"BRK-{item['ticker'][-1]}" - #Save raw data for each ticker for options page stack bar chart for ticker in chunk: ticker_filtered_data = [entry for entry in res_filtered if entry['ticker'] == ticker] @@ -92,7 +91,6 @@ def options_bubble_data(chunk): filtered_data = [item for item in res_filtered if start_date <= datetime.strptime(item['date'], '%Y-%m-%d').date() <= end_date] - ticker_filtered_data = [entry for entry in filtered_data if entry['ticker'] == ticker] put_volume, call_volume = calculate_put_call_volumes(ticker_filtered_data) avg_dte = calculate_avg_dte(ticker_filtered_data) @@ -104,36 +102,44 @@ def options_bubble_data(chunk): else: with open(f"json/options-bubble/{ticker}.json", 'w') as file: ujson.dump(bubble_data, file) - except ValueError as ve: print(ve) except Exception as e: print(e) -try: - stock_con = sqlite3.connect('stocks.db') - stock_cursor = stock_con.cursor() - stock_cursor.execute("SELECT DISTINCT symbol FROM stocks WHERE symbol NOT LIKE '%.%'") - stock_symbols = [row[0] for row in stock_cursor.fetchall()] - etf_con = sqlite3.connect('etf.db') - etf_cursor = etf_con.cursor() - etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") - etf_symbols = [row[0] for row in etf_cursor.fetchall()] +async def main(): + try: + stock_con = sqlite3.connect('stocks.db') + stock_cursor = stock_con.cursor() + stock_cursor.execute("SELECT DISTINCT symbol FROM stocks WHERE marketCap >= 500E6 AND symbol NOT LIKE '%.%'") + stock_symbols = [row[0] for row in stock_cursor.fetchall()] - stock_con.close() - etf_con.close() - - total_symbols = stock_symbols + etf_symbols - total_symbols = [item.replace("BRK-B", "BRK.B") for item in total_symbols] + etf_con = sqlite3.connect('etf.db') + etf_cursor = etf_con.cursor() + etf_cursor.execute("SELECT DISTINCT symbol FROM etfs WHERE totalAssets >= 10E9") + etf_symbols = [row[0] for row in etf_cursor.fetchall()] - chunk_size = len(total_symbols) // 40 # Divide the list into N chunks - chunks = [total_symbols[i:i + chunk_size] for i in range(0, len(total_symbols), chunk_size)] - - for chunk in chunks: - options_bubble_data(chunk) + stock_con.close() + etf_con.close() + + total_symbols = stock_symbols + etf_symbols + total_symbols = [item.replace("BRK-B", "BRK.B") for item in total_symbols] -except Exception as e: - print(e) + print(len(total_symbols)) + chunk_size = len(total_symbols) // 100 # Divide the list into N chunks + chunks = [total_symbols[i:i + chunk_size] for i in range(0, len(total_symbols), chunk_size)] + + loop = asyncio.get_running_loop() + with ThreadPoolExecutor(max_workers=2) as executor: + tasks = [loop.run_in_executor(executor, options_bubble_data, chunk) for chunk in chunks] + for f in tqdm(asyncio.as_completed(tasks), total=len(tasks)): + await f + + except Exception as e: + print(e) + +if __name__ == "__main__": + asyncio.run(main())