diff --git a/app/cron_options_oi.py b/app/cron_options_oi.py new file mode 100644 index 0000000..cbf5818 --- /dev/null +++ b/app/cron_options_oi.py @@ -0,0 +1,147 @@ +import requests +import orjson +import re +from datetime import datetime +from dotenv import load_dotenv +import os +import sqlite3 +import time +from tqdm import tqdm + +load_dotenv() + +api_key = os.getenv('UNUSUAL_WHALES_API_KEY') +headers = {"Accept": "application/json, text/plain", "Authorization": api_key} + +# Connect to the databases +con = sqlite3.connect('stocks.db') +etf_con = sqlite3.connect('etf.db') +cursor = con.cursor() +cursor.execute("PRAGMA journal_mode = wal") +#cursor.execute("SELECT DISTINCT symbol FROM stocks WHERE symbol NOT LIKE '%.%' AND marketCap > 1E9") +cursor.execute("SELECT DISTINCT symbol FROM stocks WHERE symbol NOT LIKE '%.%'") +stocks_symbols = [row[0] for row in cursor.fetchall()] + +etf_cursor = etf_con.cursor() +etf_cursor.execute("PRAGMA journal_mode = wal") +#etf_cursor.execute("SELECT DISTINCT symbol FROM etfs WHERE marketCap > 1E9") +etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") +etf_symbols = [row[0] for row in etf_cursor.fetchall()] + +con.close() +etf_con.close() + + +def get_tickers_from_directory(directory: str): + try: + # Ensure the directory exists + if not os.path.exists(directory): + raise FileNotFoundError(f"The directory '{directory}' does not exist.") + + # Get all tickers from filenames + return [file.replace(".json", "") for file in os.listdir(directory) if file.endswith(".json")] + + except Exception as e: + print(f"An error occurred: {e}") + return [] + + + + +def save_json(data, symbol, directory_path): + os.makedirs(directory_path, exist_ok=True) # Ensure the directory exists + with open(f"{directory_path}/{symbol}.json", 'wb') as file: # Use binary mode for orjson + file.write(orjson.dumps(data)) + + +def safe_round(value, decimals=2): + try: + return round(float(value), decimals) + except (ValueError, TypeError): + return value + + +def prepare_data(data, symbol, directory_path, sort_by = "date"): + data = [{k: v for k, v in item.items() if "charm" not in k and "vanna" not in k} for item in data] + res_list = [] + for item in data: + try: + new_item = { + key: safe_round(value) if isinstance(value, (int, float, str)) else value + for key, value in item.items() + } + + res_list.append(new_item) + except: + pass + + if res_list: + res_list = sorted(res_list, key=lambda x: x[sort_by], reverse=True) + save_json(res_list, symbol, directory_path) + + +def get_strike_data(): + print("Starting to download strike data...") + directory_path = "json/oi/strike" + total_symbols = get_tickers_from_directory(directory_path) + if len(total_symbols) < 100: + total_symbols = stocks_symbols+etf_symbols + + counter = 0 + #Test mode + #total_symbols = ['GME','SPY'] + for symbol in tqdm(total_symbols): + try: + url = f"https://api.unusualwhales.com/api/stock/{symbol}/oi-per-strike" + + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json()['data'] + prepare_data(data, symbol, directory_path, sort_by = 'strike') + + counter +=1 + + # If 50 chunks have been processed, sleep for 60 seconds + if counter == 260: + print("Sleeping...") + time.sleep(60) + counter = 0 + + except Exception as e: + print(f"Error for {symbol}:{e}") + +def get_expiry_data(): + print("Starting to download expiry data...") + directory_path = "json/oi/expiry" + total_symbols = get_tickers_from_directory(directory_path) + if len(total_symbols) < 100: + total_symbols = stocks_symbols+etf_symbols + + counter = 0 + + for symbol in tqdm(total_symbols): + try: + url = f"https://api.unusualwhales.com/api/stock/{symbol}/oi-per-expiry" + + response = requests.get(url, headers=headers) + if response.status_code == 200: + data = response.json()['data'] + prepare_data(data, symbol, directory_path) + + counter +=1 + + # If 50 chunks have been processed, sleep for 60 seconds + if counter == 260: + print("Sleeping...") + time.sleep(60) + counter = 0 + + except Exception as e: + print(f"Error for {symbol}:{e}") + + +if __name__ == '__main__': + get_strike_data() + #time.sleep(60) + get_expiry_data() + diff --git a/app/main.py b/app/main.py index 158d3b7..e016831 100755 --- a/app/main.py +++ b/app/main.py @@ -2702,6 +2702,35 @@ async def get_data(data:ParamsData, api_key: str = Security(get_api_key)): headers={"Content-Encoding": "gzip"} ) + +@app.post("/options-oi") +async def get_data(data:ParamsData, api_key: str = Security(get_api_key)): + ticker = data.params.upper() + category = data.category.lower() + + cache_key = f"options-oi-{ticker}-{category}" + cached_result = redis_client.get(cache_key) + if cached_result: + return StreamingResponse( + io.BytesIO(cached_result), + media_type="application/json", + headers={"Content-Encoding": "gzip"}) + try: + with open(f"json/oi/{category}/{ticker}.json", 'rb') as file: + data = orjson.loads(file.read()) + + except: + data = [] + data = orjson.dumps(data) + compressed_data = gzip.compress(data) + redis_client.set(cache_key, compressed_data) + redis_client.expire(cache_key, 3600*60) + return StreamingResponse( + io.BytesIO(compressed_data), + media_type="application/json", + headers={"Content-Encoding": "gzip"} + ) + @app.post("/options-stats-ticker") async def get_options_stats_ticker(data:TickerData, api_key: str = Security(get_api_key)): ticker = data.ticker.upper() diff --git a/app/primary_cron_job.py b/app/primary_cron_job.py index 8c16437..6fbf0d0 100755 --- a/app/primary_cron_job.py +++ b/app/primary_cron_job.py @@ -93,6 +93,8 @@ def run_options_jobs(): if week <= 5: run_command(["python3", "cron_options_gex_dex.py"]) time.sleep(60) + run_command(["python3", "cron_options_oi.py"]) + time.sleep(60) run_command(["python3", "cron_options_stats.py"]) time.sleep(60) run_command(["python3", "cron_options_hottest_contracts.py"])