diff --git a/app/cron_options_historical_volume.py b/app/cron_options_historical_volume.py index 5448bc3..b1f9de3 100644 --- a/app/cron_options_historical_volume.py +++ b/app/cron_options_historical_volume.py @@ -94,56 +94,89 @@ def get_contracts_from_directory(directory: str): +def get_contracts_from_directory(directory): + """Retrieve a list of contract files from a directory.""" + return [f.split('.')[0] for f in os.listdir(directory) if f.endswith('.json')] + + def aggregate_data_by_date(): total_symbols = ['AA'] - data_by_date = defaultdict(lambda: {"volume": 0, "open_interest": 0}) - contracts_processed = 0 + data_by_date = defaultdict(lambda: { + "date": "", # Add date field to the dictionary + "call_volume": 0, + "put_volume": 0, + "call_open_interest": 0, + "put_open_interest": 0, + "call_premium": 0, + "put_premium": 0, + }) - for symbol in tqdm(total_symbols, desc="Processing symbols"): + for symbol in tqdm(total_symbols): try: - contract_list = get_contracts_from_directory(f"json/all-options-contracts/{symbol}") + contract_dir = f"json/all-options-contracts/{symbol}" + if not os.path.exists(contract_dir): + print(f"Directory does not exist: {contract_dir}") + continue + + contract_list = get_contracts_from_directory(contract_dir) for item in tqdm(contract_list, desc=f"Processing {symbol} contracts", leave=False): try: - with open(f"json/all-options-contracts/{symbol}/{item}.json", "r") as file: + file_path = os.path.join(contract_dir, f"{item}.json") + with open(file_path, "r") as file: data = orjson.loads(file.read()) - - # Process historical data - for entry in data.get('history', []): - date = entry.get('date') - volume = entry.get('volume') - open_interest = entry.get('open_interest') - - if date: - # Aggregate volume + option_type = data.get('optionType', None) + if option_type not in ['call', 'put']: + continue + for entry in data.get('history', []): + date = entry.get('date') + volume = entry.get('volume',0) + open_interest = entry.get('open_interest',0) + total_premium = entry.get('total_premium',0) + print(total_premium) + if volume is None: + volume = 0 + if open_interest is None: + open_interest = 0 + if total_premium is None: + total_premium = 0 + + + if date: + data_by_date[date]["date"] = date # Store the date in the dictionary + if option_type == 'call': if volume is not None: - data_by_date[date]["volume"] += int(volume) - - # Aggregate open interest + data_by_date[date]["call_volume"] += int(volume) if open_interest is not None: - data_by_date[date]["open_interest"] += int(open_interest) - - contracts_processed += 1 - + data_by_date[date]["call_open_interest"] += int(open_interest) + if total_premium is not None: + data_by_date[date]["call_premium"] += int(total_premium) + elif option_type == 'put': + if volume is not None: + data_by_date[date]["put_volume"] += int(volume) + if open_interest is not None: + data_by_date[date]["put_open_interest"] += int(open_interest) + if total_premium is not None: + data_by_date[date]["put_premium"] += int(total_premium) + + except Exception as e: print(f"Error processing contract {item} for {symbol}: {e}") continue - except Exception as e: print(f"Error processing symbol {symbol}: {e}") continue + + # Convert to list of dictionaries and sort by date + result = list(data_by_date.values()) + result.sort(key=lambda x: x['date']) - # Sort results by date - sorted_results = {date: metrics for date, metrics in sorted(data_by_date.items())} - - return sorted_results, contracts_processed + return result + + if __name__ == '__main__': # Run the aggregation - results, total_processed = aggregate_data_by_date() - - print("\nData by date:") - for date, metrics in results.items(): - print(f"{date}: Volume = {metrics['volume']:,}, Open Interest = {metrics['open_interest']:,}") - - print(f"\nTotal contracts processed: {total_processed}") \ No newline at end of file + results = aggregate_data_by_date() + + print(results[-1]) \ No newline at end of file diff --git a/app/cron_options_single_contract.py b/app/cron_options_single_contract.py index 4eb9974..a1928fa 100644 --- a/app/cron_options_single_contract.py +++ b/app/cron_options_single_contract.py @@ -7,11 +7,11 @@ from datetime import datetime, timedelta import ast import orjson from tqdm import tqdm -from dotenv import load_dotenv -import os import aiohttp from concurrent.futures import ThreadPoolExecutor import sqlite3 +from dotenv import load_dotenv +import os load_dotenv() api_key = os.getenv('INTRINIO_API_KEY') @@ -48,8 +48,8 @@ before = '2100-12-31' N_year_ago = datetime.now() - timedelta(days=365) include_related_symbols = False page_size = 5000 -MAX_CONCURRENT_REQUESTS = 20 # Adjust based on API rate limits -BATCH_SIZE = 2000 +MAX_CONCURRENT_REQUESTS = 50 # Adjust based on API rate limits +BATCH_SIZE = 1500 def get_all_expirations(symbol): response = intrinio.OptionsApi().get_options_expirations_eod( @@ -149,8 +149,17 @@ async def get_single_contract_eod_data(symbol, contract_id, semaphore): except: res_list[i]['changeOI'] = None res_list[i]['changesPercentageOI'] = None + + for i in range(1,len(res_list)): + try: + volume = res_list[i]['volume'] + avg_fill = res_list[i]['mark'] + res_list[i]['total_premium'] = int(avg_fill*volume*100) + except: + res_list[i]['total_premium'] = 0 + - data = {'expiration': key_data['_expiration'], 'strike': key_data['_strike'], 'optionType': key_data['_type'], 'history': history} + data = {'expiration': key_data['_expiration'], 'strike': key_data['_strike'], 'optionType': key_data['_type'], 'history': res_list} await save_json(data, symbol, contract_id) except Exception as e: diff --git a/app/daily_options_volume.csv b/app/daily_options_volume.csv new file mode 100644 index 0000000..6cbe23d --- /dev/null +++ b/app/daily_options_volume.csv @@ -0,0 +1,15 @@ +Expiration Date,Total Call Volume,Total Put Volume,Total Volume +2025-01-24,2273.0,1846.0,4119.0 +2025-01-31,460.0,830.0,1290.0 +2025-02-07,117.0,193.0,310.0 +2025-02-14,191.0,76.0,267.0 +2025-02-21,637.0,340.0,977.0 +2025-02-28,73.0,61.0,134.0 +2025-03-21,516.0,189.0,705.0 +2025-04-17,4838.0,185.0,5023.0 +2025-06-20,535.0,1996.0,2531.0 +2025-07-18,86.0,191.0,277.0 +2025-09-19,478.0,388.0,866.0 +2026-01-16,1458.0,2072.0,3530.0 +2026-12-18,26.0,221.0,247.0 +2027-01-15,71.0,439.0,510.0 diff --git a/app/test.py b/app/test.py index e69de29..4b7abca 100644 --- a/app/test.py +++ b/app/test.py @@ -0,0 +1,23 @@ +from __future__ import print_function +import time +import intrinio_sdk as intrinio +from intrinio_sdk.rest import ApiException +from dotenv import load_dotenv +import os + +load_dotenv() +api_key = os.getenv('INTRINIO_API_KEY') + + + +intrinio.ApiClient().set_api_key(api_key) +intrinio.ApiClient().allow_retries(True) + + + + +identifier = 'AA250321C00045000' +next_page = '' + +response = intrinio.OptionsApi().get_options_prices_eod(identifier, next_page=next_page, ) +print(response) \ No newline at end of file