bugfixing unusual activity

app/cron_unusual_activity.py

@@ -100,7 +100,7 @@ async def get_data(symbol):
                 trade_data = {key.lstrip('_'): value for key, value in trade_data.items()}
                 option_symbol = trade_data['contract'].replace("___","").replace("__","").replace("_","")
                 date_expiration, option_type, strike_price = parse_option_symbol(option_symbol)
-                if trade_data['underlying_price_at_execution'] > 0 and date_expiration >= datetime.today().date():
+                if trade_data['underlying_price_at_execution'] > 0: #and date_expiration >= datetime.today().date():
 
                     res_list.append({'date': trade_data['timestamp'].strftime("%Y-%m-%d"),
                         'askprice': trade_data['ask_at_execution'],

app/test.py

@@ -1,32 +1,150 @@
 from __future__ import print_function
+import asyncio
 import time
 import intrinio_sdk as intrinio
 from intrinio_sdk.rest import ApiException
+from datetime import datetime, timedelta
+import ast
+import orjson
+from tqdm import tqdm
+import aiohttp
+from concurrent.futures import ThreadPoolExecutor
+import sqlite3
+import re
+
 from dotenv import load_dotenv
 import os
 
 load_dotenv()
+
 api_key = os.getenv('INTRINIO_API_KEY')
 
-
-
 intrinio.ApiClient().set_api_key(api_key)
-intrinio.ApiClient().allow_retries(True)
+# intrinio.ApiClient().allow_retries(True)
 
+today = datetime.today()
+start_date = (today - timedelta(150)).strftime("%Y-%m-%d")
+end_date = (today + timedelta(30)).strftime("%Y-%m-%d")
 
-
-
-source = 'delayed'
-start_date = ''
-start_time = ''
-end_date = ''
-end_time = ''
-timezone = 'UTC'
-page_size = 100
-min_size = 100
-security = 'AAPL'
 next_page = ''
+page_size = 1000
+activity_type = ''
+sentiment = ''
+minimum_total_value = 0
+maximum_total_value = 2E10
 
-response = intrinio.OptionsApi().get_option_trades(source=source, start_date=start_date, start_time=start_time, end_date=end_date, end_time=end_time, timezone=timezone, page_size=page_size, min_size=min_size, security=security, next_page=next_page)
-print(response)
-    
+# Database connection and symbol retrieval
+def get_total_symbols():
+    with sqlite3.connect('stocks.db') as con:
+        cursor = con.cursor()
+        cursor.execute("PRAGMA journal_mode = wal")
+        cursor.execute("SELECT DISTINCT symbol FROM stocks WHERE symbol NOT LIKE '%.%'")
+        stocks_symbols = [row[0] for row in cursor.fetchall()]
+
+    with sqlite3.connect('etf.db') as etf_con:
+        etf_cursor = etf_con.cursor()
+        etf_cursor.execute("PRAGMA journal_mode = wal")
+        etf_cursor.execute("SELECT DISTINCT symbol FROM etfs")
+        etf_symbols = [row[0] for row in etf_cursor.fetchall()]
+
+    return stocks_symbols + etf_symbols
+
+
+def get_tickers_from_directory():
+    directory = "json/options-historical-data/companies"
+    try:
+        # Ensure the directory exists
+        if not os.path.exists(directory):
+            raise FileNotFoundError(f"The directory '{directory}' does not exist.")
+        
+        # Get all tickers from filenames
+        return [file.replace(".json", "") for file in os.listdir(directory) if file.endswith(".json")]
+    
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return []
+
+
+async def save_json(data, symbol):
+    directory = "json/unusual-activity"
+    os.makedirs(directory, exist_ok=True)
+    with open(f"{directory}/{symbol}.json", 'wb') as file:
+        file.write(orjson.dumps(data))
+
+
+def parse_option_symbol(option_symbol):
+    # Define regex pattern to match the symbol structure
+    match = re.match(r"([A-Z]+)(\d{6})([CP])(\d+)", option_symbol)
+    if not match:
+        raise ValueError(f"Invalid option_symbol format: {option_symbol}")
+    
+    ticker, expiration, option_type, strike_price = match.groups()
+    
+    # Convert expiration to datetime
+    date_expiration = datetime.strptime(expiration, "%y%m%d").date()
+    
+    # Convert strike price to float
+    strike_price = int(strike_price) / 1000
+
+    return date_expiration, option_type, strike_price
+
+
+async def get_data(symbol):
+    response = intrinio.OptionsApi().get_unusual_activity_intraday(
+        symbol, 
+        next_page=next_page, 
+        page_size=page_size, 
+        activity_type=activity_type, 
+        sentiment=sentiment, 
+        start_date=start_date, 
+        end_date=end_date, 
+        minimum_total_value=minimum_total_value, 
+        maximum_total_value=maximum_total_value
+    )
+    data = (response.__dict__['_trades'])
+    res_list = []
+    if len(data) > 0:
+        for item in data:
+            try:
+                trade_data = item.__dict__
+                trade_data = {key.lstrip('_'): value for key, value in trade_data.items()}
+                option_symbol = trade_data['contract'].replace("___", "").replace("__", "").replace("_", "")
+                date_expiration, option_type, strike_price = parse_option_symbol(option_symbol)
+
+                res_list.append({
+                    'date': trade_data['timestamp'].strftime("%Y-%m-%d"),
+                    'askprice': trade_data['ask_at_execution'],
+                    'bidPrice': trade_data['bid_at_execution'],
+                    'premium': trade_data['total_value'],
+                    'sentiment': trade_data['sentiment'].capitalize(),
+                    'avgPrice': trade_data['average_price'],
+                    'price': trade_data['underlying_price_at_execution'],
+                    'unusualType': trade_data['type'].capitalize(),
+                    'size': trade_data['total_size'],
+                    'optionSymbol': option_symbol,
+                    'strike': strike_price,
+                    'expiry': date_expiration.strftime("%Y-%m-%d"),
+                    'optionType': option_type.replace("P", "Put").replace("C", "Call")
+                })
+            except Exception as e:
+                print(e)
+
+        res_list = sorted(res_list, key=lambda x: x['date'], reverse=True)
+        res_list = [item for item in res_list if item['date'] == '2025-01-24']
+        print(len(res_list))
+
+
+async def main():
+    total_symbols = get_tickers_from_directory()
+    if len(total_symbols) < 3000:
+        total_symbols = get_total_symbols()
+
+    for symbol in tqdm(['XLU']):
+        try:
+            data = await get_data(symbol)
+        except Exception as e:
+            print(f"Error processing {symbol}: {e}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())