From 1a5947a40f0c67dcaae2a82294946893de9ec300 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Sun, 30 Jun 2024 21:52:17 +0200 Subject: [PATCH] add fail-to-deliver endpoint --- app/cron_fail_to_deliver.py | 113 ++++++++++++++++++++++++++++++++++++ app/cron_trading_halt.py | 22 +++++++ app/main.py | 18 ++++++ 3 files changed, 153 insertions(+) create mode 100644 app/cron_fail_to_deliver.py create mode 100644 app/cron_trading_halt.py diff --git a/app/cron_fail_to_deliver.py b/app/cron_fail_to_deliver.py new file mode 100644 index 0000000..7b3d3a3 --- /dev/null +++ b/app/cron_fail_to_deliver.py @@ -0,0 +1,113 @@ +from datetime import timedelta +import os +import pandas as pd +import json +from pathlib import Path +import time +import ujson +import sqlite3 + + + +def save_json(symbol, data): + with open(f"json/fail-to-deliver/companies/{symbol}.json", 'w') as file: + ujson.dump(data, file) + +def get_total_data(files_available, limit=24): + """ + Combine all the 1/2 monthly csv into 1 large csv file. + """ + combined_df = pd.DataFrame(columns=["SETTLEMENT DATE", "SYMBOL", "QUANTITY (FAILS)", "PRICE"]) + for file in files_available[:limit]: + print(f"Processing file: {file}") + try: + # Read the CSV file with appropriate parameters + df = pd.read_csv(file, sep='|', quotechar='"', engine='python') + + # Safely remove columns if they exist + if 'CUSIP' in df.columns: + del df['CUSIP'] + if 'DESCRIPTION' in df.columns: + del df['DESCRIPTION'] + + # Safely convert SETTLEMENT DATE + if 'SETTLEMENT DATE' in df.columns: + df['SETTLEMENT DATE'] = pd.to_datetime(df['SETTLEMENT DATE'], format='%Y%m%d', errors='coerce') + + combined_df = pd.concat([combined_df, df]).drop_duplicates() + + except pd.errors.ParserError as e: + print(f"Error reading {file}: {e}") + except Exception as e: + print(f"Unexpected error with {file}: {e}") + + combined_df["SETTLEMENT DATE"] = combined_df["SETTLEMENT DATE"].astype(str) + combined_df.rename(columns={ + "SETTLEMENT DATE": "date", + "SYMBOL": "Ticker", + "QUANTITY (FAILS)": "failToDeliver", + "PRICE": "price" + }, inplace=True) + + combined_df["T+35 Date"] = pd.to_datetime(combined_df['date'], format='%Y-%m-%d', errors="coerce") + timedelta(days=35) + combined_df["T+35 Date"] = combined_df["T+35 Date"].astype(str) + + combined_df["failToDeliver"] = pd.to_numeric(combined_df["failToDeliver"], errors='coerce') + combined_df["failToDeliver"] = combined_df["failToDeliver"].fillna(0).astype(int) + + combined_df = combined_df[~combined_df["Ticker"].isna()] + combined_df.sort_values(by="date", inplace=True) + + print(combined_df) + return combined_df + +def filter_by_ticker(combined_df): + # Group by 'Ticker' column + grouped = combined_df.groupby('Ticker') + + # Dictionary to store DataFrames for each ticker + ticker_dfs = {} + + # Iterate over groups + for ticker, group in grouped: + # Store each group (DataFrame) in the dictionary + ticker_dfs[ticker] = group.copy() # Use .copy() to avoid modifying original DataFrame + + return ticker_dfs + +if __name__ == '__main__': + + con = sqlite3.connect('stocks.db') + etf_con = sqlite3.connect('etf.db') + + cursor = con.cursor() + cursor.execute("PRAGMA journal_mode = wal") + cursor.execute("SELECT DISTINCT symbol FROM stocks WHERE symbol NOT LIKE '%.%'") + stock_symbols = [row[0] for row in cursor.fetchall()] + + etf_cursor = etf_con.cursor() + etf_cursor.execute("PRAGMA journal_mode = wal") + etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") + etf_symbols = [row[0] for row in etf_cursor.fetchall()] + + con.close() + etf_con.close() + + total_symbols = stock_symbols + etf_symbols + + # Specify your directory path + directory_path = 'json/fail-to-deliver/csv' + + # List CSV files sorted by modification time + files_available = sorted(Path(directory_path).iterdir(), key=os.path.getmtime) + combined_df = get_total_data(files_available, limit=1000) + + + ticker_dataframes = filter_by_ticker(combined_df) + + # Example usage: print or access dataframes for each ticker + for ticker, df in ticker_dataframes.items(): + if ticker in total_symbols: + data= [{k: v for k, v in d.items() if k not in ['Ticker', 'T+35 Date']} for d in df.to_dict('records')] + save_json(ticker, data) + \ No newline at end of file diff --git a/app/cron_trading_halt.py b/app/cron_trading_halt.py new file mode 100644 index 0000000..3bdf43f --- /dev/null +++ b/app/cron_trading_halt.py @@ -0,0 +1,22 @@ +import os +import sys +import pandas as pd + + + +def main(method=""): + print("Getting Trading Halt...") + url = "https://www.nyse.com/api/trade-halts/current/download" + df = pd.read_csv(url) + df.fillna("N/A", inplace=True) + df["Halt Date"] = df["Halt Date"].astype(str) + df["Halt Date"] = df["Halt Date"].apply(lambda x: str(x[6:] + "-" + x[:2] + "-" + x[3:5] if x != "N/A" else "N/A")) + df["Resume Date"] = df["Resume Date"].astype(str) + #df["Resume Date"] = df["Resume Date"].apply(lambda x: str(x[6:] + "-" + x[:2] + "-" + x[3:5] if x != "N/A" else "N/A")) + del df["Name"] + print(df) + print("Trading Halt Successfully Completed...\n") + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/app/main.py b/app/main.py index 4fbf94e..a988e8e 100755 --- a/app/main.py +++ b/app/main.py @@ -2946,6 +2946,24 @@ async def get_market_maker(): except: res = [] + redis_client.set(cache_key, ujson.dumps(res)) + redis_client.expire(cache_key, 3600*3600) # Set cache expiration time to 1 day + return res + + +@app.post("/fail-to-deliver") +async def get_fail_to_deliver(data:TickerData): + ticker = data.ticker.upper() + cache_key = f"fail-to-deliver-{ticker}" + cached_result = redis_client.get(cache_key) + if cached_result: + return ujson.loads(cached_result) + try: + with open(f"json/fail-to-deliver/companies/{ticker}.json", 'r') as file: + res = ujson.load(file) + except: + res = [] + redis_client.set(cache_key, ujson.dumps(res)) redis_client.expire(cache_key, 3600*3600) # Set cache expiration time to 1 day return res \ No newline at end of file