diff --git a/app/create_etf_db.py b/app/create_etf_db.py index 5f07f2a..ba65a43 100755 --- a/app/create_etf_db.py +++ b/app/create_etf_db.py @@ -9,6 +9,7 @@ from tqdm import tqdm import re import pandas as pd from datetime import datetime +from utils.helper import get_last_completed_quarter import warnings @@ -25,24 +26,10 @@ warnings.filterwarnings("ignore", category=RuntimeWarning, message="invalid valu start_date = datetime(2015, 1, 1).strftime("%Y-%m-%d") end_date = datetime.today().strftime("%Y-%m-%d") -def get_last_completed_quarter(): - today = datetime.today() - year = today.year - month = today.month - # Calculate the current quarter (1 to 4) - current_quarter = (month - 1) // 3 + 1 - - # The previous quarter is the last completed quarter. - # If we're in Q1, the previous quarter is Q4 of last year. - if current_quarter == 1: - return 4, year - 1 - else: - return current_quarter - 1, year - -# Get last completed quarter and its year quarter, year = get_last_completed_quarter() + if os.path.exists("backup_db/etf.db"): os.remove('backup_db/etf.db') @@ -204,7 +191,7 @@ class ETFDatabase: f"https://financialmodelingprep.com/api/v3/etf-country-weightings/{symbol}?apikey={api_key}", f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}", f"https://financialmodelingprep.com/stable/dividends?symbol={symbol}&apikey={api_key}", - f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}", + f"https://financialmodelingprep.com/stable/institutional-ownership/extract-analytics/holder?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}", ] fundamental_data = {} diff --git a/app/create_stock_db.py b/app/create_stock_db.py index b3df0d8..94593ce 100755 --- a/app/create_stock_db.py +++ b/app/create_stock_db.py @@ -14,6 +14,7 @@ from ta.momentum import * from ta.trend import * from ta.volume import * import warnings +from utils.helper import get_last_completed_quarter from dotenv import load_dotenv import os @@ -26,24 +27,9 @@ warnings.filterwarnings("ignore", category=RuntimeWarning, message="invalid valu start_date = datetime(2015, 1, 1).strftime("%Y-%m-%d") end_date = datetime.today().strftime("%Y-%m-%d") - -def get_last_completed_quarter(): - today = datetime.today() - year = today.year - month = today.month - # Calculate the current quarter (1 to 4) - current_quarter = (month - 1) // 3 + 1 - - # The previous quarter is the last completed quarter. - # If we're in Q1, the previous quarter is Q4 of last year. - if current_quarter == 1: - return 4, year - 1 - else: - return current_quarter - 1, year - -# Get last completed quarter and its year quarter, year = get_last_completed_quarter() + if os.path.exists("backup_db/stocks.db"): os.remove('backup_db/stocks.db') @@ -118,7 +104,7 @@ class StockDatabase: f"https://financialmodelingprep.com/api/v4/historical/employee_count?symbol={symbol}&apikey={api_key}", f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_split/{symbol}?apikey={api_key}", f"https://financialmodelingprep.com/api/v4/stock_peers?symbol={symbol}&apikey={api_key}", - f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}", + f"https://financialmodelingprep.com/stable/institutional-ownership/extract-analytics/holder?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}", f"https://financialmodelingprep.com/api/v4/historical/shares_float?symbol={symbol}&apikey={api_key}", f"https://financialmodelingprep.com/api/v4/revenue-product-segmentation?symbol={symbol}&structure=flat&period=annual&apikey={api_key}", f"https://financialmodelingprep.com/api/v4/revenue-geographic-segmentation?symbol={symbol}&structure=flat&apikey={api_key}", @@ -182,6 +168,7 @@ class StockDatabase: elif "institutional-ownership" in url: # Handle list response, save as JSON object fundamental_data['shareholders'] = ujson.dumps(parsed_data) + elif "historical/shares_float" in url: # Handle list response, save as JSON object fundamental_data['historicalShares'] = ujson.dumps(parsed_data) @@ -315,7 +302,7 @@ class StockDatabase: self._create_ticker_table(symbol) # Ensure the table exists # Fetch OHLC data from the API - url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&apikey={api_key}" + url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&to={end_date}&apikey={api_key}" async with session.get(url) as response: data = await response.text() diff --git a/app/cron_ownership_stats.py b/app/cron_ownership_stats.py index 4c97a63..7a42331 100644 --- a/app/cron_ownership_stats.py +++ b/app/cron_ownership_stats.py @@ -7,6 +7,7 @@ import asyncio import aiohttp import random from tqdm import tqdm +from utils.helper import get_last_completed_quarter from dotenv import load_dotenv import os @@ -16,11 +17,11 @@ load_dotenv() api_key = os.getenv('FMP_API_KEY') -include_current_quarter = True +quarter, year = get_last_completed_quarter() async def get_data(session, symbol, max_retries=3, initial_delay=1): - url = f"https://financialmodelingprep.com/api/v4/institutional-ownership/symbol-ownership?symbol={symbol}&includeCurrentQuarter={include_current_quarter}&apikey={api_key}" + url = f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}" for attempt in range(max_retries): try: @@ -31,6 +32,7 @@ async def get_data(session, symbol, max_retries=3, initial_delay=1): data = await response.json() if len(data) > 0: await save_json(symbol, data[0]) + print(data[0]) return else: print(f"Unexpected content type for {symbol}: {content_type}") diff --git a/app/cron_shareholders.py b/app/cron_shareholders.py index e4cbc57..574f8b6 100644 --- a/app/cron_shareholders.py +++ b/app/cron_shareholders.py @@ -3,12 +3,10 @@ import sqlite3 import asyncio import pandas as pd from tqdm import tqdm - import requests import re - async def save_as_json(symbol, data): with open(f"json/shareholders/{symbol}.json", 'w') as file: ujson.dump(data, file) @@ -28,16 +26,9 @@ async def get_data(ticker, con): try: df = pd.read_sql_query(query_template, con, params=(ticker,)) shareholders_list = ujson.loads(df.to_dict()['shareholders'][0]) - # Keys to keep - keys_to_keep = ["cik","ownership", "investorName", "changeInSharesNumberPercentage", "weight", "sharesNumber", "marketValue"] - # Create new list with only the specified keys - shareholders_list = [ - {key: d[key] for key in keys_to_keep} - for d in shareholders_list - ] except Exception as e: - #print(e) + print(e) shareholders_list = [] return shareholders_list diff --git a/app/utils/helper.py b/app/utils/helper.py index a9b163e..715b8cc 100644 --- a/app/utils/helper.py +++ b/app/utils/helper.py @@ -77,4 +77,19 @@ def load_latest_json(directory: str, find=True): except Exception as e: print(f"Error loading JSON file: {e}") - return [] \ No newline at end of file + return [] + + +def get_last_completed_quarter(): + today = datetime.today() + year = today.year + month = today.month + # Calculate the current quarter (1 to 4) + current_quarter = (month - 1) // 3 + 1 + + # The previous quarter is the last completed quarter. + # If we're in Q1, the previous quarter is Q4 of last year. + if current_quarter == 1: + return 4, year - 1 + else: + return current_quarter - 1, year \ No newline at end of file