From cd388202a57a4be4767c65e7e5f678ff22f5d795 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Sat, 22 Feb 2025 12:58:24 +0100 Subject: [PATCH] bugfixing --- app/create_etf_db.py | 4 +-- app/create_stock_db.py | 4 +-- app/cron_dividends.py | 66 ++++++++++++++++++++++++------------------ 3 files changed, 42 insertions(+), 32 deletions(-) diff --git a/app/create_etf_db.py b/app/create_etf_db.py index c5f331b..9c1d5b4 100755 --- a/app/create_etf_db.py +++ b/app/create_etf_db.py @@ -187,7 +187,7 @@ class ETFDatabase: f"https://financialmodelingprep.com/api/v3/etf-holder/{symbol}?apikey={api_key}", f"https://financialmodelingprep.com/api/v3/etf-country-weightings/{symbol}?apikey={api_key}", f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}", - f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_dividend/{symbol}?apikey={api_key}", + f"https://financialmodelingprep.com/stable/dividends?symbol={symbol}&apikey={api_key}", f"https://financialmodelingprep.com/api/v4/institutional-ownership/institutional-holders/symbol-ownership-percent?date=2023-09-30&symbol={symbol}&page=0&apikey={api_key}", ] @@ -235,7 +235,7 @@ class ETFDatabase: elif isinstance(parsed_data, list) and "etf-country-weightings" in url: fundamental_data['country_weightings'] = ujson.dumps(parsed_data) - elif "stock_dividend" in url: + elif "dividends" in url: fundamental_data['etf_dividend'] = ujson.dumps(parsed_data) elif "institutional-ownership/institutional-holders" in url: diff --git a/app/create_stock_db.py b/app/create_stock_db.py index 78306d1..57a35fa 100755 --- a/app/create_stock_db.py +++ b/app/create_stock_db.py @@ -100,7 +100,7 @@ class StockDatabase: urls = [ f"https://financialmodelingprep.com/api/v3/profile/{symbol}?apikey={api_key}", f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}", - f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_dividend/{symbol}?limit=400&apikey={api_key}", + f"https://financialmodelingprep.com/stable/dividends?symbol={symbol}&apikey={api_key}", f"https://financialmodelingprep.com/api/v4/historical/employee_count?symbol={symbol}&apikey={api_key}", f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_split/{symbol}?apikey={api_key}", f"https://financialmodelingprep.com/api/v4/stock_peers?symbol={symbol}&apikey={api_key}", @@ -153,7 +153,7 @@ class StockDatabase: fundamental_data.update(data_dict) - elif "stock_dividend" in url: + elif "dividends" in url: # Handle list response, save as JSON object fundamental_data['stock_dividend'] = ujson.dumps(parsed_data) elif "employee_count" in url: diff --git a/app/cron_dividends.py b/app/cron_dividends.py index 88b287c..24ba81f 100644 --- a/app/cron_dividends.py +++ b/app/cron_dividends.py @@ -34,65 +34,74 @@ async def get_data(ticker, con, etf_con, stock_symbols, etf_symbols): column_name = 'stock_dividend' query_template = f""" - SELECT - {column_name}, quote - FROM - {table_name} - WHERE - symbol = ? + SELECT {column_name}, quote + FROM {table_name} + WHERE symbol = ? """ - - df = pd.read_sql_query(query_template, etf_con if table_name == 'etfs' else con, params=(ticker,)) - + + df = pd.read_sql_query(query_template, + etf_con if table_name == 'etfs' else con, + params=(ticker,)) + dividend_data = orjson.loads(df[column_name].iloc[0]) res = dividend_data.get('historical', []) filtered_res = [item for item in res if item['recordDate'] and item['paymentDate']] - + # Get the current and previous year today = datetime.today() current_year = str(today.year) previous_year = str(today.year - 1) - - # Compute the previous year's total dividend (strictly based on last year) + + # Compute the previous year's total dividend previous_year_records = [item for item in filtered_res if previous_year in item['recordDate']] previous_annual_dividend = round(sum(float(item['adjDividend']) for item in previous_year_records), 2) if previous_year_records else 0 - # Estimate the payout frequency dynamically from the current year's dividends + # Calculate payout frequency current_year_records = [item for item in filtered_res if current_year in item['recordDate']] - record_dates = sorted( - [datetime.strptime(item['recordDate'], '%Y-%m-%d') for item in current_year_records] - ) - + record_dates = sorted([datetime.strptime(item['recordDate'], '%Y-%m-%d') for item in current_year_records]) + + def map_frequency_to_standard(calculated_frequency): + if calculated_frequency >= 45: # Approximately weekly + return 5 + elif calculated_frequency >= 10: # More frequent than quarterly but less than weekly + return 5 # Default to weekly for very frequent payments + elif calculated_frequency >= 3: # Approximately quarterly + return 4 + elif calculated_frequency >= 1.5: # Approximately semi-annual + return 2 + else: # Annual or less frequent + return 1 + if len(record_dates) > 1: total_days = (record_dates[-1] - record_dates[0]).days intervals = len(record_dates) - 1 average_interval = total_days / intervals if intervals > 0 else None - estimated_frequency = round(365 / average_interval) if average_interval and average_interval > 0 else len(record_dates) + raw_frequency = round(365 / average_interval) if average_interval and average_interval > 0 else len(record_dates) + estimated_frequency = map_frequency_to_standard(raw_frequency) else: - estimated_frequency = 52 if record_dates else 0 # Default to weekly if only one record exists - + estimated_frequency = 1 # Default to annual if only one record exists + + # Process quote data quote_data = orjson.loads(df['quote'].iloc[0])[0] eps = quote_data.get('eps') current_price = quote_data.get('price') - + dividend_yield = round((previous_annual_dividend / current_price) * 100, 2) if current_price else None payout_ratio = round((1 - (eps - previous_annual_dividend) / eps) * 100, 2) if eps else None - dividend_growth = None # No calculation since we are strictly using the past year's data - + return { 'payoutFrequency': estimated_frequency, - 'annualDividend': previous_annual_dividend, # Strictly using past year’s data + 'annualDividend': previous_annual_dividend, 'dividendYield': dividend_yield, 'payoutRatio': payout_ratio, - 'dividendGrowth': dividend_growth, + 'dividendGrowth': None, 'history': filtered_res, } - + except Exception as e: print(f"Error processing ticker {ticker}: {e}") return {} - async def run(): con = sqlite3.connect('stocks.db') cursor = con.cursor() @@ -105,10 +114,11 @@ async def run(): etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") etf_symbols = [row[0] for row in etf_cursor.fetchall()] - total_symbols = stock_symbols + etf_symbols + total_symbols = ['AAPL'] #stock_symbols + etf_symbols for ticker in tqdm(total_symbols): res = await get_data(ticker, con, etf_con, stock_symbols, etf_symbols) + print(res) try: if len(res.get('history', [])) > 0: await save_as_json(ticker, res, 'json/dividends/companies')