bugfixing

app/create_etf_db.py

@@ -187,7 +187,7 @@ class ETFDatabase:
                 f"https://financialmodelingprep.com/api/v3/etf-holder/{symbol}?apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v3/etf-country-weightings/{symbol}?apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}",
-                f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_dividend/{symbol}?apikey={api_key}",
+                f"https://financialmodelingprep.com/stable/dividends?symbol={symbol}&apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v4/institutional-ownership/institutional-holders/symbol-ownership-percent?date=2023-09-30&symbol={symbol}&page=0&apikey={api_key}",
             ]
 
@@ -235,7 +235,7 @@ class ETFDatabase:
                         elif isinstance(parsed_data, list) and "etf-country-weightings" in url:
                             fundamental_data['country_weightings'] = ujson.dumps(parsed_data)
                         
-                        elif "stock_dividend" in url:
+                        elif "dividends" in url:
                             fundamental_data['etf_dividend'] = ujson.dumps(parsed_data)
         
                         elif "institutional-ownership/institutional-holders" in url:

app/create_stock_db.py

@@ -100,7 +100,7 @@ class StockDatabase:
             urls = [
                 f"https://financialmodelingprep.com/api/v3/profile/{symbol}?apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}",
-                f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_dividend/{symbol}?limit=400&apikey={api_key}",
+                f"https://financialmodelingprep.com/stable/dividends?symbol={symbol}&apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v4/historical/employee_count?symbol={symbol}&apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_split/{symbol}?apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v4/stock_peers?symbol={symbol}&apikey={api_key}",
@@ -153,7 +153,7 @@ class StockDatabase:
 
                             fundamental_data.update(data_dict)
                        
-                        elif "stock_dividend" in url:
+                        elif "dividends" in url:
                             # Handle list response, save as JSON object
                             fundamental_data['stock_dividend'] = ujson.dumps(parsed_data)
                         elif "employee_count" in url:

app/cron_dividends.py

@@ -34,65 +34,74 @@ async def get_data(ticker, con, etf_con, stock_symbols, etf_symbols):
             column_name = 'stock_dividend'
 
         query_template = f"""
-        SELECT 
+            SELECT {column_name}, quote 
-            {column_name}, quote
+            FROM {table_name}
-        FROM 
+            WHERE symbol = ?
-            {table_name}
-        WHERE
-            symbol = ?
         """
-
+        
-        df = pd.read_sql_query(query_template, etf_con if table_name == 'etfs' else con, params=(ticker,))
+        df = pd.read_sql_query(query_template, 
-    
+                             etf_con if table_name == 'etfs' else con,
+                             params=(ticker,))
+        
         dividend_data = orjson.loads(df[column_name].iloc[0])
         res = dividend_data.get('historical', [])
         filtered_res = [item for item in res if item['recordDate'] and item['paymentDate']]
-    
+
         # Get the current and previous year
         today = datetime.today()
         current_year = str(today.year)
         previous_year = str(today.year - 1)
-    
+
-        # Compute the previous year's total dividend (strictly based on last year)
+        # Compute the previous year's total dividend
         previous_year_records = [item for item in filtered_res if previous_year in item['recordDate']]
         previous_annual_dividend = round(sum(float(item['adjDividend']) for item in previous_year_records), 2) if previous_year_records else 0
 
-        # Estimate the payout frequency dynamically from the current year's dividends
+        # Calculate payout frequency
         current_year_records = [item for item in filtered_res if current_year in item['recordDate']]
-        record_dates = sorted(
+        record_dates = sorted([datetime.strptime(item['recordDate'], '%Y-%m-%d') for item in current_year_records])
-            [datetime.strptime(item['recordDate'], '%Y-%m-%d') for item in current_year_records]
+        
-        )
+        def map_frequency_to_standard(calculated_frequency):
-    
+            if calculated_frequency >= 45:  # Approximately weekly
+                return 5
+            elif calculated_frequency >= 10:  # More frequent than quarterly but less than weekly
+                return 5  # Default to weekly for very frequent payments
+            elif calculated_frequency >= 3:  # Approximately quarterly
+                return 4
+            elif calculated_frequency >= 1.5:  # Approximately semi-annual
+                return 2
+            else:  # Annual or less frequent
+                return 1
+
         if len(record_dates) > 1:
             total_days = (record_dates[-1] - record_dates[0]).days
             intervals = len(record_dates) - 1
             average_interval = total_days / intervals if intervals > 0 else None
-            estimated_frequency = round(365 / average_interval) if average_interval and average_interval > 0 else len(record_dates)
+            raw_frequency = round(365 / average_interval) if average_interval and average_interval > 0 else len(record_dates)
+            estimated_frequency = map_frequency_to_standard(raw_frequency)
         else:
-            estimated_frequency = 52 if record_dates else 0  # Default to weekly if only one record exists
+            estimated_frequency = 1  # Default to annual if only one record exists
-    
+
+        # Process quote data
         quote_data = orjson.loads(df['quote'].iloc[0])[0]
         eps = quote_data.get('eps')
         current_price = quote_data.get('price')
-    
+        
         dividend_yield = round((previous_annual_dividend / current_price) * 100, 2) if current_price else None
         payout_ratio = round((1 - (eps - previous_annual_dividend) / eps) * 100, 2) if eps else None
-        dividend_growth = None  # No calculation since we are strictly using the past year's data
+        
-    
         return {
             'payoutFrequency': estimated_frequency,
-            'annualDividend': previous_annual_dividend,  # Strictly using past year’s data
+            'annualDividend': previous_annual_dividend,
             'dividendYield': dividend_yield,
             'payoutRatio': payout_ratio,
-            'dividendGrowth': dividend_growth,
+            'dividendGrowth': None,
             'history': filtered_res,
         }
-    
+        
     except Exception as e:
         print(f"Error processing ticker {ticker}: {e}")
         return {}
 
-
 async def run():
     con = sqlite3.connect('stocks.db')
     cursor = con.cursor()
@@ -105,10 +114,11 @@ async def run():
     etf_cursor.execute("SELECT DISTINCT symbol FROM etfs")
     etf_symbols = [row[0] for row in etf_cursor.fetchall()]
 
-    total_symbols = stock_symbols + etf_symbols
+    total_symbols = ['AAPL'] #stock_symbols + etf_symbols
     
     for ticker in tqdm(total_symbols):
         res = await get_data(ticker, con, etf_con, stock_symbols, etf_symbols)
+        print(res)
         try:
             if len(res.get('history', [])) > 0:
                 await save_as_json(ticker, res, 'json/dividends/companies')