bugfixing dividends

app/cron_dividends.py

@@ -20,7 +20,7 @@ today = datetime.now(ny_tz).replace(hour=0, minute=0, second=0, microsecond=0)
 N_days_ago = today - timedelta(days=10)
 
 
-async def save_as_json(symbol, data,file_name):
+async def save_as_json(symbol, data, file_name):
     with open(f"{file_name}/{symbol}.json", 'w') as file:
         ujson.dump(data, file)
 
@@ -54,32 +54,65 @@ async def get_data(ticker, con, etf_con, stock_symbols, etf_symbols):
         df = pd.read_sql_query(query_template, etf_con if table_name == 'etfs' else con, params=(ticker,))
     
         dividend_data = orjson.loads(df[column_name].iloc[0])
-        
         res = dividend_data.get('historical', [])
-
+        filtered_res = [item for item in res if item['recordDate'] and item['paymentDate']]
-        filtered_res = [item for item in res if item['recordDate'] != '' and item['paymentDate'] != '']
+    
-
+        # Dynamically compute the current and previous year based on New York timezone
-        # Calculate payout frequency based on dividends recorded in 2023
+        current_year = str(datetime.now(ny_tz).year)
-        payout_frequency = sum(1 for item in filtered_res if '2023' in item['recordDate'])
+        previous_year = str(datetime.now(ny_tz).year - 1)
+    
+        # Filter records for the current year
+        current_year_records = [item for item in filtered_res if current_year in item['recordDate']]
+        dividends_current_year = [float(item['adjDividend']) for item in current_year_records]
+    
+        # Compute the estimated payout frequency using the intervals between record dates
+        record_dates = []
+        for item in current_year_records:
+            try:
+                record_date = datetime.strptime(item['recordDate'], '%Y-%m-%d')
+                record_dates.append(record_date)
+            except Exception as e:
+                continue
+        record_dates.sort()
+    
+        if len(record_dates) > 1:
+            total_days = (record_dates[-1] - record_dates[0]).days
+            intervals = len(record_dates) - 1
+            average_interval = total_days / intervals if intervals > 0 else None
+            estimated_frequency = round(365 / average_interval) if average_interval and average_interval > 0 else len(record_dates)
+        else:
+            # If there's only one record, assume weekly (52 payments) as a fallback;
+            # if no record exists, frequency remains 0.
+            estimated_frequency = 52 if record_dates else 0
+    
+        # Project the annual dividend using the average dividend amount
+        if dividends_current_year:
+            avg_dividend = sum(dividends_current_year) / len(dividends_current_year)
+            annual_dividend = round(avg_dividend * estimated_frequency, 2)
+        else:
+            annual_dividend = 0
+    
+        # For the previous year, assume the data is complete and sum the dividends
+        dividends_previous_year = [
+            float(item['adjDividend'])
+            for item in filtered_res
+            if previous_year in item['recordDate']
+        ]
+        previous_annual_dividend = round(sum(dividends_previous_year), 2) if dividends_previous_year else 0
+    
         quote_data = orjson.loads(df['quote'].iloc[0])[0]
         eps = quote_data.get('eps')
         current_price = quote_data.get('price')
-
+    
-        amount = filtered_res[0]['adjDividend'] if filtered_res else 0
-        annual_dividend = round(amount * payout_frequency, 2)
         dividend_yield = round((annual_dividend / current_price) * 100, 2) if current_price else None
-
         payout_ratio = round((1 - (eps - annual_dividend) / eps) * 100, 2) if eps else None
-
+        dividend_growth = (
-        previous_index = next((i for i, item in enumerate(filtered_res) if '2023' in item['recordDate']), None)
+            round(((annual_dividend - previous_annual_dividend) / previous_annual_dividend) * 100, 2)
-
+            if previous_annual_dividend else None
-        # Calculate previousAnnualDividend and dividendGrowth
+        )
-        previous_annual_dividend = (filtered_res[previous_index]['adjDividend'] * payout_frequency) if previous_index is not None else 0
+    
-        dividend_growth = round(((annual_dividend - previous_annual_dividend) / previous_annual_dividend) * 100, 2) if previous_annual_dividend else None
-
-
         return {
-            'payoutFrequency': payout_frequency,
+            'payoutFrequency': estimated_frequency,
             'annualDividend': annual_dividend,
             'dividendYield': dividend_yield,
             'payoutRatio': payout_ratio,
@@ -87,55 +120,11 @@ async def get_data(ticker, con, etf_con, stock_symbols, etf_symbols):
             'history': filtered_res,
         }
     
-    except:
+    except Exception as e:
-        res = {}
+        print(f"Error processing ticker {ticker}: {e}")
-
+        return {}
-    return res
-
-
-
-async def get_dividends_announcement(session, ticker, stock_symbols):
-    querystring = {"token": api_key, "parameters[tickers]": ticker}
-    ny_tz = pytz.timezone('America/New_York')
-    today = ny_tz.localize(datetime.now())
-    N_days_ago = today - timedelta(days=30)  # Example, adjust as needed
-
-    try:
-        async with session.get(url, params=querystring, headers=headers) as response:
-            if response.status == 200:
-                data = ujson.loads(await response.text())['dividends']
-                recent_dates = [item for item in data if N_days_ago <= ny_tz.localize(datetime.strptime(item["date"], "%Y-%m-%d")) <= today]
-                if recent_dates:
-                    nearest_recent = min(recent_dates, key=lambda x: datetime.strptime(x["date"], "%Y-%m-%d"))
-                    try:
-                        symbol = nearest_recent['ticker']
-                        dividend = float(nearest_recent['dividend']) if nearest_recent['dividend'] != '' else 0
-                        dividend_prior = float(nearest_recent['dividend_prior']) if nearest_recent['dividend_prior'] != '' else 0
-                        dividend_yield = round(float(nearest_recent['dividend_yield']) * 100, 2) if nearest_recent['dividend_yield'] != '' else 0
-                        ex_dividend_date = nearest_recent['ex_dividend_date'] if nearest_recent['ex_dividend_date'] != '' else 0
-                        payable_date = nearest_recent['payable_date'] if nearest_recent['payable_date'] != '' else 0
-                        record_date = nearest_recent['record_date'] if nearest_recent['record_date'] != '' else 0
-                        if symbol in stock_symbols and dividend != 0 and payable_date != 0 and dividend_prior != 0 and ex_dividend_date != 0 and record_date != 0 and dividend_yield != 0:
-                            res_dict = {
-                                'symbol': symbol,
-                                'date': nearest_recent['date'],
-                                'dividend': dividend,
-                                'dividendPrior': dividend_prior,
-                                'dividendYield': dividend_yield,
-                                'exDividendDate': ex_dividend_date,
-                                'payableDate': payable_date,
-                                'recordDate': record_date,
-                            }
-                            await save_as_json(symbol, res_dict,'json/dividends/announcement')
-                    except Exception as e:
-                        # Log or handle the exception
-                        print(e)
-    except:
-        pass
-
 
 async def run():
-
     con = sqlite3.connect('stocks.db')
     cursor = con.cursor()
     cursor.execute("PRAGMA journal_mode = wal")
@@ -152,23 +141,15 @@ async def run():
     for ticker in tqdm(total_symbols):
         res = await get_data(ticker, con, etf_con, stock_symbols, etf_symbols)
         try:
-            if len(res.get('history')) > 0 and res.get('dividendGrowth') != None:
+            if len(res.get('history', [])) > 0:
+                print(res)
                 await save_as_json(ticker, res, 'json/dividends/companies')
-        except:
+        except Exception as e:
-            pass
+            print(f"Error saving data for {ticker}: {e}")
     
-
     con.close()
     etf_con.close()
 
-    delete_files_in_directory("json/dividends/announcement")
-
-    async with aiohttp.ClientSession() as session:
-        tasks = [get_dividends_announcement(session, symbol, stock_symbols) for symbol in stock_symbols]
-        for f in tqdm(asyncio.as_completed(tasks), total=len(stock_symbols)):
-            await f
-
-
 try:
     asyncio.run(run())
 except Exception as e: