bugfixing

app/create_etf_db.py

@@ -9,6 +9,7 @@ from tqdm import tqdm
 import re
 import pandas as pd
 from datetime import datetime
+from utils.helper import get_last_completed_quarter
 
 import warnings
 
@@ -25,24 +26,10 @@ warnings.filterwarnings("ignore", category=RuntimeWarning, message="invalid valu
 start_date = datetime(2015, 1, 1).strftime("%Y-%m-%d")
 end_date = datetime.today().strftime("%Y-%m-%d")
 
-def get_last_completed_quarter():
-    today = datetime.today()
-    year = today.year
-    month = today.month
-    # Calculate the current quarter (1 to 4)
-    current_quarter = (month - 1) // 3 + 1
-
-    # The previous quarter is the last completed quarter.
-    # If we're in Q1, the previous quarter is Q4 of last year.
-    if current_quarter == 1:
-        return 4, year - 1
-    else:
-        return current_quarter - 1, year
-
-# Get last completed quarter and its year
 quarter, year = get_last_completed_quarter()
 
 
+
 if os.path.exists("backup_db/etf.db"):
     os.remove('backup_db/etf.db')
 
@@ -204,7 +191,7 @@ class ETFDatabase:
                 f"https://financialmodelingprep.com/api/v3/etf-country-weightings/{symbol}?apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}",
                 f"https://financialmodelingprep.com/stable/dividends?symbol={symbol}&apikey={api_key}",
-                f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
+                f"https://financialmodelingprep.com/stable/institutional-ownership/extract-analytics/holder?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
             ]
 
             fundamental_data = {}

app/create_stock_db.py

@@ -14,6 +14,7 @@ from ta.momentum import *
 from ta.trend import *
 from ta.volume import *
 import warnings
+from utils.helper import get_last_completed_quarter
 
 from dotenv import load_dotenv
 import os
@@ -26,24 +27,9 @@ warnings.filterwarnings("ignore", category=RuntimeWarning, message="invalid valu
 
 start_date = datetime(2015, 1, 1).strftime("%Y-%m-%d")
 end_date = datetime.today().strftime("%Y-%m-%d")
-
-def get_last_completed_quarter():
-    today = datetime.today()
-    year = today.year
-    month = today.month
-    # Calculate the current quarter (1 to 4)
-    current_quarter = (month - 1) // 3 + 1
-
-    # The previous quarter is the last completed quarter.
-    # If we're in Q1, the previous quarter is Q4 of last year.
-    if current_quarter == 1:
-        return 4, year - 1
-    else:
-        return current_quarter - 1, year
-
-# Get last completed quarter and its year
 quarter, year = get_last_completed_quarter()
 
+
 if os.path.exists("backup_db/stocks.db"):
     os.remove('backup_db/stocks.db')
 
@@ -118,7 +104,7 @@ class StockDatabase:
                 f"https://financialmodelingprep.com/api/v4/historical/employee_count?symbol={symbol}&apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_split/{symbol}?apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v4/stock_peers?symbol={symbol}&apikey={api_key}",
-                f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
+                f"https://financialmodelingprep.com/stable/institutional-ownership/extract-analytics/holder?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v4/historical/shares_float?symbol={symbol}&apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v4/revenue-product-segmentation?symbol={symbol}&structure=flat&period=annual&apikey={api_key}",
                 f"https://financialmodelingprep.com/api/v4/revenue-geographic-segmentation?symbol={symbol}&structure=flat&apikey={api_key}",
@@ -182,6 +168,7 @@ class StockDatabase:
                         elif "institutional-ownership" in url:
                             # Handle list response, save as JSON object
                             fundamental_data['shareholders'] = ujson.dumps(parsed_data)
+
                         elif "historical/shares_float" in url:
                             # Handle list response, save as JSON object
                             fundamental_data['historicalShares'] = ujson.dumps(parsed_data)
@@ -315,7 +302,7 @@ class StockDatabase:
             self._create_ticker_table(symbol)  # Ensure the table exists
 
             # Fetch OHLC data from the API
-            url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&apikey={api_key}"
+            url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&to={end_date}&apikey={api_key}"
             async with session.get(url) as response:
                 data = await response.text()
             

app/cron_ownership_stats.py

@@ -7,6 +7,7 @@ import asyncio
 import aiohttp
 import random
 from tqdm import tqdm
+from utils.helper import get_last_completed_quarter
 
 from dotenv import load_dotenv
 import os
@@ -16,11 +17,11 @@ load_dotenv()
 api_key = os.getenv('FMP_API_KEY')
 
 
-include_current_quarter = True
+quarter, year = get_last_completed_quarter()
 
 
 async def get_data(session, symbol, max_retries=3, initial_delay=1):
-    url = f"https://financialmodelingprep.com/api/v4/institutional-ownership/symbol-ownership?symbol={symbol}&includeCurrentQuarter={include_current_quarter}&apikey={api_key}"
+    url = f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}"
     
     for attempt in range(max_retries):
         try:
@@ -31,6 +32,7 @@ async def get_data(session, symbol, max_retries=3, initial_delay=1):
                         data = await response.json()
                         if len(data) > 0:
                             await save_json(symbol, data[0])
+                            print(data[0])
                         return
                     else:
                         print(f"Unexpected content type for {symbol}: {content_type}")

app/cron_shareholders.py

@@ -3,12 +3,10 @@ import sqlite3
 import asyncio
 import pandas as pd
 from tqdm import tqdm
-
 import requests
 import re
 
 
-
 async def save_as_json(symbol, data):
     with open(f"json/shareholders/{symbol}.json", 'w') as file:
         ujson.dump(data, file)
@@ -28,16 +26,9 @@ async def get_data(ticker, con):
     try:
         df = pd.read_sql_query(query_template, con, params=(ticker,))
         shareholders_list = ujson.loads(df.to_dict()['shareholders'][0])
-        # Keys to keep
-        keys_to_keep = ["cik","ownership", "investorName", "changeInSharesNumberPercentage", "weight", "sharesNumber", "marketValue"]
 
-        # Create new list with only the specified keys
-        shareholders_list = [
-            {key: d[key] for key in keys_to_keep}
-            for d in shareholders_list
-        ]
     except Exception as e:
-        #print(e)
+        print(e)
         shareholders_list = []
 
     return shareholders_list

app/utils/helper.py

@@ -77,4 +77,19 @@ def load_latest_json(directory: str, find=True):
 
     except Exception as e:
         print(f"Error loading JSON file: {e}")
-        return []
+        return []
+
+
+def get_last_completed_quarter():
+    today = datetime.today()
+    year = today.year
+    month = today.month
+    # Calculate the current quarter (1 to 4)
+    current_quarter = (month - 1) // 3 + 1
+
+    # The previous quarter is the last completed quarter.
+    # If we're in Q1, the previous quarter is Q4 of last year.
+    if current_quarter == 1:
+        return 4, year - 1
+    else:
+        return current_quarter - 1, year