bugfixing
This commit is contained in:
MuslemRahimi
parent
0bdf102af2
commit
18b58f3f3b
@ -9,6 +9,7 @@ from tqdm import tqdm
|
||||
import re
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from utils.helper import get_last_completed_quarter
|
||||
|
||||
import warnings
|
||||
|
||||
@ -25,24 +26,10 @@ warnings.filterwarnings("ignore", category=RuntimeWarning, message="invalid valu
|
||||
start_date = datetime(2015, 1, 1).strftime("%Y-%m-%d")
|
||||
end_date = datetime.today().strftime("%Y-%m-%d")
|
||||
|
||||
def get_last_completed_quarter():
|
||||
today = datetime.today()
|
||||
year = today.year
|
||||
month = today.month
|
||||
# Calculate the current quarter (1 to 4)
|
||||
current_quarter = (month - 1) // 3 + 1
|
||||
|
||||
# The previous quarter is the last completed quarter.
|
||||
# If we're in Q1, the previous quarter is Q4 of last year.
|
||||
if current_quarter == 1:
|
||||
return 4, year - 1
|
||||
else:
|
||||
return current_quarter - 1, year
|
||||
|
||||
# Get last completed quarter and its year
|
||||
quarter, year = get_last_completed_quarter()
|
||||
|
||||
|
||||
|
||||
if os.path.exists("backup_db/etf.db"):
|
||||
os.remove('backup_db/etf.db')
|
||||
|
||||
@ -204,7 +191,7 @@ class ETFDatabase:
|
||||
f"https://financialmodelingprep.com/api/v3/etf-country-weightings/{symbol}?apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/api/v3/quote/{symbol}?apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/stable/dividends?symbol={symbol}&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/stable/institutional-ownership/extract-analytics/holder?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
|
||||
]
|
||||
|
||||
fundamental_data = {}
|
||||
|
||||
@ -14,6 +14,7 @@ from ta.momentum import *
|
||||
from ta.trend import *
|
||||
from ta.volume import *
|
||||
import warnings
|
||||
from utils.helper import get_last_completed_quarter
|
||||
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
@ -26,24 +27,9 @@ warnings.filterwarnings("ignore", category=RuntimeWarning, message="invalid valu
|
||||
|
||||
start_date = datetime(2015, 1, 1).strftime("%Y-%m-%d")
|
||||
end_date = datetime.today().strftime("%Y-%m-%d")
|
||||
|
||||
def get_last_completed_quarter():
|
||||
today = datetime.today()
|
||||
year = today.year
|
||||
month = today.month
|
||||
# Calculate the current quarter (1 to 4)
|
||||
current_quarter = (month - 1) // 3 + 1
|
||||
|
||||
# The previous quarter is the last completed quarter.
|
||||
# If we're in Q1, the previous quarter is Q4 of last year.
|
||||
if current_quarter == 1:
|
||||
return 4, year - 1
|
||||
else:
|
||||
return current_quarter - 1, year
|
||||
|
||||
# Get last completed quarter and its year
|
||||
quarter, year = get_last_completed_quarter()
|
||||
|
||||
|
||||
if os.path.exists("backup_db/stocks.db"):
|
||||
os.remove('backup_db/stocks.db')
|
||||
|
||||
@ -118,7 +104,7 @@ class StockDatabase:
|
||||
f"https://financialmodelingprep.com/api/v4/historical/employee_count?symbol={symbol}&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/api/v3/historical-price-full/stock_split/{symbol}?apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/api/v4/stock_peers?symbol={symbol}&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/stable/institutional-ownership/extract-analytics/holder?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/api/v4/historical/shares_float?symbol={symbol}&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/api/v4/revenue-product-segmentation?symbol={symbol}&structure=flat&period=annual&apikey={api_key}",
|
||||
f"https://financialmodelingprep.com/api/v4/revenue-geographic-segmentation?symbol={symbol}&structure=flat&apikey={api_key}",
|
||||
@ -182,6 +168,7 @@ class StockDatabase:
|
||||
elif "institutional-ownership" in url:
|
||||
# Handle list response, save as JSON object
|
||||
fundamental_data['shareholders'] = ujson.dumps(parsed_data)
|
||||
|
||||
elif "historical/shares_float" in url:
|
||||
# Handle list response, save as JSON object
|
||||
fundamental_data['historicalShares'] = ujson.dumps(parsed_data)
|
||||
@ -315,7 +302,7 @@ class StockDatabase:
|
||||
self._create_ticker_table(symbol) # Ensure the table exists
|
||||
|
||||
# Fetch OHLC data from the API
|
||||
url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&apikey={api_key}"
|
||||
url = f"https://financialmodelingprep.com/api/v3/historical-price-full/{symbol}?serietype=bar&from={start_date}&to={end_date}&apikey={api_key}"
|
||||
async with session.get(url) as response:
|
||||
data = await response.text()
|
||||
|
||||
|
||||
@ -7,6 +7,7 @@ import asyncio
|
||||
import aiohttp
|
||||
import random
|
||||
from tqdm import tqdm
|
||||
from utils.helper import get_last_completed_quarter
|
||||
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
@ -16,11 +17,11 @@ load_dotenv()
|
||||
api_key = os.getenv('FMP_API_KEY')
|
||||
|
||||
|
||||
include_current_quarter = True
|
||||
quarter, year = get_last_completed_quarter()
|
||||
|
||||
|
||||
async def get_data(session, symbol, max_retries=3, initial_delay=1):
|
||||
url = f"https://financialmodelingprep.com/api/v4/institutional-ownership/symbol-ownership?symbol={symbol}&includeCurrentQuarter={include_current_quarter}&apikey={api_key}"
|
||||
url = f"https://financialmodelingprep.com/stable/institutional-ownership/symbol-positions-summary?symbol={symbol}&year={year}&quarter={quarter}&apikey={api_key}"
|
||||
|
||||
for attempt in range(max_retries):
|
||||
try:
|
||||
@ -31,6 +32,7 @@ async def get_data(session, symbol, max_retries=3, initial_delay=1):
|
||||
data = await response.json()
|
||||
if len(data) > 0:
|
||||
await save_json(symbol, data[0])
|
||||
print(data[0])
|
||||
return
|
||||
else:
|
||||
print(f"Unexpected content type for {symbol}: {content_type}")
|
||||
|
||||
@ -3,12 +3,10 @@ import sqlite3
|
||||
import asyncio
|
||||
import pandas as pd
|
||||
from tqdm import tqdm
|
||||
|
||||
import requests
|
||||
import re
|
||||
|
||||
|
||||
|
||||
async def save_as_json(symbol, data):
|
||||
with open(f"json/shareholders/{symbol}.json", 'w') as file:
|
||||
ujson.dump(data, file)
|
||||
@ -28,16 +26,9 @@ async def get_data(ticker, con):
|
||||
try:
|
||||
df = pd.read_sql_query(query_template, con, params=(ticker,))
|
||||
shareholders_list = ujson.loads(df.to_dict()['shareholders'][0])
|
||||
# Keys to keep
|
||||
keys_to_keep = ["cik","ownership", "investorName", "changeInSharesNumberPercentage", "weight", "sharesNumber", "marketValue"]
|
||||
|
||||
# Create new list with only the specified keys
|
||||
shareholders_list = [
|
||||
{key: d[key] for key in keys_to_keep}
|
||||
for d in shareholders_list
|
||||
]
|
||||
except Exception as e:
|
||||
#print(e)
|
||||
print(e)
|
||||
shareholders_list = []
|
||||
|
||||
return shareholders_list
|
||||
|
||||
@ -77,4 +77,19 @@ def load_latest_json(directory: str, find=True):
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error loading JSON file: {e}")
|
||||
return []
|
||||
return []
|
||||
|
||||
|
||||
def get_last_completed_quarter():
|
||||
today = datetime.today()
|
||||
year = today.year
|
||||
month = today.month
|
||||
# Calculate the current quarter (1 to 4)
|
||||
current_quarter = (month - 1) // 3 + 1
|
||||
|
||||
# The previous quarter is the last completed quarter.
|
||||
# If we're in Q1, the previous quarter is Q4 of last year.
|
||||
if current_quarter == 1:
|
||||
return 4, year - 1
|
||||
else:
|
||||
return current_quarter - 1, year
|
||||
Loading…
x
Reference in New Issue
Block a user