AmineTMIRI/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfixing market cap history

Browse Source
This commit is contained in:
MuslemRahimi 2025-02-20 11:07:11 +01:00
parent 98b96ebccb
commit cf81a91abb
1 changed files with 19 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
app/cron_market_cap.py
View File

@ -12,6 +12,13 @@ import os
load_dotenv() load_dotenv()
api_key = os.getenv('FMP_API_KEY') api_key = os.getenv('FMP_API_KEY')
today = datetime.today().strftime('%Y-%m-%d')
years = list(range(1995, datetime.today().year, 5))
dates = [f"{year}-01-01" for year in years] + [today]
print(dates)
async def save_json(symbol, data): async def save_json(symbol, data):
with open(f"json/market-cap/companies/{symbol}.json", 'w') as file: with open(f"json/market-cap/companies/{symbol}.json", 'w') as file:
ujson.dump(data, file) ujson.dump(data, file)
@ -20,16 +27,7 @@ async def save_json(symbol, data):
async def get_data(session, symbol): async def get_data(session, symbol):
res_list = [] res_list = []
start_date = '1990-01-01' start_date = '1990-01-01'
dates = [
'1995-01-01',
'2000-01-01',
'2005-01-01',
'2010-01-01',
'2015-01-01',
'2020-01-01',
'2025-01-01'
]
for end_date in dates: for end_date in dates:
# Construct the API URL # Construct the API URL
@ -55,7 +53,8 @@ async def get_data(session, symbol):
filtered_data = [{k: v for k, v in item.items() if k != 'symbol'} for item in unique_res_list] filtered_data = [{k: v for k, v in item.items() if k != 'symbol'} for item in unique_res_list]
# Save the filtered data # Save the filtered data
await save_json(symbol, filtered_data) if filtered_data:
await save_json(symbol, filtered_data)
async def run(): async def run():
con = sqlite3.connect('stocks.db') con = sqlite3.connect('stocks.db')
@ -68,12 +67,15 @@ async def run():
async with aiohttp.ClientSession() as session: async with aiohttp.ClientSession() as session:
tasks = [] tasks = []
for i, symbol in enumerate(tqdm(symbols), 1): for i, symbol in enumerate(tqdm(symbols), 1):
tasks.append(get_data(session, symbol)) try:
if i % 100 == 0: tasks.append(get_data(session, symbol))
await asyncio.gather(*tasks) if i % 100 == 0:
tasks = [] await asyncio.gather(*tasks)
print(f'sleeping mode: {i}') tasks = []
await asyncio.sleep(60) # Pause for 60 seconds print(f'sleeping mode: {i}')
await asyncio.sleep(60) # Pause for 60 seconds
except:
pass
if tasks: if tasks:
await asyncio.gather(*tasks) await asyncio.gather(*tasks)