AmineTMIRI/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update cron job for analyst db

Browse Source
This commit is contained in:
MuslemRahimi 2024-11-10 21:22:23 +01:00
parent dbf8f9a3e2
commit aa442a9238
1 changed files with 69 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

103
app/cron_analyst_db.py
View File

@ -4,7 +4,7 @@ import numpy as np
from scipy.stats import norm
import time
import sqlite3
import ujson
import orjson
import os
from dotenv import load_dotenv
from tqdm import tqdm
@ -20,6 +20,11 @@ api_key = os.getenv('BENZINGA_API_KEY')
headers = {"accept": "application/json"}
# Load stock screener data
with open(f"json/stock-screener/data.json", 'rb') as file:
stock_screener_data = orjson.loads(file.read())
stock_screener_data_dict = {item['symbol']: item for item in stock_screener_data}
query_template = """
SELECT date, close
FROM "{ticker}"
@ -41,23 +46,60 @@ def remove_duplicates(data, key):
new_data.append(item)
return new_data
def extract_sector(ticker, con):
query_template = f"""
SELECT
sector
FROM
stocks
WHERE
symbol = ?
"""
try:
df = pd.read_sql_query(query_template, con, params=(ticker,))
sector = df['sector'].iloc[0]
except:
sector = None
return sector
async def get_data(ticker_list, item):
"""Extract specified columns data for a given symbol."""
columns = ['sector', 'industry', 'price']
sector_list = []
industry_list = []
for ticker in ticker_list:
ticker_data = stock_screener_data_dict.get(ticker, {})
# Extract specified columns data for each ticker
sector = ticker_data.get('sector',None)
industry = ticker_data.get('industry',None)
price = ticker_data.get('price',None)
price = round(price, 2) if price is not None else None
# Append data to relevant lists if values are present
if sector:
sector_list.append(sector)
if industry:
industry_list.append(industry)
# Update ratingsList in item with price for the corresponding ticker
try:
if len(item.get('ratingsList')) > 0:
for rating in item.get('ratingsList', []):
try:
if rating.get('ticker') == ticker:
upside = round((float(rating['adjusted_pt_current'])/price-1)*100,2)
rating['price'] = price
rating['upside'] = upside
except:
rating['price'] = None
rating['upside'] = None
except:
pass
# Get the top 3 most common sectors and industries
sector_counts = Counter(sector_list)
industry_counts = Counter(industry_list)
main_sectors = [item[0] for item in sector_counts.most_common(3)]
main_industries = [item[0] for item in industry_counts.most_common(3)]
# Add main sectors and industries to the item dictionary
item['mainSectors'] = main_sectors
item['mainIndustries'] = main_industries
return item
def calculate_rating(data):
overall_average_return = float(data['avgReturn'])
@ -121,7 +163,7 @@ def calculate_rating(data):
def get_top_stocks():
with open(f"json/analyst/all-analyst-data.json", 'r') as file:
analyst_stats_list = ujson.load(file)
analyst_stats_list = orjson.loads(file.read())
filtered_data = [item for item in analyst_stats_list if item['analystScore'] >= 4]
@ -156,7 +198,7 @@ def get_top_stocks():
for ticker, info in ticker_data.items():
try:
with open(f"json/quote/{ticker}.json", 'r') as file:
res = ujson.load(file)
res = orjson.loads(file.read())
info['price'] = res.get('price', None)
info['name'] = res.get('name', None)
info['marketCap'] = res.get('marketCap', None)
@ -188,7 +230,7 @@ def get_top_stocks():
item['rank'] = rank + 1
with open(f"json/analyst/top-stocks.json", 'w') as file:
ujson.dump(result_sorted, file)
file.write(orjson.dumps(result_sorted).decode('utf-8'))
async def get_analyst_ratings(analyst_id, session):
@ -240,7 +282,7 @@ async def get_all_analyst_stats():
for response in responses:
if response.status == 200: # Check for successful response
try:
data = ujson.loads(await response.text())['analyst_ratings_analyst']
data = orjson.loads(await response.text())['analyst_ratings_analyst']
res_list += data
except Exception as e:
pass
@ -381,18 +423,11 @@ async def run():
# Step2: Get rating history for each individual analyst and score the analyst
await get_single_analyst_data(analyst_list, con)
try:
print('Start extracting main sectors')
print('Start extracting data')
for item in tqdm(analyst_list):
ticker_list = [entry['ticker'] for entry in item['ratingsList']]
sector_list = []
for ticker in ticker_list:
sector = extract_sector(ticker, con)
sector_list.append(sector)
sector_counts = Counter(sector_list)
main_sectors = sector_counts.most_common(3)
main_sectors = [item[0] for item in main_sectors if item[0] is not None]
item['mainSectors'] = main_sectors
if len(ticker_list) > 0:
await get_data(ticker_list, item)
except Exception as e:
print(e)
@ -408,7 +443,7 @@ async def run():
item['avgReturn'] = round(float(item['avgReturn']), 2)
item['successRate'] = round(float(item['successRate']), 2)
with open(f"json/analyst/analyst-db/{item['analystId']}.json", 'w') as file:
ujson.dump(item, file)
file.write(orjson.dumps(item).decode('utf-8'))
# Save top 100 analysts
top_analysts_list = []
@ -426,11 +461,11 @@ async def run():
})
with open(f"json/analyst/top-analysts.json", 'w') as file:
ujson.dump(top_analysts_list, file)
file.write(orjson.dumps(top_analysts_list).decode('utf-8'))
# Save all analyst data in raw form for the next step
with open(f"json/analyst/all-analyst-data.json", 'w') as file:
ujson.dump(analyst_list, file)
file.write(orjson.dumps(analyst_list).decode('utf-8'))
# Save top stocks with strong buys from 5-star analysts
get_top_stocks()