From a212a3614213ae29f25058e44a8d6dcfe3cc4b51 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Thu, 28 Nov 2024 19:02:01 +0100 Subject: [PATCH] refactor news cron job to reduce bandwidth --- app/cron_company_news.py | 2 +- app/cron_market_news.py | 2 +- app/primary_cron_job.py | 18 ++++++++++++------ 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/app/cron_company_news.py b/app/cron_company_news.py index 49bb66a..fa2da38 100644 --- a/app/cron_company_news.py +++ b/app/cron_company_news.py @@ -55,7 +55,7 @@ async def get_data(session, chunk, rate_limiter): """ await rate_limiter.acquire() company_tickers = ','.join(chunk) - url = f'https://financialmodelingprep.com/api/v3/stock_news?tickers={company_tickers}&page=0&limit=2000&apikey={api_key}' + url = f'https://financialmodelingprep.com/api/v3/stock_news?tickers={company_tickers}&page=0&limit=50&apikey={api_key}' async with session.get(url) as response: if response.status == 200: diff --git a/app/cron_market_news.py b/app/cron_market_news.py index 4fe0218..03671da 100755 --- a/app/cron_market_news.py +++ b/app/cron_market_news.py @@ -76,7 +76,7 @@ async def run(): stock_symbols = [row[0] for row in cursor.fetchall()] print(len(stock_symbols)) con.close() - limit = 200 + limit = 100 company_tickers = ','.join(stock_symbols) urls = [ f'https://financialmodelingprep.com/api/v3/stock_news?tickers={company_tickers}&limit={limit}&apikey={api_key}', diff --git a/app/primary_cron_job.py b/app/primary_cron_job.py index b8570e7..ca8a546 100755 --- a/app/primary_cron_job.py +++ b/app/primary_cron_job.py @@ -112,7 +112,11 @@ def run_share_statistics(): def run_cron_market_news(): run_command(["python3", "cron_market_news.py"]) - run_command(["python3", "cron_company_news.py"]) + +def run_cron_company_news(): + week = datetime.today().weekday() + if week <= 5: + run_command(["python3", "cron_company_news.py"]) def run_cron_heatmap(): run_command(["python3", "cron_heatmap.py"]) @@ -290,8 +294,8 @@ schedule.every().day.at("08:00").do(run_threaded, run_economy_indicator).tag('ec schedule.every().day.at("08:00").do(run_threaded, run_cron_insider_trading).tag('insider_trading_job') schedule.every().day.at("08:30").do(run_threaded, run_dividends).tag('dividends_job') schedule.every().day.at("10:00").do(run_threaded, run_shareholders).tag('shareholders_job') -schedule.every().day.at("10:30").do(run_threaded, run_sec_filings).tag('sec_filings_job') -schedule.every().day.at("11:00").do(run_threaded, run_executive).tag('executive_job') +#schedule.every().day.at("10:30").do(run_threaded, run_sec_filings).tag('sec_filings_job') +#schedule.every().day.at("11:00").do(run_threaded, run_executive).tag('executive_job') schedule.every().day.at("12:00").do(run_threaded, run_market_cap).tag('market_cap_josb') #schedule.every().day.at("05:00").do(run_threaded, run_implied_volatility).tag('implied_volatility_job') @@ -305,13 +309,14 @@ schedule.every().day.at("14:00").do(run_threaded, run_cron_sector).tag('sector_j schedule.every(2).days.at("08:30").do(run_threaded, run_financial_score).tag('financial_score_job') schedule.every().saturday.at("05:00").do(run_threaded, run_ownership_stats).tag('ownership_stats_job') -schedule.every().saturday.at("06:00").do(run_threaded, run_sentiment_analysis).tag('sentiment_analysis_job') -schedule.every().saturday.at("10:00").do(run_threaded, run_price_analysis).tag('price_analysis_job') +#schedule.every().saturday.at("06:00").do(run_threaded, run_sentiment_analysis).tag('sentiment_analysis_job') +#schedule.every().saturday.at("10:00").do(run_threaded, run_price_analysis).tag('price_analysis_job') schedule.every(30).minutes.do(run_threaded, run_dividend_list).tag('dividend_list_job') schedule.every(3).hours.do(run_threaded, run_congress_trading).tag('congress_job') -schedule.every(15).minutes.do(run_threaded, run_cron_market_news).tag('market_news_job') +schedule.every(30).minutes.do(run_threaded, run_cron_market_news).tag('market_news_job') + schedule.every(30).minutes.do(run_threaded, run_cron_industry).tag('industry_job') schedule.every(7).minutes.do(run_threaded, run_one_day_price).tag('one_day_price_job') @@ -328,6 +333,7 @@ schedule.every(3).hours.do(run_threaded, run_options_net_flow).tag('options_net_ #schedule.every(4).hours.do(run_threaded, run_share_statistics).tag('share_statistics_job') schedule.every(1).hours.do(run_threaded, run_analyst_rating).tag('analyst_job') +schedule.every(1).hours.do(run_threaded, run_cron_company_news).tag('company_news_job') schedule.every(2).minutes.do(run_threaded, run_dashboard).tag('dashboard_job')