From d6f04a5afb187b522631a8938c21b9c860c8ba53 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Wed, 9 Oct 2024 20:51:58 +0200 Subject: [PATCH] update primary cron schedule --- app/cron_share_statistics.py | 82 ++------- app/primary_cron_job.py | 310 ++--------------------------------- 2 files changed, 24 insertions(+), 368 deletions(-) diff --git a/app/cron_share_statistics.py b/app/cron_share_statistics.py index 47cd4a4..dcfb8b2 100644 --- a/app/cron_share_statistics.py +++ b/app/cron_share_statistics.py @@ -6,8 +6,7 @@ from tqdm import tqdm from datetime import datetime import yfinance as yf import time -import requests -from requests.exceptions import RequestException + async def save_as_json(symbol, forward_pe_dict, short_dict): with open(f"json/share-statistics/{symbol}.json", 'w') as file: @@ -36,74 +35,17 @@ def filter_data_quarterly(data): return filtered_data -def get_yahoo_data(ticker, outstanding_shares, float_shares, max_retries=3): - # Configure yfinance with custom headers - session = requests.Session() - session.headers.update({ - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', - 'Accept-Language': 'en-US,en;q=0.5', - 'Connection': 'keep-alive', - }) +def get_yahoo_data(ticker, outstanding_shares, float_shares): + try: + data_dict = yf.Ticker(ticker).info + forward_pe = round(data_dict['forwardPE'],2) + short_outstanding_percent = round((data_dict['sharesShort']/outstanding_shares)*100,2) + short_float_percent = round((data_dict['sharesShort']/float_shares)*100,2) + return {'forwardPE': forward_pe}, {'sharesShort': data_dict['sharesShort'], 'shortRatio': data_dict['shortRatio'], 'sharesShortPriorMonth': data_dict['sharesShortPriorMonth'], 'shortOutStandingPercent': short_outstanding_percent, 'shortFloatPercent': short_float_percent} + except Exception as e: + print(e) + return {'forwardPE': 0}, {'sharesShort': 0, 'shortRatio': 0, 'sharesShortPriorMonth': 0, 'shortOutStandingPercent': 0, 'shortFloatPercent': 0} - for attempt in range(max_retries): - try: - ticker_obj = yf.Ticker(ticker) - ticker_obj.session = session - data_dict = ticker_obj.info - - # Check if we got the necessary data - if 'forwardPE' not in data_dict or 'sharesShort' not in data_dict: - raise ValueError("Missing required data fields") - - forward_pe = round(data_dict.get('forwardPE', 0), 2) - shares_short = data_dict.get('sharesShort', 0) - short_ratio = data_dict.get('shortRatio', 0) - shares_short_prior_month = data_dict.get('sharesShortPriorMonth', 0) - - # Calculate percentages only if we have valid numbers - if outstanding_shares and outstanding_shares > 0: - short_outstanding_percent = round((shares_short/outstanding_shares)*100, 2) - else: - short_outstanding_percent = 0 - - if float_shares and float_shares > 0: - short_float_percent = round((shares_short/float_shares)*100, 2) - else: - short_float_percent = 0 - - return { - 'forwardPE': forward_pe - }, { - 'sharesShort': shares_short, - 'shortRatio': short_ratio, - 'sharesShortPriorMonth': shares_short_prior_month, - 'shortOutStandingPercent': short_outstanding_percent, - 'shortFloatPercent': short_float_percent - } - - except (RequestException, ValueError) as e: - if attempt == max_retries - 1: # Last attempt - print(f"Failed to fetch data for {ticker} after {max_retries} attempts: {str(e)}") - return {'forwardPE': 0}, { - 'sharesShort': 0, - 'shortRatio': 0, - 'sharesShortPriorMonth': 0, - 'shortOutStandingPercent': 0, - 'shortFloatPercent': 0 - } - else: - print(f"Attempt {attempt + 1} failed for {ticker}, retrying after delay...") - time.sleep(2 ** attempt) # Exponential backoff - except Exception as e: - print(f"Unexpected error for {ticker}: {str(e)}") - return {'forwardPE': 0}, { - 'sharesShort': 0, - 'shortRatio': 0, - 'sharesShortPriorMonth': 0, - 'shortOutStandingPercent': 0, - 'shortFloatPercent': 0 - } async def get_data(ticker, con): @@ -156,7 +98,7 @@ async def run(): counter += 1 if counter % 50 == 0: - print(f"Processed {counter} tickers, waiting for 30 seconds...") + print(f"Processed {counter} tickers, waiting for 60 seconds...") await asyncio.sleep(60) con.close() diff --git a/app/primary_cron_job.py b/app/primary_cron_job.py index 885a0e5..b2cf2bd 100755 --- a/app/primary_cron_job.py +++ b/app/primary_cron_job.py @@ -62,23 +62,11 @@ def run_cron_insider_trading(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_insider_trading.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/insider-trading", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_congress_trading(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_congress_trading.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/congress-trading", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_dividend_list(): week = datetime.today().weekday() @@ -89,126 +77,56 @@ def run_dividend_list(): if week <= 4 and start_time <= current_time < end_time: run_command(["python3", "cron_dividend_kings.py"]) run_command(["python3", "cron_dividend_aristocrats.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/stocks-list", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_cron_var(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_var.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/var", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_cron_sector(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_sector.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/sector", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_cron_industry(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_industry.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/industry", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_analyst_estimate(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_analyst_estimate.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/analyst-estimate", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_shareholders(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_shareholders.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/shareholders", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_share_statistics(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_share_statistics.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/share-statistics", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_retail_volume(): run_command(["python3", "cron_retail_volume.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/retail-volume", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_cron_market_movers(): run_command(["python3", "cron_market_movers.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/market-movers", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) - + def run_cron_market_news(): run_command(["python3", "cron_market_news.py"]) run_command(["python3", "cron_company_news.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/market-news", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_cron_heatmap(): run_command(["python3", "cron_heatmap.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/heatmaps", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_cron_quote(): week = datetime.today().weekday() if week <= 4: run_command(["python3", "cron_quote.py"]) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/quote", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/pre-post-quote", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) - def run_cron_options_flow(): week = datetime.today().weekday() @@ -218,99 +136,55 @@ def run_cron_options_flow(): if week <= 4 and start_time <= current_time < end_time: run_command(["python3", "cron_options_flow.py"]) - - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/options-flow/feed/", - f"root@{useast_ip_address}:/root/backend/app/json/options-flow/feed/" - ] - run_command(command) - + def run_ta_rating(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_ta_rating.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/ta-rating", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_similar_stocks(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_similar_stocks.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/similar-stocks", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_historical_price(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_historical_price.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/historical-price", - f"root@{useast_ip_address}:/root/backend/json" - ] - run_command(command) def run_one_day_price(): week = datetime.today().weekday() if week <= 4: run_command(["python3", "cron_one_day_price.py"]) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/one-day-price/", f"root@{useast_ip_address}:/root/backend/app/json/one-day-price/"] - run_command(command) def run_sec_filings(): week = datetime.today().weekday() if week <= 4: run_command(["python3", "cron_sec_filings.py"]) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/sec-filings", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) def run_executive(): week = datetime.today().weekday() if week <= 4: run_command(["python3", "cron_executive.py"]) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/executives", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) - def run_options_bubble_ticker(): week = datetime.today().weekday() - if week <= 4: + if week <= 5: run_command(["python3", "cron_options_bubble.py"]) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/options-bubble", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) - - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/options-flow/company", f"root@{useast_ip_address}:/root/backend/app/json/options-flow"] - run_command(command) - def run_analyst_rating(): run_command(["python3", "cron_analyst_insight.py"]) run_command(["python3", "cron_analyst_db.py"]) run_command(["python3", "cron_analyst_ticker.py"]) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/analyst", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) def run_market_moods(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_bull_bear_say.py"]) run_command(["python3", "cron_wiim.py"]) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/bull_bear_say", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) - command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/wiim", f"root@{useast_ip_address}:/root/backend/app/json"] - run_command(command) def run_db_schedule_job(): @@ -324,76 +198,33 @@ def run_dark_pool(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_dark_pool.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/dark-pool", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_dark_pool_flow(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_dark_pool_flow.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/dark-pool/flow", - f"root@{useast_ip_address}:/root/backend/app/json/dark-pool" - ] - run_command(command) - def run_market_maker(): run_command(["python3", "cron_market_maker.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/market-maker", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_ownership_stats(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_ownership_stats.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/ownership-stats", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_clinical_trial(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_clinical_trial.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/clinical-trial", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_fda_calendar(): week = datetime.today().weekday() if week <= 5: run_command(["python3", "cron_fda_calendar.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/fda-calendar", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_borrowed_share(): run_command(["python3", "cron_borrowed_share.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/borrowed-share", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) ''' def run_implied_volatility(): @@ -408,65 +239,21 @@ def run_implied_volatility(): def run_options_net_flow(): run_command(["python3", "cron_options_net_flow.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/options-net-flow", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_options_gex(): run_command(["python3", "cron_options_gex.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/options-gex", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) - run_command(["python3", "cron_options_historical_flow.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/options-historical-data", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) - - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/options-chain", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_government_contract(): run_command(["python3", "cron_government_contract.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/government-contract", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_hedge_fund(): run_command(["python3", "cron_hedge_funds.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/hedge-funds", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_dashboard(): run_command(["python3", "cron_dashboard.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/dashboard", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_tracker(): # Run Python scripts @@ -480,120 +267,47 @@ def run_tracker(): for script in scripts: run_command(["python3", script]) - # Rsync commands - rsync_commands = [ - ("/root/backend/app/json/reddit-tracker", "/root/backend/app/json"), - ("/root/backend/app/json/cramer-tracker", "/root/backend/app/json"), - ("/root/backend/app/json/corporate-lobbying/tracker", "/root/backend/app/json/corporate-lobbying"), - ("/root/backend/app/json/sentiment-tracker", "/root/backend/app/json") - ] - - base_command = ["sudo", "rsync", "-avz", "-e", "ssh"] - for source, dest in rsync_commands: - command = base_command + [source, f"root@{useast_ip_address}:{dest}"] - run_command(command) - def run_financial_statements(): run_command(["python3", "cron_financial_statements.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/financial-statements", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) def run_financial_score(): run_command(["python3", "cron_financial_score.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/financial-score", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_market_cap(): run_command(["python3", "cron_market_cap.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/market-cap", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_dividends(): run_command(["python3", "cron_dividends.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/dividends", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_earnings(): run_command(["python3", "cron_earnings.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/earnings", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_fomc_impact(): run_command(["python3", "cron_fomc_impact.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/fomc-impact", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_economy_indicator(): run_command(["python3", "cron_economic_indicator.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/economy-indicator", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_sentiment_analysis(): run_command(["python3", "cron_sentiment_analysis.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/sentiment-analysis", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_price_analysis(): run_command(["python3", "cron_price_analysis.py"]) - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - "/root/backend/app/json/price-analysis", - f"root@{useast_ip_address}:/root/backend/app/json" - ] - run_command(command) + def run_ai_score(): run_command(["python3", "cron_ai_score.py"]) run_command(["python3", "cron_stockdeck.py"]) run_command(["python3", "restart_json.py"]) - - # Define the directories for rsync - directories = ["stockdeck", "stock-screener", "ai-score"] - base_dir = "/root/backend/app/json" - remote_dir = f"root@{useast_ip_address}:{base_dir}" - - # Rsync commands in a loop - for directory in directories: - command = [ - "sudo", "rsync", "-avz", "-e", "ssh", - f"{base_dir}/{directory}", - f"{remote_dir}" - ] - run_command(command) # Create functions to run each schedule in a separate thread def run_threaded(job_func):