update primary cron schedule

app/cron_share_statistics.py

@@ -6,8 +6,7 @@ from tqdm import tqdm
 from datetime import datetime
 import yfinance as yf
 import time
-import requests
-from requests.exceptions import RequestException
+
 
 async def save_as_json(symbol, forward_pe_dict, short_dict):
     with open(f"json/share-statistics/{symbol}.json", 'w') as file:
@@ -36,74 +35,17 @@ def filter_data_quarterly(data):
     
     return filtered_data
 
-def get_yahoo_data(ticker, outstanding_shares, float_shares, max_retries=3):
-    # Configure yfinance with custom headers
-    session = requests.Session()
-    session.headers.update({
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-        'Accept-Language': 'en-US,en;q=0.5',
-        'Connection': 'keep-alive',
-    })
+def get_yahoo_data(ticker, outstanding_shares, float_shares):
+    try:
+        data_dict = yf.Ticker(ticker).info
+        forward_pe = round(data_dict['forwardPE'],2)
+        short_outstanding_percent = round((data_dict['sharesShort']/outstanding_shares)*100,2)
+        short_float_percent = round((data_dict['sharesShort']/float_shares)*100,2)
+        return {'forwardPE': forward_pe}, {'sharesShort': data_dict['sharesShort'], 'shortRatio': data_dict['shortRatio'], 'sharesShortPriorMonth': data_dict['sharesShortPriorMonth'], 'shortOutStandingPercent': short_outstanding_percent, 'shortFloatPercent': short_float_percent}
+    except Exception as e:
+        print(e)
+        return {'forwardPE': 0}, {'sharesShort': 0, 'shortRatio': 0, 'sharesShortPriorMonth': 0, 'shortOutStandingPercent': 0, 'shortFloatPercent': 0}
 
-    for attempt in range(max_retries):
-        try:
-            ticker_obj = yf.Ticker(ticker)
-            ticker_obj.session = session
-            data_dict = ticker_obj.info
-            
-            # Check if we got the necessary data
-            if 'forwardPE' not in data_dict or 'sharesShort' not in data_dict:
-                raise ValueError("Missing required data fields")
-
-            forward_pe = round(data_dict.get('forwardPE', 0), 2)
-            shares_short = data_dict.get('sharesShort', 0)
-            short_ratio = data_dict.get('shortRatio', 0)
-            shares_short_prior_month = data_dict.get('sharesShortPriorMonth', 0)
-
-            # Calculate percentages only if we have valid numbers
-            if outstanding_shares and outstanding_shares > 0:
-                short_outstanding_percent = round((shares_short/outstanding_shares)*100, 2)
-            else:
-                short_outstanding_percent = 0
-
-            if float_shares and float_shares > 0:
-                short_float_percent = round((shares_short/float_shares)*100, 2)
-            else:
-                short_float_percent = 0
-
-            return {
-                'forwardPE': forward_pe
-            }, {
-                'sharesShort': shares_short,
-                'shortRatio': short_ratio,
-                'sharesShortPriorMonth': shares_short_prior_month,
-                'shortOutStandingPercent': short_outstanding_percent,
-                'shortFloatPercent': short_float_percent
-            }
-
-        except (RequestException, ValueError) as e:
-            if attempt == max_retries - 1:  # Last attempt
-                print(f"Failed to fetch data for {ticker} after {max_retries} attempts: {str(e)}")
-                return {'forwardPE': 0}, {
-                    'sharesShort': 0,
-                    'shortRatio': 0,
-                    'sharesShortPriorMonth': 0,
-                    'shortOutStandingPercent': 0,
-                    'shortFloatPercent': 0
-                }
-            else:
-                print(f"Attempt {attempt + 1} failed for {ticker}, retrying after delay...")
-                time.sleep(2 ** attempt)  # Exponential backoff
-        except Exception as e:
-            print(f"Unexpected error for {ticker}: {str(e)}")
-            return {'forwardPE': 0}, {
-                'sharesShort': 0,
-                'shortRatio': 0,
-                'sharesShortPriorMonth': 0,
-                'shortOutStandingPercent': 0,
-                'shortFloatPercent': 0
-            }
 
 async def get_data(ticker, con):
 
@@ -156,7 +98,7 @@ async def run():
 
         counter += 1
         if counter % 50 == 0:
-            print(f"Processed {counter} tickers, waiting for 30 seconds...")
+            print(f"Processed {counter} tickers, waiting for 60 seconds...")
             await asyncio.sleep(60)
 
     con.close()

app/primary_cron_job.py

@@ -62,23 +62,11 @@ def run_cron_insider_trading():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_insider_trading.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/insider-trading",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_congress_trading():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_congress_trading.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/congress-trading",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_dividend_list():
     week = datetime.today().weekday()
@@ -89,126 +77,56 @@ def run_dividend_list():
     if week <= 4 and start_time <= current_time < end_time:
         run_command(["python3", "cron_dividend_kings.py"])
         run_command(["python3", "cron_dividend_aristocrats.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/stocks-list",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_cron_var():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_var.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/var",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_cron_sector():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_sector.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/sector",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_cron_industry():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_industry.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/industry",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_analyst_estimate():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_analyst_estimate.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/analyst-estimate",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_shareholders():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_shareholders.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/shareholders",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_share_statistics():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_share_statistics.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/share-statistics",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_retail_volume():
     run_command(["python3", "cron_retail_volume.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/retail-volume",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_cron_market_movers():
     run_command(["python3", "cron_market_movers.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/market-movers",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
-    
+
 def run_cron_market_news():
     run_command(["python3", "cron_market_news.py"])
     run_command(["python3", "cron_company_news.py"])
-    command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/market-news",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_cron_heatmap():
     run_command(["python3", "cron_heatmap.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/heatmaps",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 def run_cron_quote():
     week = datetime.today().weekday()
     if week <= 4:
         run_command(["python3", "cron_quote.py"])
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/quote", f"root@{useast_ip_address}:/root/backend/app/json"]
-        run_command(command)
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/pre-post-quote", f"root@{useast_ip_address}:/root/backend/app/json"]
-        run_command(command)
-
 
 def run_cron_options_flow():
     week = datetime.today().weekday()
@@ -218,99 +136,55 @@ def run_cron_options_flow():
 
     if week <= 4 and start_time <= current_time < end_time:
         run_command(["python3", "cron_options_flow.py"])
-        
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/options-flow/feed/",
-            f"root@{useast_ip_address}:/root/backend/app/json/options-flow/feed/"
-        ]
-        run_command(command)
-        
+
         
 def run_ta_rating():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_ta_rating.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/ta-rating",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 
 def run_similar_stocks():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_similar_stocks.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/similar-stocks",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_historical_price():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_historical_price.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/historical-price",
-            f"root@{useast_ip_address}:/root/backend/json"
-        ]
-        run_command(command)
 
 def run_one_day_price():
     week = datetime.today().weekday()
     if week <= 4:
         run_command(["python3", "cron_one_day_price.py"])
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/one-day-price/", f"root@{useast_ip_address}:/root/backend/app/json/one-day-price/"]
-        run_command(command)
 
 def run_sec_filings():
     week = datetime.today().weekday()
     if week <= 4:
         run_command(["python3", "cron_sec_filings.py"])
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/sec-filings", f"root@{useast_ip_address}:/root/backend/app/json"]
-        run_command(command)
 
 def run_executive():
     week = datetime.today().weekday()
     if week <= 4:
         run_command(["python3", "cron_executive.py"])
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/executives", f"root@{useast_ip_address}:/root/backend/app/json"]
-        run_command(command)
-
 
 def run_options_bubble_ticker():
     week = datetime.today().weekday()
-    if week <= 4:
+    if week <= 5:
         run_command(["python3", "cron_options_bubble.py"])
 
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/options-bubble", f"root@{useast_ip_address}:/root/backend/app/json"]
-        run_command(command)
-
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/options-flow/company", f"root@{useast_ip_address}:/root/backend/app/json/options-flow"]
-        run_command(command)
-
 
 def run_analyst_rating():
     run_command(["python3", "cron_analyst_insight.py"])
     run_command(["python3", "cron_analyst_db.py"])
     run_command(["python3", "cron_analyst_ticker.py"])
-    command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/analyst", f"root@{useast_ip_address}:/root/backend/app/json"]
-    run_command(command)
 
 def run_market_moods():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_bull_bear_say.py"])
         run_command(["python3", "cron_wiim.py"])
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/bull_bear_say", f"root@{useast_ip_address}:/root/backend/app/json"]
-        run_command(command)
-        command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/wiim", f"root@{useast_ip_address}:/root/backend/app/json"]
-        run_command(command)
 
 
 def run_db_schedule_job():
@@ -324,76 +198,33 @@ def run_dark_pool():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_dark_pool.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/dark-pool",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 
 def run_dark_pool_flow():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_dark_pool_flow.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/dark-pool/flow",
-            f"root@{useast_ip_address}:/root/backend/app/json/dark-pool"
-        ]
-        run_command(command)
-
 
 def run_market_maker():
     run_command(["python3", "cron_market_maker.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/market-maker",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_ownership_stats():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_ownership_stats.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/ownership-stats",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_clinical_trial():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_clinical_trial.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/clinical-trial",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_fda_calendar():
     week = datetime.today().weekday()
     if week <= 5:
         run_command(["python3", "cron_fda_calendar.py"])
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            "/root/backend/app/json/fda-calendar",
-            f"root@{useast_ip_address}:/root/backend/app/json"
-        ]
-        run_command(command)
 
 def run_borrowed_share():
     run_command(["python3", "cron_borrowed_share.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/borrowed-share",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 '''
 def run_implied_volatility():
@@ -408,65 +239,21 @@ def run_implied_volatility():
 
 def run_options_net_flow():
     run_command(["python3", "cron_options_net_flow.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/options-net-flow",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_options_gex():
     run_command(["python3", "cron_options_gex.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/options-gex",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
-
     run_command(["python3", "cron_options_historical_flow.py"])
     
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/options-historical-data",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
-
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/options-chain",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 
 def run_government_contract():
     run_command(["python3", "cron_government_contract.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/government-contract",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_hedge_fund():
     run_command(["python3", "cron_hedge_funds.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/hedge-funds",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_dashboard():
     run_command(["python3", "cron_dashboard.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/dashboard",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_tracker():
     # Run Python scripts
@@ -480,120 +267,47 @@ def run_tracker():
     for script in scripts:
         run_command(["python3", script])
 
-    # Rsync commands
-    rsync_commands = [
-        ("/root/backend/app/json/reddit-tracker", "/root/backend/app/json"),
-        ("/root/backend/app/json/cramer-tracker", "/root/backend/app/json"),
-        ("/root/backend/app/json/corporate-lobbying/tracker", "/root/backend/app/json/corporate-lobbying"),
-        ("/root/backend/app/json/sentiment-tracker", "/root/backend/app/json")
-    ]
-
-    base_command = ["sudo", "rsync", "-avz", "-e", "ssh"]
-    for source, dest in rsync_commands:
-        command = base_command + [source, f"root@{useast_ip_address}:{dest}"]
-        run_command(command)
-
 def run_financial_statements():
     run_command(["python3", "cron_financial_statements.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/financial-statements",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
 
 def run_financial_score():
     run_command(["python3", "cron_financial_score.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/financial-score",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+ 
 
 def run_market_cap():
     run_command(["python3", "cron_market_cap.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/market-cap",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 
 def run_dividends():
     run_command(["python3", "cron_dividends.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/dividends",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 def run_earnings():
     run_command(["python3", "cron_earnings.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/earnings",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 def run_fomc_impact():
     run_command(["python3", "cron_fomc_impact.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/fomc-impact",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 def run_economy_indicator():
     run_command(["python3", "cron_economic_indicator.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/economy-indicator",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 
 def run_sentiment_analysis():
     run_command(["python3", "cron_sentiment_analysis.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/sentiment-analysis",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 def run_price_analysis():
     run_command(["python3", "cron_price_analysis.py"])
-    command = [
-        "sudo", "rsync", "-avz", "-e", "ssh",
-        "/root/backend/app/json/price-analysis",
-        f"root@{useast_ip_address}:/root/backend/app/json"
-    ]
-    run_command(command)
+
 
 def run_ai_score():
     run_command(["python3", "cron_ai_score.py"])
     run_command(["python3", "cron_stockdeck.py"])
     run_command(["python3", "restart_json.py"])
-    
-    # Define the directories for rsync
-    directories = ["stockdeck", "stock-screener", "ai-score"]
-    base_dir = "/root/backend/app/json"
-    remote_dir = f"root@{useast_ip_address}:{base_dir}"
-
-    # Rsync commands in a loop
-    for directory in directories:
-        command = [
-            "sudo", "rsync", "-avz", "-e", "ssh",
-            f"{base_dir}/{directory}",
-            f"{remote_dir}"
-        ]
-        run_command(command)
 
 # Create functions to run each schedule in a separate thread
 def run_threaded(job_func):