AmineTMIRI/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add loggin to cron job

Browse Source
This commit is contained in:
MuslemRahimi 2024-07-09 17:29:30 +02:00
parent 991184a00f
commit d46906438b
2 changed files with 104 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -9,6 +9,7 @@ app/*.sh
app/.env*
app/ml_models/weights
app/json/*
app/logs/*
fastify/node_modules
pocketbase/*
helper.txt

183
app/primary_cron_job.py
View File

@ -14,6 +14,8 @@ import sqlite3
import pandas as pd
import numpy as np
import threading # Import threading module for parallel execution
import logging # Import logging module
from logging.handlers import RotatingFileHandler
from dotenv import load_dotenv
@ -23,372 +25,393 @@ load_dotenv()
useast_ip_address = os.getenv('USEAST_IP_ADDRESS')
# Setup logging
log_file = 'logs/cron_job.log'
logger = logging.getLogger('CronJobLogger')
logger.setLevel(logging.INFO)
handler = RotatingFileHandler(log_file, maxBytes=5*1024*1024, backupCount=5) # 5MB per file, 5 backup files
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
# Set the system's timezone to Berlin at the beginning
subprocess.run(["timedatectl", "set-timezone", "Europe/Berlin"])
# Function to run commands and log output
def run_command(command):
process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
stdout, stderr = process.communicate()
# Log stdout and stderr
logger.info(f"Command: {' '.join(command)}")
logger.info("Output:\n" + stdout)
if stderr:
logger.error("Error:\n" + stderr)
def run_json_job():
# Run the asynchronous function inside an asyncio loop
subprocess.run(["python3", "restart_json.py"])
subprocess.run(["pm2", "restart","fastapi"])
run_command(["python3", "restart_json.py"])
run_command(["pm2", "restart","fastapi"])
def run_pocketbase():
# Run the asynchronous function inside an asyncio loop
subprocess.run(["python3", "cron_pocketbase.py"])
run_command(["python3", "cron_pocketbase.py"])
def run_cron_insider_trading():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_insider_trading.py"])
run_command(["python3", "cron_insider_trading.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/insider-trading",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_congress_trading():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_congress_trading.py"])
run_command(["python3", "cron_congress_trading.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/congress-trading",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_cron_var():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_var.py"])
run_command(["python3", "cron_var.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/var",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_analyst_estimate():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_analyst_estimate.py"])
run_command(["python3", "cron_analyst_estimate.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/analyst-estimate",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_shareholders():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_shareholders.py"])
run_command(["python3", "cron_shareholders.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/shareholders",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_share_statistics():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_share_statistics.py"])
run_command(["python3", "cron_share_statistics.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/share-statistics",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_retail_volume():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_retail_volume.py"])
run_command(["python3", "cron_retail_volume.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/retail-volume",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_cron_market_movers():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_market_movers.py"])
run_command(["python3", "cron_market_movers.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/market-movers",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_cron_market_news():
subprocess.run(["python3", "cron_market_news.py"])
run_command(["python3", "cron_market_news.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/market-news",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_cron_heatmap():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_heatmap.py"])
run_command(["python3", "cron_heatmap.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/heatmaps",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_cron_quote():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_quote.py"])
run_command(["python3", "cron_quote.py"])
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/quote", f"root@{useast_ip_address}:/root/backend/app/json"]
subprocess.run(command)
run_command(command)
def run_cron_price_alert():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_price_alert.py"])
run_command(["python3", "cron_price_alert.py"])
def run_cron_portfolio():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_portfolio.py"])
run_command(["python3", "cron_portfolio.py"])
def run_cron_options_flow():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_options_flow.py"])
run_command(["python3", "cron_options_flow.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/options-flow/feed/",
f"root@{useast_ip_address}:/root/backend/app/json/options-flow/feed/"
]
subprocess.run(command)
run_command(command)
def run_cron_options_zero_dte():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_options_zero_dte.py"])
run_command(["python3", "cron_options_zero_dte.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/options-flow/zero-dte/",
f"root@{useast_ip_address}:/root/backend/app/json/options-flow/zero-dte/"
]
subprocess.run(command)
run_command(command)
def run_ta_rating():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_ta_rating.py"])
run_command(["python3", "cron_ta_rating.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/ta-rating",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_stockdeck():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_stockdeck.py"])
run_command(["python3", "cron_stockdeck.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/stockdeck",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_similar_stocks():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_similar_stocks.py"])
run_command(["python3", "cron_similar_stocks.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/similar-stocks",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_historical_price():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_historical_price.py"])
run_command(["python3", "cron_historical_price.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/historical-price",
f"root@{useast_ip_address}:/root/backend/json"
]
subprocess.run(command)
run_command(command)
def run_one_day_price():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_one_day_price.py"])
run_command(["python3", "cron_one_day_price.py"])
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/one-day-price/", f"root@{useast_ip_address}:/root/backend/app/json/one-day-price/"]
subprocess.run(command)
run_command(command)
def run_sec_filings():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_sec_filings.py"])
run_command(["python3", "cron_sec_filings.py"])
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/sec-filings", f"root@{useast_ip_address}:/root/backend/app/json"]
subprocess.run(command)
run_command(command)
def run_executive():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_executive.py"])
run_command(["python3", "cron_executive.py"])
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/executives", f"root@{useast_ip_address}:/root/backend/app/json"]
subprocess.run(command)
run_command(command)
def run_options_bubble_ticker():
week = datetime.today().weekday()
if week <= 4:
subprocess.run(["python3", "cron_options_bubble.py"])
run_command(["python3", "cron_options_bubble.py"])
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/options-bubble/", f"root@{useast_ip_address}:/root/backend/app/json/options-bubble/"]
subprocess.run(command)
run_command(command)
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/options-flow/company/", f"root@{useast_ip_address}:/root/backend/app/json/options-flow/company/"]
subprocess.run(command)
run_command(command)
def run_analyst_rating():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_analyst_db.py"])
subprocess.run(["python3", "cron_analyst_ticker.py"])
subprocess.run(["python3", "cron_analyst_insight.py"])
run_command(["python3", "cron_analyst_db.py"])
run_command(["python3", "cron_analyst_ticker.py"])
run_command(["python3", "cron_analyst_insight.py"])
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/analyst", f"root@{useast_ip_address}:/root/backend/app/json"]
subprocess.run(command)
run_command(command)
def run_market_moods():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_bull_bear_say.py"])
subprocess.run(["python3", "cron_wiim.py"])
run_command(["python3", "cron_bull_bear_say.py"])
run_command(["python3", "cron_wiim.py"])
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/bull_bear_say", f"root@{useast_ip_address}:/root/backend/app/json"]
subprocess.run(command)
run_command(command)
command = ["sudo", "rsync", "-avz", "-e", "ssh", "/root/backend/app/json/wiim", f"root@{useast_ip_address}:/root/backend/app/json"]
subprocess.run(command)
run_command(command)
def run_db_schedule_job():
#update db daily
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["bash", "run_universe.sh"])
run_command(["bash", "run_universe.sh"])
def run_restart_cache():
#update db daily
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["pm2", "restart","fastapi"])
subprocess.run(["pm2", "restart","fastify"])
#subprocess.run(["python3", "cache_endpoints.py"])
run_command(["pm2", "restart","fastapi"])
run_command(["pm2", "restart","fastify"])
#run_command(["python3", "cache_endpoints.py"])
def run_dark_pool():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_dark_pool.py"])
run_command(["python3", "cron_dark_pool.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/dark-pool",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_market_maker():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_market_maker.py"])
run_command(["python3", "cron_market_maker.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/market-maker",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_ownership_stats():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_ownership_stats.py"])
run_command(["python3", "cron_ownership_stats.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/ownership-stats",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_clinical_trial():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_clinical_trial.py"])
run_command(["python3", "cron_clinical_trial.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/clinical-trial",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_fda_calendar():
week = datetime.today().weekday()
if week <= 5:
subprocess.run(["python3", "cron_fda_calendar.py"])
run_command(["python3", "cron_fda_calendar.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/fda-calendar",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_borrowed_share():
subprocess.run(["python3", "cron_borrowed_share.py"])
run_command(["python3", "cron_borrowed_share.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/borrowed-share",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_implied_volatility():
subprocess.run(["python3", "cron_implied_volatility.py"])
run_command(["python3", "cron_implied_volatility.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/implied-volatility",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_options_net_flow():
subprocess.run(["python3", "cron_options_net_flow.py"])
run_command(["python3", "cron_options_net_flow.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/options-net-flow",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
def run_government_contract():
subprocess.run(["python3", "cron_government_contract.py"])
run_command(["python3", "cron_government_contract.py"])
command = [
"sudo", "rsync", "-avz", "-e", "ssh",
"/root/backend/app/json/government-contract",
f"root@{useast_ip_address}:/root/backend/app/json"
]
subprocess.run(command)
run_command(command)
# Create functions to run each schedule in a separate thread
def run_threaded(job_func):
@ -436,14 +459,14 @@ schedule.every(5).minutes.do(run_threaded, run_cron_market_movers).tag('market_m
schedule.every(15).minutes.do(run_threaded, run_cron_market_news).tag('market_news_job')
schedule.every(10).minutes.do(run_threaded, run_one_day_price).tag('one_day_price_job')
schedule.every(5).minutes.do(run_threaded, run_cron_heatmap).tag('heatmap_job')
schedule.every(15).minutes.do(run_threaded, run_cron_heatmap).tag('heatmap_job')
schedule.every(1).minutes.do(run_threaded, run_cron_quote).tag('quote_job')
schedule.every(1).minutes.do(run_threaded, run_cron_price_alert).tag('price_alert_job')
schedule.every(15).minutes.do(run_threaded, run_market_moods).tag('market_moods_job')
schedule.every(2).hours.do(run_threaded, run_fda_calendar).tag('fda_calendar_job')
schedule.every(3).hours.do(run_threaded, run_json_job).tag('json_job')
schedule.every(6).hours.do(run_threaded, run_analyst_rating).tag('analyst_job')
schedule.every(12).hours.do(run_threaded, run_analyst_rating).tag('analyst_job')
schedule.every(10).seconds.do(run_threaded, run_cron_options_flow).tag('options_flow_job')
schedule.every(10).seconds.do(run_threaded, run_cron_options_zero_dte).tag('options_zero_dte_job')