From 001ff4ba6743e40f5e9344fe85f7eaa10e2f9304 Mon Sep 17 00:00:00 2001
From: MuslemRahimi <moslem_rahimi@hotmail.de>
Date: Wed, 25 Sep 2024 00:37:47 +0200
Subject: [PATCH] streamline ml training better

---
 app/cron_trend_analysis.py                    |  10 ++++-
 .../classification.cpython-310.pyc            | Bin 7693 -> 8070 bytes
 app/ml_models/classification.py               |  22 +++++++---
 app/primary_cron_job.py                       |  40 ++++++++++++++++++
 4 files changed, 66 insertions(+), 6 deletions(-)

diff --git a/app/cron_trend_analysis.py b/app/cron_trend_analysis.py
index 992bef8..c8267dd 100755
--- a/app/cron_trend_analysis.py
+++ b/app/cron_trend_analysis.py
@@ -9,7 +9,7 @@ import pandas as pd
 from tqdm import tqdm
 import concurrent.futures
 import re
-
+import subprocess
 
 async def save_json(symbol, data):
     with open(f"json/trend-analysis/{symbol}.json", 'w') as file:
@@ -89,6 +89,14 @@ async def process_symbol(ticker, start_date, end_date, crypto_symbols):
         print(e)
 
 async def run():
+
+    #Train first model
+    try:
+        print('training...')
+        subprocess.run(["python3", "ml_models/classification.py", "--train"], check=True)
+    except subprocess.CalledProcessError as e:
+        print(f"Error running classification.py: {e}")
+
     con = sqlite3.connect('stocks.db')
     etf_con = sqlite3.connect('etf.db')
     crypto_con = sqlite3.connect('crypto.db')
diff --git a/app/ml_models/__pycache__/classification.cpython-310.pyc b/app/ml_models/__pycache__/classification.cpython-310.pyc
index a7418becc454f53d0df67c2c8e54499609ed3e98..905d98d96d30bdc0ee58a09bd76c3e5486379525 100644
GIT binary patch
delta 2294
zcma)8U2Gdg5cb-Rb9L+_4rzYsxOJMpUQ;{$3D6%aZIb>4C{3HthERGvyldxL`_Am0
zTbdUm(xM^})OK4W`a&a+P<bg*098WpfP@4H1VVi}-~}E~5kg3PtNOsq+9Zviuyo(e
z%+Act&dlw2KK1(Uek&Xfdf+Eg_cK#{9?$Q{Jo>8vW)D2})Rq3bB@ZRJ&go9pd3rsq
zqoH{+-Nm|2chfMfrwz1m-bXyF>t@e_XS$b0XwwDHbl(9FeT+6=@X+Qt@AL+`nzn#c
zrLD9L-WzE<T?6lawuwg1c=?B{_jH24&tf-|tbfk4KnUzO;b8-`V-DybGWe)5v6cK<
znq51|b;FG7CXGAHcH#wYrI~HV?KC$EPEsXq9c5+4u*})oM8ZJ>H)uN+XPU!{Y&JT<
zoVa7fCxO8?FPd?wWtdsj>jw0+1GcVzhUE%r|0B7nU5t6ZAY{Av#Wz7Z#b$q3%}!WO
zUx~CYzU+^XCkBCY{RQ2bQGKr8X1NSU5#NuneF=7;nxbijX*imeT)b0r+e_NTjMAq?
zQK<vheL8)`tp(5umR%-HmQ{UxNrksdy>!~(F`!g`)smFDzI?_|y;7IQ#NSGi922os
z!%wetBX#+)vcr64g)ekoSz`IK_rkcBui{WHiCe3pC5mJkK}R@+a1!AJ!ixx#2n^u`
zgyRTD5hf7E5S~JK2H_CGvj~R~#u1JHsL^FjcU4|bQ;iy#%(^NAH#9mrHfB;ItwZJc
zxOge}2{|s>>aLGs1aN=@=<#s?x87vuv;t?K47zr8r6O5oGOjyJ%P`$3a%K;Tf9k#^
zz2dXb6m&y#IME`DE9)9e<qX?NTlvBe701If;U3&BP}xp=7gp<}@@u&Lj;r-8<b-Ig
ze>k)cBME1qBcK;eJ7e)Y--|jIaUoDJ(r0qa_0wX$z}KK$HWjuPo<$sKXz8fxusYe)
zTHc~8H?ug`aFhh*(eN$tbK|+8%Sc{9xQZYUt|4F`{5rzQZTQ<DbNx9>r+lxt9BFOX
zjDi|&6-}xt5W}YtZCzCi_3W&kD`M_s7zHudw2!33Y}3Yr2XTv8gf|gxAlyb+b|ERu
zD&GZ=fX5C2D4rlu0?Gy@6jJ!E#lM=OUh<_-R{zv^7j-`daD7vU5Aol`pR2joL&V9J
zYze;>Ue%?tK~Ler|L9WLpxA=tW|U34vP=KC@ecmJ*?sTFT2OlzbRQ@^&nJ5FiJlsI
zi||1))|%Rm$5IL3i7ziNp~o#q`}j7bv8H@0!t(hq0J|VQZ_PX@W!=D5OyRzP&INba
z%2gI0plCe-7!=cO18u!%)&ezz4t0Gtb2yfeC&faWs$z=ap0UrHX~S{@S-8a*mj!YI
zlv@Q;=h(`8lepj37hWot8V2Wm?MKKZalXB8K%UKC4yNif^79Cno3e6mUlTvH50haL
zS@V4<hH<V%Scibg<8h=q0cLAQCPx#`P3$daryJ$+G$E)j&<PYI5%3r?wdLO0g0gbl
zxR6BHiZX09HCT4H0|fi|P87?$%fNRbE0cN&jfRljgCHxgR2nygg9xq<If(9v_oAI-
zM0^#UE@@zPFNL<FXokmtnhhg8@}HpEnLiKH>T9sE6#(z63TY(Ggzp1=oEW&OX|t(G
z9BMGG;-=-qaf}%7iP2<yT!&d$nR(lm2Au8a9421HCaG?AmGc?H)Xg;G@D~7Y7Z|9z
z&HKe09S=(bxLK`kSTl}OsC@N~k!P-=^K3!qHgoHTdA68mrZbK#M+fp9(kazIORl*=
zDMj9P{fOnd$O<i|0nf%i!Z2!Nh)LH66S>@(%c)+@w~~UZq?lt^5RW|~+}OsbH{@$1
Xicj&?hm@?+tu!h%0g!~e{s;d8!NwuS

delta 1905
zcma)+O>7%Q6vsVl$6hCE<9xYsKAJj-NjJesD%zw%ZrapsNoY%<p(06(+?98oEZAOa
zb`um3q=g6-XrX09#fO9toDf0<SrHNkID<e`2qCnHasa8yg)0*D!2HK<oJ1V3<loM`
zdGGD}+Ii{Vl~kh@3e|h~Z_;};x0djDerMfvF(O?X{PF$<U0L<a#UJ$OAw9h6(ZerF
zb3OV!dW1`R^+vskKl}7%y@fv$I+^yUt?Ux{wfvLxEg^f^hrVgj#{Tm4lF@QU-4Gca
z;jeLhCCyn-y{^wT@;M3-Jq+yKz&@zSnT%O9olGWCUh&_MNGlr-_WO50sg0crcBLvy
z>|3Gl&1Ov+<15uiw;l<p>n-F=RT8>1#@2#E<TU#uIC1<xbm{4e!$NM0FF3Cr8HJ_w
z5Z>jjIuu8&RNt|39?J~y1aJ<R17?6(;1NIr9tWm@hk;YTQQ#Ob2}}VG0mp$8z)23Z
zqo(Nw3tCps=w>e0z0JT?(&?$GqHbn2$D;HEQybREH2baL%@88s38dk154)(eG;C@C
zdtZr?LH4=w4e4Pogl2d<e+ea{LPPYYK9x6ZCu<c-<7e5<@IojKcmBKXSXk{5%Fn~S
zuxQK~nzKla3~#)Vbxf-`&2EM7tW0nXH<aVOcBodga~3VoL$JDt1AocPK9x6IpT1Zq
z(J16%TrfD*=-t2!OwFPjSgtI#4^x9U&J--&$fGwlCO%5)i6Vu|vyn9MzYNE(un!ua
z)-Pju1$Yg39k>c0P<jp68U}rX%UoaH(sU8H8z@nOSKOg~DC1r9KB&}DYq6-SGEKAR
znljy+%8k>ov83e}HOHvLInVxSI!gAliROXHBXF?<Tmr5G9|JWP8=awJ{GgBjZG}Vj
z)Dzh+@0Jxsreo!DbB9E}WZ$>`5V;BaPdQxg%&C*~7W*tpr8^#$*fGC?FN&TAYLhI6
zSd8=km}D`;7}LtYioxDG$+zEm8^5m)uHOuBwRgGh-*RF$nJ6R^b^1qyj<85uYCqCa
z2_3~lqR+x(5^FCV#(D%80&4kR;M=#@YFlnh$ae8<)s){oyfNI}9xmHv@g77w3C9SF
z$A;T_U>4<F<x}o@ZNm{Qqyuarrm85h8_?|KV%D@=|2)6Z3@RGr`gLlRiW-f>-){Cr
ztUt8TE;YoR{~3Fftg=Jx{lg+RUnQ7r*@zD&RGHcmwX%k7zsjz+Pmlxb=l1VbVu-UH
z=m1bW+KE*c$1{P$kEN4mrjJyrvkP*OjY1_Tst^nSNJEsiGFwT=;v#Nf02qP{qo&qZ
z-0kCnV{{Z^QFjse7`CFQt1x;1+Y}%=u+bV<aSYqZR1><vp6ra1adx?LZsjaDTTj9E
zAk2^k@7V#Y$L|WZJM$IzC;t_<v1JbFf-FTyi$o`2d#@`m=ggv3%o?<ppYm%&E9#m}
z`{7NL!3%N3=68!m*(Y7M&kXV--c-|CaGX;0n|92CQK^uqRV^UAWm7=46y+4ZWft=W
V!uHrBY@&N0=nX6KyxgZm{spO$p925@

diff --git a/app/ml_models/classification.py b/app/ml_models/classification.py
index c1bef55..ecb2831 100755
--- a/app/ml_models/classification.py
+++ b/app/ml_models/classification.py
@@ -20,6 +20,15 @@ import aiohttp
 import pickle
 import time
 
+import argparse
+
+# Set up argument parser
+parser = argparse.ArgumentParser(description="Train and test process script.")
+parser.add_argument('--train', action='store_true', help="Set to True to run training")
+    
+# Parse the arguments
+args = parser.parse_args()
+
 
 async def download_data(ticker, start_date, end_date, nth_day):
     try:
@@ -189,11 +198,14 @@ async def test_process(nth_day):
 
 
 async def main():
-    
-    for nth_day in [5,20,60]:
+    for nth_day in [5, 20, 60]:
         await train_process(nth_day)
-    
     await test_process(nth_day=5)
 
-# Run the main function
-#asyncio.run(main())
\ No newline at end of file
+if __name__ == "__main__":
+    
+    # Run main if --train is set to True
+    if args.train:
+        asyncio.run(main())
+    else:
+        print("Training not initiated. Pass --train True to start training.")
\ No newline at end of file
diff --git a/app/primary_cron_job.py b/app/primary_cron_job.py
index 734f0ca..54ddfd4 100755
--- a/app/primary_cron_job.py
+++ b/app/primary_cron_job.py
@@ -548,6 +548,15 @@ def run_earnings():
     ]
     run_command(command)
 
+def run_fomc_impact():
+    run_command(["python3", "cron_fomc_impact.py"])
+    command = [
+        "sudo", "rsync", "-avz", "-e", "ssh",
+        "/root/backend/app/json/fomc-impact",
+        f"root@{useast_ip_address}:/root/backend/app/json"
+    ]
+    run_command(command)
+
 def run_economy_indicator():
     run_command(["python3", "cron_economic_indicator.py"])
     command = [
@@ -557,6 +566,33 @@ def run_economy_indicator():
     ]
     run_command(command)
 
+def run_trend_analysis():
+    run_command(["python3", "cron_trend_analysis.py"])
+    command = [
+        "sudo", "rsync", "-avz", "-e", "ssh",
+        "/root/backend/app/json/trend-analysis",
+        f"root@{useast_ip_address}:/root/backend/app/json"
+    ]
+    run_command(command)
+
+def run_sentiment_analysis():
+    run_command(["python3", "cron_sentiment_analysis.py"])
+    command = [
+        "sudo", "rsync", "-avz", "-e", "ssh",
+        "/root/backend/app/json/sentiment-analysis",
+        f"root@{useast_ip_address}:/root/backend/app/json"
+    ]
+    run_command(command)
+
+def run_price_analysis():
+    run_command(["python3", "cron_price_analysis.py"])
+    command = [
+        "sudo", "rsync", "-avz", "-e", "ssh",
+        "/root/backend/app/json/price-analysis",
+        f"root@{useast_ip_address}:/root/backend/app/json"
+    ]
+    run_command(command)
+
 # Create functions to run each schedule in a separate thread
 def run_threaded(job_func):
     job_thread = threading.Thread(target=job_func)
@@ -577,6 +613,7 @@ schedule.every().day.at("07:30").do(run_threaded, run_financial_statements).tag(
 schedule.every().day.at("08:00").do(run_threaded, run_economy_indicator).tag('economy_indicator_job')
 schedule.every().day.at("08:00").do(run_threaded, run_cron_insider_trading).tag('insider_trading_job')
 schedule.every().day.at("08:30").do(run_threaded, run_dividends).tag('dividends_job')
+schedule.every().day.at("08:30").do(run_threaded, run_fomc_impact).tag('fomc_impact_job')
 schedule.every().day.at("09:00").do(run_threaded, run_congress_trading).tag('congress_job')
 schedule.every().day.at("10:00").do(run_threaded, run_shareholders).tag('shareholders_job')
 schedule.every().day.at("10:30").do(run_threaded, run_sec_filings).tag('sec_filings_job')
@@ -598,6 +635,9 @@ schedule.every().day.at("14:00").do(run_threaded, run_cron_sector).tag('sector_j
 schedule.every(2).days.at("01:00").do(run_threaded, run_market_maker).tag('markt_maker_job')
 schedule.every(2).days.at("08:30").do(run_threaded, run_financial_score).tag('financial_score_job')
 schedule.every().saturday.at("05:00").do(run_threaded, run_ownership_stats).tag('ownership_stats_job')
+schedule.every().saturday.at("08:00").do(run_threaded, run_trend_analysis).tag('trend_analysis_job')
+schedule.every().saturday.at("08:00").do(run_threaded, run_sentiment_analysis).tag('sentiment_analysis_job')
+schedule.every().saturday.at("08:00").do(run_threaded, run_price_analysis).tag('price_analysis_job')
 
 
 schedule.every(5).minutes.do(run_threaded, run_cron_market_movers).tag('market_movers_job')