AmineTMIRI/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update cramer job

Browse Source
This commit is contained in:
MuslemRahimi 2024-07-24 22:38:32 +02:00
parent 4ab4a7e70c
commit 1613785bb4
1 changed files with 8 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
app/cron_cramer_tracker.py
View File

@ -5,7 +5,6 @@ from selenium import webdriver
from selenium.webdriver.common.by import By from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.service import Service from selenium.webdriver.chrome.service import Service
from selenium.webdriver.chrome.options import Options from selenium.webdriver.chrome.options import Options
from webdriver_manager.chrome import ChromeDriverManager
from dotenv import load_dotenv from dotenv import load_dotenv
import sqlite3 import sqlite3
@ -13,7 +12,6 @@ def save_json(data, file_path):
with open(file_path, 'w') as file: with open(file_path, 'w') as file:
ujson.dump(data, file) ujson.dump(data, file)
query_template = """ query_template = """
SELECT SELECT
name, sector name, sector
@ -23,7 +21,6 @@ query_template = """
symbol = ? symbol = ?
""" """
def main(): def main():
# Load environment variables # Load environment variables
con = sqlite3.connect('stocks.db') con = sqlite3.connect('stocks.db')
@ -32,24 +29,22 @@ def main():
# Set up the WebDriver options # Set up the WebDriver options
options = Options() options = Options()
options.headless = True # Run in headless mode options.add_argument("--headless")
options.add_argument("--no-sandbox")
options.add_argument("--disable-dev-shm-usage")
# Initialize the WebDriver # Initialize the WebDriver
driver = webdriver.Chrome(service=Service(ChromeDriverManager().install()), options=options) driver = webdriver.Chrome(options=options)
try: try:
# Fetch the website # Fetch the website
driver.get(url) driver.get(url)
# Find the table element # Find the table element
table = driver.find_element(By.TAG_NAME, 'table') table = driver.find_element(By.TAG_NAME, 'table')
# Extract the table HTML # Extract the table HTML
table_html = table.get_attribute('outerHTML') table_html = table.get_attribute('outerHTML')
# Use pandas to read the HTML table # Use pandas to read the HTML table
df = pd.read_html(table_html)[0] df = pd.read_html(table_html)[0]
# Rename the columns # Rename the columns
df = df.rename(columns={ df = df.rename(columns={
'Ticker': 'ticker', 'Ticker': 'ticker',
@ -57,10 +52,8 @@ def main():
'Date': 'date', 'Date': 'date',
'Return Since': 'returnSince' 'Return Since': 'returnSince'
}) })
# Convert the DataFrame to JSON # Convert the DataFrame to JSON
data = ujson.loads(df.to_json(orient='records')) data = ujson.loads(df.to_json(orient='records'))
res = [] res = []
for item in data: for item in data:
symbol = item['ticker'] symbol = item['ticker']
@ -70,9 +63,10 @@ def main():
res.append({**item, 'name': db_data['name'].iloc[0], 'sector': db_data['sector'].iloc[0]}) res.append({**item, 'name': db_data['name'].iloc[0], 'sector': db_data['sector'].iloc[0]})
except Exception as e: except Exception as e:
pass pass
# Save the JSON data # Save the JSON data
save_json(res, 'json/cramer-tracker/data.json') if len(res) > 0:
save_json(res, 'json/cramer-tracker/data.json')
finally: finally:
# Ensure the WebDriver is closed # Ensure the WebDriver is closed
@ -80,4 +74,4 @@ def main():
con.close() con.close()
if __name__ == '__main__': if __name__ == '__main__':
main() main()