From c43c2ef4d6c6dc3b1f488a349e3765fb8775c4fa Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Mon, 11 Nov 2024 20:30:36 +0100 Subject: [PATCH] refactor dividend cron job --- app/cron_dividend_aristocrats.py | 31 ++++++++++++++----------------- app/cron_dividend_kings.py | 31 ++++++++++++++----------------- app/main.py | 4 ++-- 3 files changed, 30 insertions(+), 36 deletions(-) diff --git a/app/cron_dividend_aristocrats.py b/app/cron_dividend_aristocrats.py index dc15011..963a462 100644 --- a/app/cron_dividend_aristocrats.py +++ b/app/cron_dividend_aristocrats.py @@ -7,24 +7,14 @@ from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager from selenium.webdriver.chrome.options import Options from dotenv import load_dotenv -import sqlite3 def save_json(data, file_path): with open(file_path, 'w') as file: ujson.dump(data, file) -query_template = """ - SELECT - name, sector - FROM - stocks - WHERE - symbol = ? -""" def main(): # Load environment variables - con = sqlite3.connect('stocks.db') load_dotenv() url = os.getenv('DIVIDEND_ARISTOCRATS') @@ -53,7 +43,7 @@ def main(): 'Company Name': 'name', 'Stock Price': 'price', '% Change': 'changesPercentage', - 'Div. Yield': 'dividiendYield', + 'Div. Yield': 'dividendYield', 'Years': 'years' }) df = df.drop(columns=['No.']) @@ -63,21 +53,28 @@ def main(): for item in data: symbol = item['symbol'] try: - item['changesPercentage'] = round(float(item['changesPercentage'].replace('%','')),2) - item['dividiendYield'] = round(float(item['dividiendYield'].replace('%','')),2) - db_data = pd.read_sql_query(query_template, con, params=(symbol,)) - res.append({**item,'sector': db_data['sector'].iloc[0]}) + with open(f"json/quote/{symbol}.json") as file: + quote_data = ujson.load(file) + + item['changesPercentage'] = round(quote_data['changesPercentage'],2) + item['price'] = round(quote_data['price'],2) + item['dividendYield'] = round(float(item['dividendYield'].replace('%','')),2) + res.append({**item}) except Exception as e: + print(e) pass # Save the JSON data if len(res) > 0: - save_json(res, 'json/stocks-list/dividend-aristocrats.json') + res = sorted(res, key=lambda x: x['years'], reverse=True) + for rank, item in enumerate(res, start=1): + item['rank'] = rank + + save_json(res, 'json/dividends/list/dividend-aristocrats.json') finally: # Ensure the WebDriver is closed driver.quit() - con.close() if __name__ == '__main__': main() \ No newline at end of file diff --git a/app/cron_dividend_kings.py b/app/cron_dividend_kings.py index a25cd15..4ab21bb 100644 --- a/app/cron_dividend_kings.py +++ b/app/cron_dividend_kings.py @@ -7,24 +7,14 @@ from selenium.webdriver.chrome.service import Service from webdriver_manager.chrome import ChromeDriverManager from selenium.webdriver.chrome.options import Options from dotenv import load_dotenv -import sqlite3 def save_json(data, file_path): with open(file_path, 'w') as file: ujson.dump(data, file) -query_template = """ - SELECT - name, sector - FROM - stocks - WHERE - symbol = ? -""" def main(): # Load environment variables - con = sqlite3.connect('stocks.db') load_dotenv() url = os.getenv('DIVIDEND_KINGS') @@ -53,7 +43,7 @@ def main(): 'Company Name': 'name', 'Stock Price': 'price', '% Change': 'changesPercentage', - 'Div. Yield': 'dividiendYield', + 'Div. Yield': 'dividendYield', 'Years': 'years' }) df = df.drop(columns=['No.']) @@ -63,21 +53,28 @@ def main(): for item in data: symbol = item['symbol'] try: - item['changesPercentage'] = round(float(item['changesPercentage'].replace('%','')),2) - item['dividiendYield'] = round(float(item['dividiendYield'].replace('%','')),2) - db_data = pd.read_sql_query(query_template, con, params=(symbol,)) - res.append({**item,'sector': db_data['sector'].iloc[0]}) + with open(f"json/quote/{symbol}.json") as file: + quote_data = ujson.load(file) + + item['changesPercentage'] = round(quote_data['changesPercentage'],2) + item['price'] = round(quote_data['price'],2) + item['dividendYield'] = round(float(item['dividendYield'].replace('%','')),2) + res.append({**item}) except Exception as e: + print(e) pass # Save the JSON data if len(res) > 0: - save_json(res, 'json/stocks-list/dividend-kings.json') + res = sorted(res, key=lambda x: x['years'], reverse=True) + for rank, item in enumerate(res, start=1): + item['rank'] = rank + + save_json(res, 'json/dividends/list/dividend-kings.json') finally: # Ensure the WebDriver is closed driver.quit() - con.close() if __name__ == '__main__': main() \ No newline at end of file diff --git a/app/main.py b/app/main.py index d893d2f..f275b02 100755 --- a/app/main.py +++ b/app/main.py @@ -3750,7 +3750,7 @@ async def get_dividend_kings(): headers={"Content-Encoding": "gzip"} ) try: - with open(f"json/stocks-list/dividend-kings.json", 'rb') as file: + with open(f"json/dividends/list/dividend-kings.json", 'rb') as file: res = orjson.loads(file.read()) except: res = [] @@ -3778,7 +3778,7 @@ async def get_dividend_kings(): headers={"Content-Encoding": "gzip"} ) try: - with open(f"json/stocks-list/dividend-aristocrats.json", 'rb') as file: + with open(f"json/dividends/list/dividend-aristocrats.json", 'rb') as file: res = orjson.loads(file.read()) except: res = []