From f8b9280509b498b304b5de6d1f45ed1ccff5bba9 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Sun, 4 Aug 2024 18:36:10 +0200 Subject: [PATCH] add dividend kings && aristocrats --- app/cron_dividend_aristocrats.py | 83 ++++++++++++++++++++++++++++++++ app/cron_dividend_kings.py | 83 ++++++++++++++++++++++++++++++++ app/main.py | 18 +++++++ 3 files changed, 184 insertions(+) create mode 100644 app/cron_dividend_aristocrats.py create mode 100644 app/cron_dividend_kings.py diff --git a/app/cron_dividend_aristocrats.py b/app/cron_dividend_aristocrats.py new file mode 100644 index 0000000..dc15011 --- /dev/null +++ b/app/cron_dividend_aristocrats.py @@ -0,0 +1,83 @@ +import os +import pandas as pd +import ujson +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.service import Service +from webdriver_manager.chrome import ChromeDriverManager +from selenium.webdriver.chrome.options import Options +from dotenv import load_dotenv +import sqlite3 + +def save_json(data, file_path): + with open(file_path, 'w') as file: + ujson.dump(data, file) + +query_template = """ + SELECT + name, sector + FROM + stocks + WHERE + symbol = ? +""" + +def main(): + # Load environment variables + con = sqlite3.connect('stocks.db') + load_dotenv() + url = os.getenv('DIVIDEND_ARISTOCRATS') + + # Set up the WebDriver options + options = Options() + options.add_argument("--headless") + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + + # Initialize the WebDriver + service = Service(ChromeDriverManager().install()) + driver = webdriver.Chrome(options=options) + + try: + # Fetch the website + driver.get(url) + # Find the table element + table = driver.find_element(By.TAG_NAME, 'table') + # Extract the table HTML + table_html = table.get_attribute('outerHTML') + # Use pandas to read the HTML table + df = pd.read_html(table_html)[0] + # Rename the columns + df = df.rename(columns={ + 'Symbol': 'symbol', + 'Company Name': 'name', + 'Stock Price': 'price', + '% Change': 'changesPercentage', + 'Div. Yield': 'dividiendYield', + 'Years': 'years' + }) + df = df.drop(columns=['No.']) + # Convert the DataFrame to JSON + data = ujson.loads(df.to_json(orient='records')) + res = [] + for item in data: + symbol = item['symbol'] + try: + item['changesPercentage'] = round(float(item['changesPercentage'].replace('%','')),2) + item['dividiendYield'] = round(float(item['dividiendYield'].replace('%','')),2) + db_data = pd.read_sql_query(query_template, con, params=(symbol,)) + res.append({**item,'sector': db_data['sector'].iloc[0]}) + except Exception as e: + pass + + # Save the JSON data + if len(res) > 0: + save_json(res, 'json/stocks-list/dividend-aristocrats.json') + + finally: + # Ensure the WebDriver is closed + driver.quit() + con.close() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/app/cron_dividend_kings.py b/app/cron_dividend_kings.py new file mode 100644 index 0000000..a25cd15 --- /dev/null +++ b/app/cron_dividend_kings.py @@ -0,0 +1,83 @@ +import os +import pandas as pd +import ujson +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.chrome.service import Service +from webdriver_manager.chrome import ChromeDriverManager +from selenium.webdriver.chrome.options import Options +from dotenv import load_dotenv +import sqlite3 + +def save_json(data, file_path): + with open(file_path, 'w') as file: + ujson.dump(data, file) + +query_template = """ + SELECT + name, sector + FROM + stocks + WHERE + symbol = ? +""" + +def main(): + # Load environment variables + con = sqlite3.connect('stocks.db') + load_dotenv() + url = os.getenv('DIVIDEND_KINGS') + + # Set up the WebDriver options + options = Options() + options.add_argument("--headless") + options.add_argument("--no-sandbox") + options.add_argument("--disable-dev-shm-usage") + + # Initialize the WebDriver + service = Service(ChromeDriverManager().install()) + driver = webdriver.Chrome(options=options) + + try: + # Fetch the website + driver.get(url) + # Find the table element + table = driver.find_element(By.TAG_NAME, 'table') + # Extract the table HTML + table_html = table.get_attribute('outerHTML') + # Use pandas to read the HTML table + df = pd.read_html(table_html)[0] + # Rename the columns + df = df.rename(columns={ + 'Symbol': 'symbol', + 'Company Name': 'name', + 'Stock Price': 'price', + '% Change': 'changesPercentage', + 'Div. Yield': 'dividiendYield', + 'Years': 'years' + }) + df = df.drop(columns=['No.']) + # Convert the DataFrame to JSON + data = ujson.loads(df.to_json(orient='records')) + res = [] + for item in data: + symbol = item['symbol'] + try: + item['changesPercentage'] = round(float(item['changesPercentage'].replace('%','')),2) + item['dividiendYield'] = round(float(item['dividiendYield'].replace('%','')),2) + db_data = pd.read_sql_query(query_template, con, params=(symbol,)) + res.append({**item,'sector': db_data['sector'].iloc[0]}) + except Exception as e: + pass + + # Save the JSON data + if len(res) > 0: + save_json(res, 'json/stocks-list/dividend-kings.json') + + finally: + # Ensure the WebDriver is closed + driver.quit() + con.close() + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/app/main.py b/app/main.py index e43f629..aa31fbe 100755 --- a/app/main.py +++ b/app/main.py @@ -3317,6 +3317,24 @@ async def get_reddit_tracker(api_key: str = Security(get_api_key)): headers={"Content-Encoding": "gzip"} ) +@app.get("/dividend-kings") +async def get_dividend_kings(): + try: + with open(f"json/stocks-list/dividend-kings.json", 'rb') as file: + res = orjson.loads(file.read()) + except: + res = [] + return res + +@app.get("/dividend-aristocrats") +async def get_dividend_kings(): + try: + with open(f"json/stocks-list/dividend-aristocrats.json", 'rb') as file: + res = orjson.loads(file.read()) + except: + res = [] + return res + @app.get("/newsletter") async def get_newsletter(): try: