From 90680b687a9bab6fb0e0d1fe384c7f793aac8794 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Sat, 12 Oct 2024 14:47:27 +0200 Subject: [PATCH] refactor code --- app/create_stock_db.py | 2 +- app/main.py | 34 +++++++++++++--------- app/restart_json.py | 66 +++++++++++++++++++++++++----------------- 3 files changed, 62 insertions(+), 40 deletions(-) diff --git a/app/create_stock_db.py b/app/create_stock_db.py index 63aa610..37ace3b 100755 --- a/app/create_stock_db.py +++ b/app/create_stock_db.py @@ -273,7 +273,7 @@ class StockDatabase: if i % 60 == 0: await asyncio.gather(*tasks) tasks = [] - print('sleeping mode: ', i) + print('sleeping mode 30 seconds') await asyncio.sleep(30) # Pause for 60 seconds diff --git a/app/main.py b/app/main.py index bf68d27..22f0740 100755 --- a/app/main.py +++ b/app/main.py @@ -1999,7 +1999,7 @@ async def historical_sector_price(data:FilterStockList, api_key: str = Security( @app.post("/filter-stock-list") -async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_api_key)): +async def filter_stock_list(data: FilterStockList, api_key: str = Security(get_api_key)): data = data.dict() filter_list = data['filterList'] cache_key = f"filter-list-{filter_list}" @@ -2012,7 +2012,7 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap cursor.execute("PRAGMA journal_mode = wal") base_query = """ - SELECT symbol, name, price, changesPercentage, marketCap, revenue, netIncome + SELECT symbol, name, price, changesPercentage, marketCap FROM stocks WHERE (price IS NOT NULL OR changesPercentage IS NOT NULL) AND {} @@ -2064,16 +2064,25 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap 'price': price, 'changesPercentage': changesPercentage, 'marketCap': marketCap, - 'revenue': revenue, - 'netIncome': netIncome - } for (symbol, name, price, changesPercentage, marketCap, revenue, netIncome) in raw_data] + 'revenue': None, # Placeholder for revenue + 'netIncome': None # Placeholder for netIncome + } for (symbol, name, price, changesPercentage, marketCap) in raw_data] - # Update res_list with dividendYield + # Create the dictionary keyed by symbol for revenue and netIncome + stock_screener_data_dict = {item['symbol']: item for item in stock_screener_data} + + # Update revenue and netIncome for each item in res_list + for item in res_list: + symbol = item['symbol'] + if symbol in stock_screener_data_dict: + item['revenue'] = stock_screener_data_dict[symbol].get('revenue', None) + item['netIncome'] = stock_screener_data_dict[symbol].get('netIncome', None) + + # Optional: Filter or process the list further, e.g., for REITs as in your original code. if filter_list == 'reit': - # Create the dictionary keyed by symbol - stock_screener_data_dict = {item['symbol']: item for item in stock_screener_data} - - # Update dividendYield for each item in res_list + # No filtering based on dividendYield + # This includes all REITs in the list regardless of their dividendYield + # Simply check if the item is in the REIT condition for item in res_list: symbol = item['symbol'] if symbol in stock_screener_data_dict: @@ -2081,9 +2090,7 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap # Remove elements where dividendYield is None res_list = [item for item in res_list if item.get('dividendYield') is not None] - - - + sorted_res_list = sorted(res_list, key=lambda x: x['marketCap'], reverse=True) # Cache the result @@ -2093,6 +2100,7 @@ async def filter_stock_list(data:FilterStockList, api_key: str = Security(get_ap return sorted_res_list + def remove_text_before_operator(text): # Find the index of the first occurrence of "Operator" operator_index = text.find("Operator") diff --git a/app/restart_json.py b/app/restart_json.py index 4e0c556..df8fa0b 100755 --- a/app/restart_json.py +++ b/app/restart_json.py @@ -683,7 +683,7 @@ async def get_dividends_calendar(con,symbols): #Database read 1y and 3y data query_template = """ SELECT - name, marketCap, revenue + name, marketCap FROM stocks WHERE @@ -699,10 +699,15 @@ async def get_dividends_calendar(con,symbols): for entry in filtered_data: try: symbol = entry['symbol'] + try: + with open(f"json/financial-statements/income-statement/annual/{symbol}.json", 'rb') as file: + entry['revenue'] = orjson.loads(file.read())[0]['revenue'] + except: + entry['revenue'] = None + data = pd.read_sql_query(query_template, con, params=(symbol,)) entry['name'] = data['name'].iloc[0] entry['marketCap'] = int(data['marketCap'].iloc[0]) - entry['revenue'] = int(data['revenue'].iloc[0]) except: entry['name'] = 'n/a' entry['marketCap'] = None @@ -734,7 +739,7 @@ async def get_earnings_calendar(con, symbols): query_template = """ SELECT - name,marketCap,revenue,eps + name,marketCap,eps FROM stocks WHERE @@ -749,10 +754,15 @@ async def get_earnings_calendar(con, symbols): for entry in filtered_data: try: symbol = entry['symbol'] + try: + with open(f"json/financial-statements/income-statement/annual/{symbol}.json", 'rb') as file: + entry['revenue'] = orjson.loads(file.read())[0]['revenue'] + except: + entry['revenue'] = None + fundamental_data = pd.read_sql_query(query_template, con, params=(symbol,)) entry['name'] = fundamental_data['name'].iloc[0] entry['marketCap'] = int(fundamental_data['marketCap'].iloc[0]) - entry['revenue'] = int(fundamental_data['revenue'].iloc[0]) entry['eps'] = float(fundamental_data['eps'].iloc[0]) except: entry['marketCap'] = 'n/a' @@ -810,7 +820,7 @@ async def get_stock_splits_calendar(con,symbols): #Database read 1y and 3y data query_template = """ SELECT - name, marketCap,eps, revenue, netIncome + name, marketCap,eps FROM stocks WHERE @@ -826,17 +836,14 @@ async def get_stock_splits_calendar(con,symbols): for entry in filtered_data: try: symbol = entry['symbol'] + data = pd.read_sql_query(query_template, con, params=(symbol,)) entry['name'] = data['name'].iloc[0] entry['marketCap'] = int(data['marketCap'].iloc[0]) - entry['revenue'] = int(data['revenue'].iloc[0]) - entry['netIncome'] = int(data['netIncome'].iloc[0]) entry['eps'] = float(data['eps'].iloc[0]) except: entry['name'] = 'n/a' entry['marketCap'] = None - entry['revenue'] = None - entry['netIncome'] = None entry['eps'] = None filtered_data = [d for d in filtered_data if d['symbol'] in symbols] @@ -1157,7 +1164,7 @@ async def get_index_list(con,symbols, index_list): query_template = """ SELECT - price, changesPercentage, marketCap, revenue, netIncome + price, changesPercentage, marketCap FROM stocks WHERE @@ -1172,12 +1179,12 @@ async def get_index_list(con,symbols, index_list): res_list = [] for entry in filtered_data: + symbol = entry['symbol'] + query_data = pd.read_sql_query(query_template, con, params=(entry['symbol'],)) - if query_data['marketCap'].iloc[0] != None and query_data['revenue'].iloc[0] !=None and query_data['price'].iloc[0] != None and query_data['changesPercentage'].iloc[0] != None: + if query_data['marketCap'].iloc[0] != None and query_data['price'].iloc[0] != None and query_data['changesPercentage'].iloc[0] != None: entry['marketCap'] = int(query_data['marketCap'].iloc[0]) - entry['revenue'] = int(query_data['revenue'].iloc[0]) - entry['netIncome'] = int(query_data['netIncome'].iloc[0]) entry['price'] = round(float(query_data['price'].iloc[0]),2) entry['changesPercentage'] = round(float(query_data['changesPercentage'].iloc[0]),2) res_list.append(entry) @@ -1539,15 +1546,25 @@ async def get_magnificent_seven(con): query_template = """ SELECT - symbol, name, price, changesPercentage, revenue, netIncome, marketCap,pe + symbol, name, price, changesPercentage, marketCap,pe FROM stocks WHERE symbol = ? """ + res_list = [] for symbol in symbol_list: try: + try: + with open(f"json/financial-statements/income-statement/annual/{symbol}.json", 'rb') as file: + json_data = orjson.loads(file.read())[0] + revenue = json_data['revenue'] + netIncome = json_data['netIncome'] + except: + revenue = None + netIncome = None + data = pd.read_sql_query(query_template, con, params=(symbol,)) name = data['name'].iloc[0] @@ -1555,8 +1572,6 @@ async def get_magnificent_seven(con): price = round(float(data['price'].iloc[0]),2) changesPercentage = round(float(data['changesPercentage'].iloc[0]),2) marketCap = int(data['marketCap'].iloc[0]) - revenue = int(data['revenue'].iloc[0]) - netIncome = int(data['netIncome'].iloc[0]) pe = round(float(data['pe'].iloc[0]),2) res_list.append({'symbol': symbol, 'name': name, 'price': price, \ @@ -1849,7 +1864,8 @@ async def save_json_files(): crypto_cursor.execute("SELECT DISTINCT symbol FROM cryptos") crypto_symbols = [row[0] for row in crypto_cursor.fetchall()] - + + stock_screener_data = await get_stock_screener(con) with open(f"json/stock-screener/data.json", 'w') as file: @@ -1860,6 +1876,10 @@ async def save_json_files(): with open(f"json/economic-calendar/calendar.json", 'w') as file: ujson.dump(economic_list, file) + dividends_list = await get_dividends_calendar(con,symbols) + with open(f"json/dividends-calendar/calendar.json", 'w') as file: + ujson.dump(dividends_list, file) + earnings_list = await get_earnings_calendar(con,symbols) with open(f"json/earnings-calendar/calendar.json", 'w') as file: @@ -1874,11 +1894,6 @@ async def save_json_files(): data = await get_congress_rss_feed(symbols, etf_symbols, crypto_symbols) with open(f"json/congress-trading/rss-feed/data.json", 'w') as file: ujson.dump(data, file) - - - data = await get_magnificent_seven(con) - with open(f"json/magnificent-seven/data.json", 'w') as file: - ujson.dump(data, file) data = await get_ipo_calendar(con, symbols) with open(f"json/ipo-calendar/data.json", 'w') as file: @@ -1910,9 +1925,6 @@ async def save_json_files(): with open(f"json/delisted-companies/data.json", 'w') as file: ujson.dump(delisted_data, file) - dividends_list = await get_dividends_calendar(con,symbols) - with open(f"json/dividends-calendar/calendar.json", 'w') as file: - ujson.dump(dividends_list, file) stock_splits_data = await get_stock_splits_calendar(con,symbols) with open(f"json/stock-splits-calendar/calendar.json", 'w') as file: @@ -1930,8 +1942,10 @@ async def save_json_files(): data = await get_index_list(con,symbols,'sp500_constituent') with open(f"json/stocks-list/sp500_constituent.json", 'w') as file: ujson.dump(data, file) - + data = await get_magnificent_seven(con) + with open(f"json/magnificent-seven/data.json", 'w') as file: + ujson.dump(data, file) con.close() etf_con.close()