From 64f286360f8ddc8f864d6f0b6c83defe559a4bc3 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Tue, 12 Nov 2024 12:27:03 +0100 Subject: [PATCH] update cron list --- app/cron_list.py | 77 +++++++++++++++----- app/main.py | 171 +------------------------------------------- app/restart_json.py | 15 ---- 3 files changed, 64 insertions(+), 199 deletions(-) diff --git a/app/cron_list.py b/app/cron_list.py index b49aabc..8fefa04 100644 --- a/app/cron_list.py +++ b/app/cron_list.py @@ -33,15 +33,6 @@ async def get_quote_data(symbol): return None async def process_category(cursor, category, condition, category_type='market-cap'): - """ - Process stocks for a specific category (market cap or sector) - - Args: - cursor: Database cursor - category: Category name - condition: SQL WHERE condition - category_type: Either 'market-cap' or 'sector' - """ base_query = """ SELECT DISTINCT s.symbol, s.name, s.exchangeShortName, s.marketCap, s.sector FROM stocks s @@ -84,7 +75,8 @@ async def process_category(cursor, category, condition, category_type='market-ca print(f"Processed and saved {len(sorted_result)} stocks for {category}") return sorted_result -def get_etf_holding(etf_symbols, etf_con): + +async def get_etf_holding(etf_symbols, etf_con): for ticker in tqdm(etf_symbols): res = [] @@ -124,8 +116,7 @@ def get_etf_holding(etf_symbols, etf_con): with open(f"json/etf/holding/{ticker}.json", 'wb') as file: file.write(orjson.dumps(res)) -def get_etf_provider(etf_con): - +async def get_etf_provider(etf_con): cursor = etf_con.cursor() cursor.execute("SELECT DISTINCT etfProvider FROM etfs") @@ -263,6 +254,58 @@ async def etf_bitcoin_list(): except Exception as e: print(f"Database error: {e}") +async def get_all_reits_list(cursor): + base_query = """ + SELECT DISTINCT s.symbol, s.name, s.exchangeShortName, s.marketCap, s.sector + FROM stocks s + WHERE {} + """ + + # Use the specific condition within the dictionary + condition = "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND industry LIKE '%REIT%' AND symbol NOT LIKE '%-%'" + full_query = base_query.format(condition) + + # Execute the query and fetch all rows + cursor.execute(full_query) # Assuming cursor is async + raw_data = cursor.fetchall() + + result_list = [] + for row in raw_data: + symbol = row[0] + + # Fetch quote data asynchronously + try: + quote_data = await get_quote_data(symbol) + except Exception as e: + print(f"Error fetching quote data for {symbol}: {e}") + continue + + if quote_data: + item = { + 'symbol': symbol, + 'name': row[1], + 'price': round(quote_data.get('price', 0), 2), + 'changesPercentage': round(quote_data.get('changesPercentage', 0), 2), + 'marketCap': quote_data.get('marketCap', 0), + } + + # Get dividend yield if available + item['dividendYield'] = stock_screener_data_dict.get(symbol, {}).get('dividendYield', None) + + # Append item if conditions are met + if item['marketCap'] > 0 and item['dividendYield'] is not None: + result_list.append(item) + + if result_list: + result_list = sorted(result_list, key=lambda x: x['marketCap'] or 0, reverse=True) + + # Add rank to each item + for rank, item in enumerate(result_list, 1): + item['rank'] = rank + + with open("json/industry/list/reits.json", 'wb') as file: + file.write(orjson.dumps(result_list)) + async def run(): await etf_bitcoin_list() @@ -292,6 +335,7 @@ async def run(): 'utilities': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Utilities')" } + try: con = sqlite3.connect('stocks.db') cursor = con.cursor() @@ -303,8 +347,9 @@ async def run(): etf_cursor.execute("SELECT DISTINCT symbol FROM etfs") etf_symbols = [row[0] for row in etf_cursor.fetchall()] - # Process market cap categories - + await get_all_reits_list(cursor) + + for category, condition in market_cap_conditions.items(): await process_category(cursor, category, condition, 'market-cap') await asyncio.sleep(1) # Small delay between categories @@ -315,9 +360,9 @@ async def run(): await asyncio.sleep(1) # Small delay between categories - get_etf_holding(etf_symbols, etf_con) + await get_etf_holding(etf_symbols, etf_con) - get_etf_provider(etf_con) + await get_etf_provider(etf_con) except Exception as e: diff --git a/app/main.py b/app/main.py index 9481522..f9af2dd 100755 --- a/app/main.py +++ b/app/main.py @@ -1465,78 +1465,6 @@ async def get_quant_stats(data: TickerData, api_key: str = Security(get_api_key) -@app.post("/trading-signals") -async def get_trading_signals(data: TickerData, api_key: str = Security(get_api_key)): - - data = data.dict() - ticker = data['ticker'].upper() - cache_key = f"get-trading-signals-{ticker}" - cached_result = redis_client.get(cache_key) - if cached_result: - return orjson.loads(cached_result) - if ticker in etf_symbols: - table_name = 'etfs' - else: - table_name = 'stocks' - # If the hash doesn't exist or doesn't match, fetch data from the database - query = f""" - SELECT - tradingSignals - FROM - {table_name} - WHERE - symbol = ? - """ - try: - # Execute the query and read the result into a DataFrame - query_result = pd.read_sql_query(query, etf_con if table_name == 'etfs' else con, params=(ticker,)) - - # Convert the DataFrame to a JSON object - if not query_result.empty: - res = query_result['tradingSignals'][0] - res = orjson.loads(res)[0] # Assuming 'tradingSignals' column contains JSON strings - res = replace_nan_inf_with_none(res) - #res = {'Start': '2021-07-14', 'End': '2023-11-10', 'Return [%]': 59.99997000000007, 'Buy & Hold Return [%]': -90.58823529411765, 'Return (Ann.) [%]': 77.21227166877182, 'Duration': '849', 'Volatility (Ann.) [%]': 34.82411687248909, 'Sharpe Ratio': 2.21720688428338, 'Sortino Ratio': None, 'Calmar Ratio': None, 'Max. Drawdown [%]': -0.0, 'Avg. Drawdown [%]': None, 'Max. Drawdown Duration': None, 'Avg. Drawdown Duration': None, '# Trades': 7, 'Win Rate [%]': 85.71428571428571, 'Best Trade [%]': 10.000000000000009, 'Worst Trade [%]': 0.0, 'Avg. Trade [%]': 6.944880005339327, 'Max. Trade Duration': '189', 'Avg. Trade Duration': '53', 'Profit Factor': None, 'Expectancy [%]': 6.989439132296275, 'SQN': 5.999999999999999, 'nextSignal': 'Hold'} - - else: - res = [] # Set a default value if query_result is empty - except Exception as e: - print("Error fetching data from the database:", e) - res = [] # Set a default value in case of an error - - redis_client.set(cache_key, orjson.dumps(res)) - redis_client.expire(cache_key, 3600 * 24) # Set cache expiration time to Infinity - return res - - -@app.post("/fair-price") -async def get_fair_price(data: TickerData, api_key: str = Security(get_api_key)): - data = data.dict() - ticker = data['ticker'].upper() - cache_key = f"get-fair-price-{ticker}" - cached_result = redis_client.get(cache_key) - if cached_result: - return orjson.loads(cached_result) - - query_template = """ - SELECT - discounted_cash_flow - FROM - stocks - WHERE - symbol = ? - """ - - try: - df = pd.read_sql_query(query_template, con, params=(ticker,)) - dcf_value = float(df['discounted_cash_flow'].iloc[0]) - except: - dcf_value = None - - - redis_client.set(cache_key, orjson.dumps(dcf_value)) - redis_client.expire(cache_key, 3600 * 24) # Set cache expiration time to Infinity - return dcf_value @app.post("/congress-trading-ticker") @@ -1994,78 +1922,7 @@ async def get_congress_rss_feed(api_key: str = Security(get_api_key)): redis_client.expire(cache_key, 60*15) return res -@app.get("/analysts-price-targets-rss-feed") -async def get_analysts_price_targets_rss_feed(api_key: str = Security(get_api_key)): - cache_key = f"analysts-price-targets-rss-feed" - cached_result = redis_client.get(cache_key) - if cached_result: - return StreamingResponse( - io.BytesIO(cached_result), - media_type="application/json", - headers={"Content-Encoding": "gzip"}) - try: - with open(f"json/analysts-rss-feed/price-targets.json", 'rb') as file: - res = orjson.loads(file.read()) - except: - res = [] - - data = orjson.dumps(res) - compressed_data = gzip.compress(data) - redis_client.set(cache_key, compressed_data) - redis_client.expire(cache_key, 60*60) # Set cache expiration time to 1 day - - return StreamingResponse( - io.BytesIO(compressed_data), - media_type="application/json", - headers={"Content-Encoding": "gzip"} - ) - - -@app.get("/analysts-upgrades-downgrades-rss-feed") -async def get_analysts_upgrades_downgrades_rss_feed(api_key: str = Security(get_api_key)): - cache_key = f"analysts-upgrades-downgrades-rss-feed" - cached_result = redis_client.get(cache_key) - if cached_result: - return StreamingResponse( - io.BytesIO(cached_result), - media_type="application/json", - headers={"Content-Encoding": "gzip"}) - - try: - with open(f"json/analysts-rss-feed/upgrades-downgrades.json", 'rb') as file: - res = orjson.loads(file.read()) - except: - res = [] - - data = orjson.dumps(res) - compressed_data = gzip.compress(data) - redis_client.set(cache_key, compressed_data) - redis_client.expire(cache_key, 60*60) # Set cache expiration time to 1 day - - return StreamingResponse( - io.BytesIO(compressed_data), - media_type="application/json", - headers={"Content-Encoding": "gzip"} - ) - -@app.get("/delisted-companies") -async def get_delisted_companies(api_key: str = Security(get_api_key)): - - cache_key = f"delisted-companies" - cached_result = redis_client.get(cache_key) - if cached_result: - return orjson.loads(cached_result) - - try: - with open(f"json/delisted-companies/data.json", 'rb') as file: - res = orjson.loads(file.read()) - except: - res = [] - - redis_client.set(cache_key, orjson.dumps(res)) - redis_client.expire(cache_key, 3600 * 24) # Set cache expiration time to 1 day - return res @app.post("/historical-sector-price") @@ -2132,18 +1989,6 @@ async def filter_stock_list(data: FilterStockList, api_key: str = Security(get_a 'IL': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' or exchangeShortName = 'AMEX') AND country = 'IL'", 'GB': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' or exchangeShortName = 'AMEX') AND country = 'GB'", 'JP': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' or exchangeShortName = 'AMEX') AND country = 'JP'", - 'financial': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Financials' OR sector = 'Financial Services')", - 'healthcare': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Healthcare')", - 'technology': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Technology')", - 'industrials': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Industrials')", - 'consumer-cyclical': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Consumer Cyclical')", - 'real-estate': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Real Estate')", - 'basic-materials': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Basic Materials')", - 'communication-services': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Communication Services')", - 'energy': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Energy')", - 'consumer-defensive': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Consumer Defensive')", - 'utilities': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND (sector = 'Utilities')", - 'reit': "(exchangeShortName = 'NYSE' OR exchangeShortName = 'NASDAQ' OR exchangeShortName = 'AMEX') AND industry LIKE '%REIT%' AND symbol NOT LIKE '%-%'", } # Execute the query with the relevant country @@ -2174,19 +2019,7 @@ async def filter_stock_list(data: FilterStockList, api_key: str = Security(get_a item['revenue'] = stock_screener_data_dict[symbol].get('revenue', None) item['netIncome'] = stock_screener_data_dict[symbol].get('netIncome', None) - # Optional: Filter or process the list further, e.g., for REITs as in your original code. - if filter_list == 'reit': - # No filtering based on dividendYield - # This includes all REITs in the list regardless of their dividendYield - # Simply check if the item is in the REIT condition - for item in res_list: - symbol = item['symbol'] - if symbol in stock_screener_data_dict: - item['dividendYield'] = stock_screener_data_dict[symbol].get('dividendYield', None) - - # Remove elements where dividendYield is None - res_list = [item for item in res_list if item.get('dividendYield') is not None] - + sorted_res_list = sorted(res_list, key=lambda x: x['marketCap'], reverse=True) # Cache the result @@ -4198,6 +4031,8 @@ async def get_statistics(data: FilterStockList, api_key: str = Security(get_api_ if filter_list in ['financial','healthcare','technology','industrials','consumer-cyclical','real-estate','basic-materials','communication-services','energy','consumer-defensive','utilities']: category_type = 'sector' + elif filter_list == 'reits': + category_type = 'industry' else: category_type = 'market-cap' try: diff --git a/app/restart_json.py b/app/restart_json.py index 7ad476c..b4f66b8 100755 --- a/app/restart_json.py +++ b/app/restart_json.py @@ -1385,15 +1385,6 @@ async def get_index_list(con,symbols, index_list): return sorted_res_list -async def get_delisted_list(): - - async with aiohttp.ClientSession() as session: - url = f"https://financialmodelingprep.com/api/v3/delisted-companies?apikey={api_key}" - async with session.get(url) as response: - data = await response.json() - return data - - def replace_representative(office): @@ -2020,16 +2011,10 @@ async def save_json_files(): ujson.dump(data, file) - - data = await etf_providers(etf_con, etf_symbols) with open(f"json/all-etf-providers/data.json", 'w') as file: ujson.dump(data, file) - - delisted_data = await get_delisted_list() - with open(f"json/delisted-companies/data.json", 'w') as file: - ujson.dump(delisted_data, file) stock_splits_data = await get_stock_splits_calendar(con,symbols)