diff --git a/app/cron_financial_statements.py b/app/cron_financial_statements.py index 52d654d..7aca481 100644 --- a/app/cron_financial_statements.py +++ b/app/cron_financial_statements.py @@ -1,6 +1,5 @@ import os import ujson -import random import asyncio import aiohttp import sqlite3 @@ -35,18 +34,32 @@ async def save_json(symbol, period, data_type, data): async def get_financial_statements(session, symbol, semaphore, request_counter): base_url = "https://financialmodelingprep.com/api/v3" periods = ['quarter', 'annual'] - data_types = ['income-statement', 'balance-sheet-statement', 'cash-flow-statement', 'ratios'] + financial_data_types = ['income-statement', 'balance-sheet-statement', 'cash-flow-statement', 'ratios'] + growth_data_types = ['income-statement-growth', 'balance-sheet-statement-growth', 'cash-flow-statement-growth'] async with semaphore: for period in periods: - for data_type in data_types: + # Fetch regular financial statements + for data_type in financial_data_types: url = f"{base_url}/{data_type}/{symbol}?period={period}&apikey={api_key}" data = await fetch_data(session, url, symbol) if data: await save_json(symbol, period, data_type, data) request_counter[0] += 1 # Increment the request counter - if request_counter[0] >= 1000: + if request_counter[0] >= 500: + await asyncio.sleep(60) # Pause for 60 seconds + request_counter[0] = 0 # Reset the request counter after the pause + + # Fetch financial statement growth data + for growth_type in growth_data_types: + growth_url = f"{base_url}/{growth_type}/{symbol}?period={period}&apikey={api_key}" + growth_data = await fetch_data(session, growth_url, symbol) + if growth_data: + await save_json(symbol, period, growth_type, growth_data) + + request_counter[0] += 1 # Increment the request counter + if request_counter[0] >= 500: await asyncio.sleep(60) # Pause for 60 seconds request_counter[0] = 0 # Reset the request counter after the pause @@ -70,4 +83,4 @@ async def run(): await asyncio.gather(*tasks) if __name__ == "__main__": - asyncio.run(run()) \ No newline at end of file + asyncio.run(run()) diff --git a/app/main.py b/app/main.py index 2a96322..3a30760 100755 --- a/app/main.py +++ b/app/main.py @@ -111,6 +111,14 @@ with db_connection(INSTITUTE_DB) as cursor: cik_list = [row[0] for row in cursor.fetchall()] #------End Institute DB------------# +#------Start Stock Screener--------# +with open(f"json/stock-screener/data.json", 'rb') as file: + stock_screener_data = orjson.loads(file.read()) + +print([item for item in stock_screener_data if item['symbol'] == 'TYL']) +#------End Stock Screener--------# + + ### TECH DEBT ### con = sqlite3.connect('stocks.db') etf_con = sqlite3.connect('etf.db') @@ -1148,12 +1156,10 @@ async def stock_finder(data:StockScreenerData, api_key: str = Security(get_api_k always_include = ['symbol', 'marketCap', 'price', 'changesPercentage', 'name','volume','pe'] try: - with open(f"json/stock-screener/data.json", 'rb') as file: - data = orjson.loads(file.read()) - filtered_data = [ - {key: item.get(key) for key in set(always_include + rule_of_list) if key in item} - for item in data - ] + filtered_data = [ + {key: item.get(key) for key in set(always_include + rule_of_list) if key in item} + for item in stock_screener_data + ] except Exception as e: filtered_data = [] diff --git a/app/restart_json.py b/app/restart_json.py index ac3f8cc..46078c7 100755 --- a/app/restart_json.py +++ b/app/restart_json.py @@ -63,15 +63,29 @@ def process_financial_data(file_path, key_list): if key in res: try: value = float(res[key]) - if key in ['grossProfitMargin','netProfitMargin','pretaxProfitMargin','operatingProfitMargin']: - value *= 100 # Multiply by 100 if key is 'grossProfitMargin' + if 'growth' in file_path or key in ['grossProfitMargin','netProfitMargin','pretaxProfitMargin','operatingProfitMargin']: + value *= 100 # Multiply by 100 for percentage data[key] = round(value, 2) except (ValueError, TypeError): + # If there's an issue converting the value, leave it as None data[key] = None - except: + except (FileNotFoundError, KeyError, IndexError): + # If the file doesn't exist or there's an issue reading the data, + # data will retain None as the default value for all keys pass + return data +def check_and_process(file_path, key_list): + """ + Check if the file exists, and then process the financial data if it does. + If the file doesn't exist, return a dictionary with all keys set to None. + """ + if os.path.exists(file_path): + return process_financial_data(file_path, key_list) + else: + return {key: None for key in key_list} + def get_financial_statements(item, symbol): """ Update item with financial data from various JSON files. @@ -134,12 +148,123 @@ def get_financial_statements(item, symbol): "totalInvestments", "totalDebt", "netDebt" ] - # Process each financial statement - item.update(process_financial_data(f"json/financial-statements/ratios/annual/{symbol}.json", key_ratios)) - item.update(process_financial_data(f"json/financial-statements/cash-flow-statement/annual/{symbol}.json", key_cash_flow)) - item.update(process_financial_data(f"json/financial-statements/income-statement/annual/{symbol}.json", key_income)) - item.update(process_financial_data(f"json/financial-statements/balance-sheet-statement/annual/{symbol}.json", key_balance_sheet)) + key_income_growth = [ + "growthRevenue", + "growthCostOfRevenue", + "growthGrossProfit", + "growthGrossProfitRatio", + "growthResearchAndDevelopmentExpenses", + "growthGeneralAndAdministrativeExpenses", + "growthSellingAndMarketingExpenses", + "growthOtherExpenses", + "growthOperatingExpenses", + "growthCostAndExpenses", + "growthInterestExpense", + "growthDepreciationAndAmortization", + "growthEBITDA", + "growthEBITDARatio", + "growthOperatingIncome", + "growthOperatingIncomeRatio", + "growthTotalOtherIncomeExpensesNet", + "growthIncomeBeforeTax", + "growthIncomeBeforeTaxRatio", + "growthIncomeTaxExpense", + "growthNetIncome", + "growthNetIncomeRatio", + "growthEPS", + "growthEPSDiluted", + "growthWeightedAverageShsOut", + "growthWeightedAverageShsOutDil" + ] + key_cash_flow_growth = [ + "growthNetIncome", + "growthDepreciationAndAmortization", + "growthDeferredIncomeTax", + "growthStockBasedCompensation", + "growthChangeInWorkingCapital", + "growthAccountsReceivables", + "growthInventory", + "growthAccountsPayables", + "growthOtherWorkingCapital", + "growthOtherNonCashItems", + "growthNetCashProvidedByOperatingActivites", + "growthInvestmentsInPropertyPlantAndEquipment", + "growthAcquisitionsNet", + "growthPurchasesOfInvestments", + "growthSalesMaturitiesOfInvestments", + "growthOtherInvestingActivites", + "growthNetCashUsedForInvestingActivites", + "growthDebtRepayment", + "growthCommonStockIssued", + "growthCommonStockRepurchased", + "growthDividendsPaid", + "growthOtherFinancingActivites", + "growthNetCashUsedProvidedByFinancingActivities", + "growthEffectOfForexChangesOnCash", + "growthNetChangeInCash", + "growthCashAtEndOfPeriod", + "growthCashAtBeginningOfPeriod", + "growthOperatingCashFlow", + "growthCapitalExpenditure", + "growthFreeCashFlow" +] + key_balance_sheet_growth = [ + "growthCashAndCashEquivalents", + "growthShortTermInvestments", + "growthCashAndShortTermInvestments", + "growthNetReceivables", + "growthInventory", + "growthOtherCurrentAssets", + "growthTotalCurrentAssets", + "growthPropertyPlantEquipmentNet", + "growthGoodwill", + "growthIntangibleAssets", + "growthGoodwillAndIntangibleAssets", + "growthLongTermInvestments", + "growthTaxAssets", + "growthOtherNonCurrentAssets", + "growthTotalNonCurrentAssets", + "growthOtherAssets", + "growthTotalAssets", + "growthAccountPayables", + "growthShortTermDebt", + "growthTaxPayables", + "growthDeferredRevenue", + "growthOtherCurrentLiabilities", + "growthTotalCurrentLiabilities", + "growthLongTermDebt", + "growthDeferredRevenueNonCurrent", + "growthDeferredTaxLiabilitiesNonCurrent", + "growthOtherNonCurrentLiabilities", + "growthTotalNonCurrentLiabilities", + "growthOtherLiabilities", + "growthTotalLiabilities", + "growthCommonStock", + "growthRetainedEarnings", + "growthAccumulatedOtherComprehensiveIncomeLoss", + "growthOtherTotalStockholdersEquity", + "growthTotalStockholdersEquity", + "growthTotalLiabilitiesAndStockholdersEquity", + "growthTotalInvestments", + "growthTotalDebt", + "growthNetDebt" + ] + # Process each financial statement + statements = [ + (f"json/financial-statements/ratios/annual/{symbol}.json", key_ratios), + (f"json/financial-statements/cash-flow-statement/annual/{symbol}.json", key_cash_flow), + (f"json/financial-statements/income-statement/annual/{symbol}.json", key_income), + (f"json/financial-statements/balance-sheet-statement/annual/{symbol}.json", key_balance_sheet), + (f"json/financial-statements/income-statement-growth/annual/{symbol}.json", key_income_growth), + (f"json/financial-statements/balance-sheet-statement-growth/annual/{symbol}.json", key_balance_sheet_growth), + (f"json/financial-statements/cash-flow-statement-growth/annual/{symbol}.json", key_cash_flow_growth) + ] + + # Process each financial statement + for file_path, key_list in statements: + item.update(check_and_process(file_path, key_list)) + try: item['freeCashFlowMargin'] = round((item['freeCashFlow'] / item['revenue']) * 100,2) except: @@ -180,7 +305,7 @@ async def get_stock_screener(con): #Stock Screener Data - cursor.execute("SELECT symbol, name, change_1W, change_1M, change_3M, change_6M, change_1Y, change_3Y, sma_50, sma_200, ema_50, ema_200, rsi, atr, stoch_rsi, mfi, cci, pe, marketCap, growthRevenue, growthNetIncome, growthGrossProfit, growthCostOfRevenue, growthCostAndExpenses, growthInterestExpense, growthResearchAndDevelopmentExpenses, growthEBITDA, growthEPS, growthOperatingExpenses, growthOperatingIncome, beta FROM stocks WHERE symbol NOT LIKE '%.%' AND eps IS NOT NULL AND revenue IS NOT NULL AND marketCap IS NOT NULL AND beta IS NOT NULL") + cursor.execute("SELECT symbol, name, change_1W, change_1M, change_3M, change_6M, change_1Y, change_3Y, sma_50, sma_200, ema_50, ema_200, rsi, atr, stoch_rsi, mfi, cci, pe, marketCap, beta FROM stocks WHERE symbol NOT LIKE '%.%' AND eps IS NOT NULL AND marketCap IS NOT NULL AND beta IS NOT NULL") raw_data = cursor.fetchall() stock_screener_data = [{ 'symbol': symbol, @@ -202,23 +327,14 @@ async def get_stock_screener(con): 'cci': cci, 'pe': pe, 'marketCap': marketCap, - 'growthRevenue': growthRevenue, - 'growthNetIncome': growthNetIncome, - 'growthGrossProfit': growthGrossProfit, - 'growthCostOfRevenue': growthCostOfRevenue, - 'growthCostAndExpenses': growthCostAndExpenses, - 'growthInterestExpense': growthInterestExpense, - 'growthResearchAndDevelopmentExpenses': growthResearchAndDevelopmentExpenses, - 'growthEBITDA': growthEBITDA, - 'growthEPS': growthEPS, - 'growthOperatingExpenses': growthOperatingExpenses, - 'growthOperatingIncome': growthOperatingIncome, 'beta': beta, - } for (symbol, name, change_1W, change_1M, change_3M, change_6M, change_1Y, change_3Y, sma_50, sma_200, ema_50, ema_200, rsi, atr, stoch_rsi, mfi, cci, pe, marketCap, growthRevenue, growthNetIncome, growthGrossProfit, growthCostOfRevenue, growthCostAndExpenses, growthInterestExpense, growthResearchAndDevelopmentExpenses, growthEBITDA, growthEPS, growthOperatingExpenses, growthOperatingIncome, beta) in raw_data] + } for (symbol, name, change_1W, change_1M, change_3M, change_6M, change_1Y, change_3Y, sma_50, sma_200, ema_50, ema_200, rsi, atr, stoch_rsi, mfi, cci, pe, marketCap, beta) in raw_data] stock_screener_data = [{k: round(v, 2) if isinstance(v, (int, float)) else v for k, v in entry.items()} for entry in stock_screener_data] + cursor.execute("SELECT symbol, name, change_1W, change_1M, change_3M, change_6M, change_1Y, change_3Y, sma_50, sma_200, ema_50, ema_200, rsi, atr, stoch_rsi, mfi, cci, pe, marketCap, beta FROM stocks WHERE symbol NOT LIKE '%.%' AND eps IS NOT NULL AND marketCap IS NOT NULL AND beta IS NOT NULL") + raw_data = cursor.fetchall() # Iterate through stock_screener_data and update 'price' and 'changesPercentage' if symbols match # Add VaR value to stock screener