bugfixing

app/cron_hedge_funds.py

@@ -2,6 +2,7 @@ import sqlite3
 import os
 import orjson
 import time
+from datetime import datetime
 from collections import Counter
 from tqdm import tqdm
 
@@ -19,6 +20,7 @@ keys_to_keep = [
 
 quote_cache = {}
 
+cutoff_date = datetime.strptime("2015-01-01", "%Y-%m-%d")
 
 def get_quote_data(symbol):
     """Get quote data for a symbol from JSON file"""
@@ -101,22 +103,33 @@ def all_hedge_funds(con):
 
 
 def get_data(cik, stock_sectors):
-    cursor.execute("SELECT cik, name, numberOfStocks, performancePercentage3year, performancePercentage5year, performanceSinceInceptionPercentage, averageHoldingPeriod, turnover, marketValue, winRate, holdings, summary FROM institutes WHERE cik = ?", (cik,))
+    cursor.execute("SELECT cik, name, numberOfStocks, performancePercentage3year, averageHoldingPeriod, marketValue, winRate, holdings FROM institutes WHERE cik = ?", (cik,))
     cik_data = cursor.fetchall()
     res = [{
         'cik': row[0],
         'name': row[1],
         'numberOfStocks': row[2],
         'performancePercentage3Year': row[3],
-        'averageHoldingPeriod': row[6],
+        'averageHoldingPeriod': row[4],
-        'marketValue': row[8],
+        'marketValue': row[5],
-        'winRate': row[9],
+        'winRate': row[6],
-        'holdings': orjson.loads(row[10]),
+        'holdings': orjson.loads(row[7]),
     } for row in cik_data]
 
     if not res:
         return None  # Exit if no data is found
 
+    '''
+    filtered_data = []
+    for item in res:
+        try:
+            filtered_data+=item['holdings']
+        except:
+            pass
+    filtered_data = [item for item in filtered_data if datetime.strptime(item['date'], "%Y-%m-%d") >= cutoff_date]
+    print(filtered_data)
+    '''
+
     res = res[0] #latest data
 
     filtered_holdings = [
@@ -196,7 +209,7 @@ if __name__ == '__main__':
     cursor.execute("SELECT DISTINCT cik FROM institutes")
     cik_symbols = [row[0] for row in cursor.fetchall()]
     #Test mode
-    #cik_symbols = ['0000102909']
+    #cik_symbols = ['0001649339']
     try:
         stock_cursor = stock_con.cursor()
         stock_cursor.execute("SELECT DISTINCT symbol, sector FROM stocks")

app/primary_cron_job.py

@@ -344,6 +344,7 @@ schedule.every(1).hours.do(run_threaded, run_cron_company_news).tag('company_new
 
 schedule.every(2).minutes.do(run_threaded, run_dashboard).tag('dashboard_job')
 
+
 schedule.every(20).seconds.do(run_threaded, run_if_not_running(run_cron_options_flow, 'options_flow_job')).tag('options_flow_job')
 
 

app/restart_json.py

@@ -9,6 +9,7 @@ import aiofiles
 import sqlite3
 import pandas as pd
 import numpy as np
+import math
 from collections import defaultdict
 from collections import Counter
 import re
@@ -193,7 +194,7 @@ def process_financial_data(file_path, key_list):
                         value = float(res[key])
                         if 'growth' in file_path or key in ['grossProfitMargin','netProfitMargin','pretaxProfitMargin','operatingProfitMargin','longTermDebtToCapitalization','totalDebtToCapitalization']:
                             value *= 100  # Multiply by 100 for percentage
-                        data[key] = round(value, 2)
+                        data[key] = round(value, 2) if value is not None else None
                     except (ValueError, TypeError):
                         # If there's an issue converting the value, leave it as None
                         data[key] = None
@@ -447,7 +448,6 @@ def get_financial_statements(item, symbol):
         item['ebitMargin'] = None
 
 
-
     return item
 
 def get_halal_compliant(item, debt_threshold=30, interest_threshold=30, revenue_threshold=5, liquidity_threshold=30, forbidden_industries=None):
@@ -819,6 +819,13 @@ async def get_stock_screener(con):
             item['netIncomeGrowthYears'] = None
             item['grossProfitGrowthYears'] = None
 
+    for item in stock_screener_data:
+        for key, value in item.items():
+            if isinstance(value, float):
+                if math.isnan(value) or math.isinf(value):
+                    item[key] = None
+                    print(key)
+
     return stock_screener_data