optimize share statistics

app/cron_share_statistics.py

@@ -5,108 +5,96 @@ import pandas as pd
 from tqdm import tqdm
 from datetime import datetime
 import yfinance as yf
-import time
 
+# Constants
+JSON_DIR = "json/"
+QUARTERLY_FREQ = 'QE'
 
-async def save_as_json(symbol, forward_pe_dict, short_dict):
+# SQL Query
-    with open(f"json/share-statistics/{symbol}.json", 'w') as file:
+QUERY_TEMPLATE = """
-        ujson.dump(short_dict, file)
+    SELECT historicalShares
-    with open(f"json/forward-pe/{symbol}.json", 'w') as file:
+    FROM stocks
-        ujson.dump(forward_pe_dict, file)
+    WHERE symbol = ?
-
-
-query_template = f"""
-    SELECT 
-        historicalShares
-    FROM 
-        stocks
-    WHERE
-        symbol = ?
 """
 
-def filter_data_quarterly(data):
+def filter_quarterly_data(data):
-    # Generate a range of quarter-end dates from the start to the end date
+    """Filter data to keep only quarter-end dates."""
-    start_date = data[0]['date']
+    quarter_ends = pd.date_range(start=data[0]['date'], end=datetime.now(), freq=QUARTERLY_FREQ).strftime('%Y-%m-%d').tolist()
-    end_date = datetime.today().strftime('%Y-%m-%d')
+    return [entry for entry in data if entry['date'] in quarter_ends]
-    quarter_ends = pd.date_range(start=start_date, end=end_date, freq='QE').strftime('%Y-%m-%d').tolist()
 
-    # Filter data to keep only entries with dates matching quarter-end dates
+def get_yahoo_finance_data(ticker, shares):
-    filtered_data = [entry for entry in data if entry['date'] in quarter_ends]
+    """Fetch and process Yahoo Finance data."""
-    
-    return filtered_data
-
-def get_yahoo_data(ticker, outstanding_shares, float_shares):
     try:
-        data_dict = yf.Ticker(ticker).info
+        info = yf.Ticker(ticker).info
-        forward_pe = round(data_dict['forwardPE'],2)
+        return {
-        short_outstanding_percent = round((data_dict['sharesShort']/outstanding_shares)*100,2)
+            'forwardPE': round(info.get('forwardPE', 0), 2),
-        short_float_percent = round((data_dict['sharesShort']/float_shares)*100,2)
+            'short': {
-        return {'forwardPE': forward_pe}, {'sharesShort': data_dict['sharesShort'], 'shortRatio': data_dict['shortRatio'], 'sharesShortPriorMonth': data_dict['sharesShortPriorMonth'], 'shortOutStandingPercent': short_outstanding_percent, 'shortFloatPercent': short_float_percent}
+                'shares': info.get('sharesShort', 0),
-    except:
+                'ratio': info.get('shortRatio', 0),
-        return {'forwardPE': 0}, {'sharesShort': 0, 'shortRatio': 0, 'sharesShortPriorMonth': 0, 'shortOutStandingPercent': 0, 'shortFloatPercent': 0}
+                'priorMonth': info.get('sharesShortPriorMonth', 0),
-
+                'outstandingPercent': round((info.get('sharesShort', 0) / shares['outstandingShares']) * 100, 2),
-
+                'floatPercent': round((info.get('sharesShort', 0) / shares['floatShares']) * 100, 2)
-async def get_data(ticker, con):
+            }
-
+        }
-    try:
-        df = pd.read_sql_query(query_template, con, params=(ticker,))
-        shareholder_statistics = ujson.loads(df.to_dict()['historicalShares'][0])
-        # Keys to keep
-        keys_to_keep = ["date","floatShares", "outstandingShares"]
-
-        # Create new list with only the specified keys and convert floatShares and outstandingShares to integers
-        shareholder_statistics = [
-            {key: int(d[key]) if key in ["floatShares", "outstandingShares"] else d[key] 
-             for key in keys_to_keep}
-            for d in shareholder_statistics
-        ]
-
-        shareholder_statistics = sorted(shareholder_statistics, key=lambda x: datetime.strptime(x['date'], '%Y-%m-%d'), reverse=False)
-        
-        latest_outstanding_shares = shareholder_statistics[-1]['outstandingShares']
-        latest_float_shares = shareholder_statistics[-1]['floatShares']
-
-        # Filter out only quarter-end dates
-        historical_shares = filter_data_quarterly(shareholder_statistics)
-
-        forward_pe_data, short_data = get_yahoo_data(ticker, latest_outstanding_shares, latest_float_shares)
-        short_data = {**short_data, 'latestOutstandingShares': latest_outstanding_shares, 'latestFloatShares': latest_float_shares,'historicalShares': historical_shares}
     except Exception as e:
-        print(e)
+        #print(ticker)
-        short_data = {}
+        #print(e)
-        forward_pe_data = {}
+        #print("============")
+        return {'forwardPE': 0, 'short': {k: 0 for k in ['shares', 'ratio', 'priorMonth', 'outstandingPercent', 'floatPercent']}}
 
-    return forward_pe_data, short_data
+async def save_json(symbol, data):
+    """Save data to JSON files."""
+    for key, path in [("forwardPE", f"{JSON_DIR}forward-pe/{symbol}.json"), ("short", f"{JSON_DIR}share-statistics/{symbol}.json")]:
+        with open(path, 'w') as file:
+            ujson.dump(data.get(key, {}), file)
 
+async def process_ticker(ticker, con):
+    """Process a single ticker."""
+    try:
+        df = pd.read_sql_query(QUERY_TEMPLATE, con, params=(ticker,))
+        stats = ujson.loads(df.to_dict()['historicalShares'][0])
+        
+        # Filter and convert data
+        filtered_stats = [
+            {k: int(v) if k in ["floatShares", "outstandingShares"] else v 
+             for k, v in d.items() if k in ["date", "floatShares", "outstandingShares"]}
+            for d in sorted(stats, key=lambda x: datetime.strptime(x['date'], '%Y-%m-%d'))
+        ]
+        
+        latest_shares = filtered_stats[-1]
+        
+        quarterly_stats = filter_quarterly_data(filtered_stats)
+        
+        data = get_yahoo_finance_data(ticker, latest_shares)
+        data['short'].update({
+            'latestOutstandingShares': latest_shares['outstandingShares'],
+            'latestFloatShares': latest_shares['floatShares'],
+            'historicalShares': quarterly_stats
+        })
+        
+        await save_json(ticker, data)
+        return True
+    except Exception as e:
+        print(f"Error processing {ticker}: {e}")
+        return False
 
 async def run():
-
+    """Main function to process all tickers."""
     con = sqlite3.connect('stocks.db')
-
+    con.execute("PRAGMA journal_mode = wal")
-    cursor = con.cursor()
-    cursor.execute("PRAGMA journal_mode = wal")
-    cursor.execute("SELECT DISTINCT symbol FROM stocks WHERE symbol NOT LIKE '%.%'")
-    stock_symbols = [row[0] for row in cursor.fetchall()]
     
-    counter = 0
+    with con:
+        stock_symbols = [row[0] for row in con.execute("SELECT DISTINCT symbol FROM stocks WHERE symbol NOT LIKE '%.%'")]
 
+    processed = 0
     for ticker in tqdm(stock_symbols):
-        try:
+        if await process_ticker(ticker, con):
-            forward_pe_dict, short_dict = await get_data(ticker, con)
+            processed += 1
-            if forward_pe_dict.keys() and short_dict.keys():
+            if processed % 50 == 0:
-                await save_as_json(ticker, forward_pe_dict, short_dict)
+                print(f"Processed {processed} tickers, waiting for 60 seconds...")
-
-            counter += 1
-            if counter % 50 == 0:
-                print(f"Processed {counter} tickers, waiting for 60 seconds...")
                 await asyncio.sleep(60)
-        except Exception as e:
-            print(ticker, e)
-
 
     con.close()
 
-try:
+if __name__ == "__main__":
-    asyncio.run(run())
+    asyncio.run(run())
-except Exception as e:
-    print(e)