From 21ba8b8a5b30bf3f3ef5625726807cafd2b233ea Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Fri, 9 Aug 2024 16:19:09 +0200 Subject: [PATCH] bugfixing --- app/cron_corporate_lobbying.py | 6 +++--- app/cron_dashboard.py | 19 +++++++++++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/cron_corporate_lobbying.py b/app/cron_corporate_lobbying.py index e06d3a0..ae73618 100644 --- a/app/cron_corporate_lobbying.py +++ b/app/cron_corporate_lobbying.py @@ -380,15 +380,15 @@ def create_dataset(): con = sqlite3.connect('stocks.db') cursor = con.cursor() cursor.execute("PRAGMA journal_mode = wal") - cursor.execute("SELECT DISTINCT symbol, name FROM stocks WHERE marketCap >= 10E9 AND symbol NOT LIKE '%.%' AND symbol NOT LIKE '%-%'") + cursor.execute("SELECT DISTINCT symbol, name FROM stocks WHERE symbol NOT LIKE '%.%' AND symbol NOT LIKE '%-%'") stock_data = [{'symbol': row[0], 'name': row[1]} for row in cursor.fetchall()] print(f"Total stocks: {len(stock_data)}") con.close() - batch_size = 3 + batch_size = 5 stock_batches = [stock_data[i:i+batch_size] for i in range(0, len(stock_data), batch_size)] - with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor: + with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor: futures = [executor.submit(process_stocks_batch, batch, csv_files, reports_folder, threshold) for batch in stock_batches] for future in concurrent.futures.as_completed(futures): diff --git a/app/cron_dashboard.py b/app/cron_dashboard.py index 3408dc2..c735b90 100644 --- a/app/cron_dashboard.py +++ b/app/cron_dashboard.py @@ -8,7 +8,7 @@ import pytz import time import os from dotenv import load_dotenv -from datetime import datetime, timedelta +from datetime import datetime, timedelta, date import sqlite3 @@ -34,6 +34,20 @@ async def save_json(data): ujson.dump(data, file) +def parse_time(time_str): + try: + # Try parsing as full datetime + return datetime.strptime(time_str, '%Y-%m-%d %H:%M:%S') + except ValueError: + try: + # Try parsing as time only + time_obj = datetime.strptime(time_str, '%H:%M:%S').time() + # Combine with today's date + return datetime.combine(date.today(), time_obj) + except ValueError: + # If all else fails, return a default datetime + return datetime.min + def remove_duplicates(elements): seen = set() unique_elements = [] @@ -151,7 +165,8 @@ async def get_recent_earnings(session): except Exception as e: pass res_list = remove_duplicates(res_list) - res_list.sort(key=lambda x: x['marketCap'], reverse=True) + #res_list.sort(key=lambda x: x['marketCap'], reverse=True) + res_list.sort(key=lambda x: (-parse_time(x['time']).timestamp(), -x['marketCap'])) res_list = [{k: v for k, v in d.items() if k != 'marketCap'} for d in res_list] return res_list[0:5]