From 26e71556a80cfd1180274c3d07aaac818a0d2aa7 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Fri, 9 Aug 2024 22:03:59 +0200 Subject: [PATCH] update lobbying tracker --- app/cron_corporate_lobbying.py | 4 ++-- app/cron_lobbying_tracker.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app/cron_corporate_lobbying.py b/app/cron_corporate_lobbying.py index ae73618..6f6203b 100644 --- a/app/cron_corporate_lobbying.py +++ b/app/cron_corporate_lobbying.py @@ -385,10 +385,10 @@ def create_dataset(): print(f"Total stocks: {len(stock_data)}") con.close() - batch_size = 5 + batch_size = 3 stock_batches = [stock_data[i:i+batch_size] for i in range(0, len(stock_data), batch_size)] - with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor: + with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor: futures = [executor.submit(process_stocks_batch, batch, csv_files, reports_folder, threshold) for batch in stock_batches] for future in concurrent.futures.as_completed(futures): diff --git a/app/cron_lobbying_tracker.py b/app/cron_lobbying_tracker.py index 22eec59..d233f6a 100644 --- a/app/cron_lobbying_tracker.py +++ b/app/cron_lobbying_tracker.py @@ -55,11 +55,12 @@ def main(): company = columns[0].find_element(By.TAG_NAME, 'span').text amount = columns[1].text.strip() date = columns[2].text.strip() + amount_int = int(amount.replace('$', '').replace(',', '')) data.append({ 'ticker': ticker, 'company': company, - 'amount': amount, + 'amount': amount_int, 'date': date })