update lobbying tracker

This commit is contained in:
MuslemRahimi 2024-08-09 22:03:59 +02:00
parent 21ba8b8a5b
commit 26e71556a8
2 changed files with 4 additions and 3 deletions

View File

@ -385,10 +385,10 @@ def create_dataset():
print(f"Total stocks: {len(stock_data)}")
con.close()
batch_size = 5
batch_size = 3
stock_batches = [stock_data[i:i+batch_size] for i in range(0, len(stock_data), batch_size)]
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:
futures = [executor.submit(process_stocks_batch, batch, csv_files, reports_folder, threshold) for batch in stock_batches]
for future in concurrent.futures.as_completed(futures):

View File

@ -55,11 +55,12 @@ def main():
company = columns[0].find_element(By.TAG_NAME, 'span').text
amount = columns[1].text.strip()
date = columns[2].text.strip()
amount_int = int(amount.replace('$', '').replace(',', ''))
data.append({
'ticker': ticker,
'company': company,
'amount': amount,
'amount': amount_int,
'date': date
})