update lobbying tracker
This commit is contained in:
parent
21ba8b8a5b
commit
26e71556a8
@ -385,10 +385,10 @@ def create_dataset():
|
||||
print(f"Total stocks: {len(stock_data)}")
|
||||
con.close()
|
||||
|
||||
batch_size = 5
|
||||
batch_size = 3
|
||||
stock_batches = [stock_data[i:i+batch_size] for i in range(0, len(stock_data), batch_size)]
|
||||
|
||||
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor:
|
||||
with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:
|
||||
futures = [executor.submit(process_stocks_batch, batch, csv_files, reports_folder, threshold) for batch in stock_batches]
|
||||
|
||||
for future in concurrent.futures.as_completed(futures):
|
||||
|
||||
@ -55,11 +55,12 @@ def main():
|
||||
company = columns[0].find_element(By.TAG_NAME, 'span').text
|
||||
amount = columns[1].text.strip()
|
||||
date = columns[2].text.strip()
|
||||
amount_int = int(amount.replace('$', '').replace(',', ''))
|
||||
|
||||
data.append({
|
||||
'ticker': ticker,
|
||||
'company': company,
|
||||
'amount': amount,
|
||||
'amount': amount_int,
|
||||
'date': date
|
||||
})
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user