AmineTMIRI/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update lobbying tracker

Browse Source
This commit is contained in:
MuslemRahimi 2024-08-09 22:03:59 +02:00
parent 21ba8b8a5b
commit 26e71556a8
2 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/cron_corporate_lobbying.py
View File

@ -385,10 +385,10 @@ def create_dataset():
print(f"Total stocks: {len(stock_data)}") print(f"Total stocks: {len(stock_data)}")
con.close() con.close()
batch_size = 5 batch_size = 3
stock_batches = [stock_data[i:i+batch_size] for i in range(0, len(stock_data), batch_size)] stock_batches = [stock_data[i:i+batch_size] for i in range(0, len(stock_data), batch_size)]
with concurrent.futures.ProcessPoolExecutor(max_workers=5) as executor: with concurrent.futures.ProcessPoolExecutor(max_workers=3) as executor:
futures = [executor.submit(process_stocks_batch, batch, csv_files, reports_folder, threshold) for batch in stock_batches] futures = [executor.submit(process_stocks_batch, batch, csv_files, reports_folder, threshold) for batch in stock_batches]
for future in concurrent.futures.as_completed(futures): for future in concurrent.futures.as_completed(futures):

3
app/cron_lobbying_tracker.py
View File

@ -55,11 +55,12 @@ def main():
company = columns[0].find_element(By.TAG_NAME, 'span').text company = columns[0].find_element(By.TAG_NAME, 'span').text
amount = columns[1].text.strip() amount = columns[1].text.strip()
date = columns[2].text.strip() date = columns[2].text.strip()
amount_int = int(amount.replace('$', '').replace(',', ''))
data.append({ data.append({
'ticker': ticker, 'ticker': ticker,
'company': company, 'company': company,
'amount': amount, 'amount': amount_int,
'date': date 'date': date
}) })