AmineTMIRI/backend
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

bugfixing

Browse Source
This commit is contained in:
MuslemRahimi 2025-01-10 18:34:13 +01:00
parent 907ef0545b
commit 1f312deae2
1 changed files with 6 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
app/cron_market_news.py
View File

@ -8,20 +8,18 @@ load_dotenv()
api_key = os.getenv('FMP_API_KEY') api_key = os.getenv('FMP_API_KEY')
headers = {"accept": "application/json"}
def filter_and_deduplicate(data,deduplicate_key='title'): def filter_and_deduplicate(data,deduplicate_key='title'):
seen_keys = set() seen_keys = set()
filtered_data = [] filtered_data = []
excluded_domains = ['accesswire.com']
for item in data: for item in data:
key = item.get(deduplicate_key) if not any(domain in item['url'] for domain in excluded_domains):
if key and key not in seen_keys: key = item.get(deduplicate_key)
filtered_data.append(item) if key and key not in seen_keys:
seen_keys.add(key) filtered_data.append(item)
seen_keys.add(key)
return filtered_data return filtered_data