From 3003a2fb20acf4f5498d4ca8862410086c6c1dd7 Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Fri, 22 Nov 2024 15:48:56 +0100 Subject: [PATCH] update reddit tracker --- app/cron_reddit_statistics.py | 114 +++++++++++++++++++--------------- app/cron_reddit_tracker.py | 22 +++++-- app/main.py | 4 +- 3 files changed, 84 insertions(+), 56 deletions(-) diff --git a/app/cron_reddit_statistics.py b/app/cron_reddit_statistics.py index 5a0a849..bace780 100644 --- a/app/cron_reddit_statistics.py +++ b/app/cron_reddit_statistics.py @@ -123,58 +123,72 @@ def compute_daily_statistics(file_path): def compute_trending_tickers(daily_stats): today = datetime.now().date() - seven_days_ago = today - timedelta(days=14) - - trending = defaultdict(lambda: {'total': 0, 'PUT': 0, 'CALL': 0, 'sentiment': []}) - - for date, stats in daily_stats.items(): - if seven_days_ago <= date <= today: - for ticker, counts in stats['ticker_mentions'].items(): - trending[ticker]['total'] += counts['total'] - trending[ticker]['PUT'] += counts['PUT'] - trending[ticker]['CALL'] += counts['CALL'] - trending[ticker]['sentiment'].extend(counts['sentiment']) - - trending_list = [ - { - 'symbol': symbol, - 'count': counts['total'], - 'put': counts['PUT'], - 'call': counts['CALL'], - 'avgSentiment': round(sum(counts['sentiment']) / len(counts['sentiment']),2) if counts['sentiment'] else 0 - } - for symbol, counts in trending.items() if symbol in total_symbols - ] - trending_list.sort(key=lambda x: x['count'], reverse=True) + period_list = [2,7,30,90] + res_dict = {} - for item in trending_list: - symbol = item['symbol'] - try: - with open(f'json/quote/{symbol}.json') as f: - data = json.load(f) - name = data['name'] - price = round(data['price'],2) - changes_percentage = round(data['changesPercentage'],2) - except Exception as e: - print(e) - name = None - price = None - changes_percentage = None + for time_period in period_list: + res_list = [] - if symbol in stock_symbols: - item['assetType'] = 'stocks' - item['name'] = name - item['price'] = price - item['changesPercentage'] = changes_percentage - elif symbol in etf_symbols: - item['assetType'] = 'etf' - item['name'] = name - item['price'] = price - item['changesPercentage'] = changes_percentage - else: - item['assetType'] = '' - - return trending_list + N_day_ago = today - timedelta(days=time_period) + + trending = defaultdict(lambda: {'total': 0, 'PUT': 0, 'CALL': 0, 'sentiment': []}) + + for date, stats in daily_stats.items(): + if N_day_ago <= date <= today: + for ticker, counts in stats['ticker_mentions'].items(): + trending[ticker]['total'] += counts['total'] + trending[ticker]['PUT'] += counts['PUT'] + trending[ticker]['CALL'] += counts['CALL'] + trending[ticker]['sentiment'].extend(counts['sentiment']) + + res_list = [ + { + 'symbol': symbol, + 'count': counts['total'], + 'put': counts['PUT'], + 'call': counts['CALL'], + 'avgSentiment': round(sum(counts['sentiment']) / len(counts['sentiment']),2) if counts['sentiment'] else 0 + } + for symbol, counts in trending.items() if symbol in total_symbols + ] + res_list.sort(key=lambda x: x['count'], reverse=True) + + for item in res_list: + symbol = item['symbol'] + try: + with open(f'json/quote/{symbol}.json') as f: + data = json.load(f) + name = data['name'] + price = round(data['price'],2) + changes_percentage = round(data['changesPercentage'],2) + except Exception as e: + print(e) + name = None + price = None + changes_percentage = None + + if symbol in stock_symbols: + item['assetType'] = 'stocks' + item['name'] = name + item['price'] = price + item['changesPercentage'] = changes_percentage + elif symbol in etf_symbols: + item['assetType'] = 'etf' + item['name'] = name + item['price'] = price + item['changesPercentage'] = changes_percentage + else: + item['assetType'] = '' + + if time_period == 2: + res_dict['oneDay'] = res_list + elif time_period == 7: + res_dict['oneWeek'] = res_list + elif time_period == 30: + res_dict['oneMonth'] = res_list + elif time_period == 90: + res_dict['threeMonths'] = res_list + return res_dict # Usage file_path = 'json/reddit-tracker/wallstreetbets/data.json' diff --git a/app/cron_reddit_tracker.py b/app/cron_reddit_tracker.py index 99e53dd..8a7829a 100644 --- a/app/cron_reddit_tracker.py +++ b/app/cron_reddit_tracker.py @@ -48,8 +48,17 @@ data_changed = False subreddit = reddit.subreddit("wallstreetbets") # Iterate through new submissions -for submission in subreddit.new(limit=1000): +for submission in subreddit.hot(limit=1000): post_id = submission.id + + # Check if the post was deleted by moderators + if submission.removed_by_category == "mod": + # Remove post from existing data if it was deleted by moderators + if post_id in existing_posts: + del existing_posts[post_id] + data_changed = True + continue # Skip this post + # Check if this post is already in our data if post_id in existing_posts: # Update existing post @@ -57,14 +66,20 @@ for submission in subreddit.new(limit=1000): existing_posts[post_id]['num_comments'] = submission.num_comments data_changed = True else: + # Try to get a high-quality thumbnail URL + thumbnail = None + if hasattr(submission, 'preview'): + thumbnail = submission.preview['images'][0]['source']['url'] + # Extract the required fields for new post extracted_post = { "id": post_id, "permalink": submission.permalink, "title": submission.title, + "thumbnail": thumbnail, "selftext": submission.selftext, "created_utc": int(submission.created_utc), - "upvote_ratio": submission.upvote_ratio, + "upvote_ratio": round(submission.upvote_ratio * 100, 2), "num_comments": submission.num_comments, "link_flair_text": submission.link_flair_text, "author": str(submission.author), @@ -74,7 +89,6 @@ for submission in subreddit.new(limit=1000): existing_posts[post_id] = extracted_post data_changed = True - if data_changed: # Convert the dictionary back to a list and sort by created_utc updated_data = list(existing_posts.values()) @@ -84,4 +98,4 @@ if data_changed: save_data(updated_data) print(f"Data updated and saved to {file_path}") else: - print("No new data to add or update.") \ No newline at end of file + print("No new data to add or update.") diff --git a/app/main.py b/app/main.py index c0cd9c7..1e69e6c 100755 --- a/app/main.py +++ b/app/main.py @@ -3444,9 +3444,9 @@ async def get_reddit_tracker(api_key: str = Security(get_api_key)): try: with open(f"json/reddit-tracker/wallstreetbets/trending.json", 'rb') as file: - trending = orjson.loads(file.read())[0:5] + trending = orjson.loads(file.read()) except: - trending = [] + trending = {} res = {'posts': latest_post, 'stats': stats, 'trending': trending}