From c936d71e10752939f7bdf01f317e88c186294e8c Mon Sep 17 00:00:00 2001 From: MuslemRahimi Date: Thu, 10 Oct 2024 22:10:41 +0200 Subject: [PATCH] update cron job and watchlist api endpoint --- app/main.py | 96 ++++++------- app/primary_cron_job.py | 2 +- fastify/get-post/server.js | 288 +++++++++++++++++-------------------- 3 files changed, 184 insertions(+), 202 deletions(-) diff --git a/app/main.py b/app/main.py index d7ff0be..25fb835 100755 --- a/app/main.py +++ b/app/main.py @@ -1103,57 +1103,58 @@ async def get_watchlist(data: GetWatchList, api_key: str = Security(get_api_key) watchlist_id = data['watchListId'] result = pb.collection("watchlist").get_one(watchlist_id) ticker_list = result.ticker - - + rule_of_list = result.rule_of_list or [] + print(rule_of_list) combined_results = [] # List to store the combined results combined_news = [] - - for ticker in ticker_list: - ticker = ticker.upper() + + def load_json(file_path): + try: + with open(file_path, 'rb') as file: + return orjson.loads(file.read()) + except FileNotFoundError: + return None + + # Categorize tickers and fetch data + for ticker in map(str.upper, ticker_list): + ticker_type = 'stock' if ticker in etf_symbols: - try: - with open(f"json/quote/{ticker}.json", 'rb') as file: - quote_dict = orjson.loads(file.read()) - quote_dict['type'] = 'etf' - combined_results.append(quote_dict) - except: - pass - try: - with open(f"json/market-news/companies/{ticker}.json", 'rb') as file: - news_dict = orjson.loads(file.read())[0] - combined_news.append(news_dict) - except: - pass + ticker_type = 'etf' elif ticker in crypto_symbols: - try: - with open(f"json/quote/{ticker}.json", 'rb') as file: - quote_dict = orjson.loads(file.read()) - quote_dict['type'] = 'crypto' - combined_results.append(quote_dict) - except: - pass - try: - with open(f"json/market-news/companies/{ticker}.json", 'rb') as file: - news_dict = orjson.loads(file.read())[0] - combined_news.append(news_dict) - except: - pass - else: - try: - with open(f"json/quote/{ticker}.json", 'rb') as file: - quote_dict = orjson.loads(file.read()) - quote_dict['type'] = 'stock' - combined_results.append(quote_dict) - except: - pass - try: - with open(f"json/market-news/companies/{ticker}.json", 'rb') as file: - news_dict = orjson.loads(file.read())[0] - combined_news.append(news_dict) - except: - pass - res = [combined_results, combined_news] - return res + ticker_type = 'crypto' + + # Load quote data + quote_dict = load_json(f"json/quote/{ticker}.json") + if quote_dict: + quote_dict['type'] = ticker_type + combined_results.append(quote_dict) + + # Load news data + news_dict = load_json(f"json/market-news/companies/{ticker}.json") + if news_dict: + combined_news.append(news_dict[0]) + + # Keys to always include in the combined results + always_include = ['symbol', 'name', 'price', 'changesPercentage'] + + try: + # Create a mapping of stock_screener_data based on symbol for fast lookup + screener_dict = { + item['symbol']: {key: item.get(key) for key in set(always_include + rule_of_list) if key in item} + for item in stock_screener_data + } + + # Merge the filtered stock_screener_data into combined_results + for result in combined_results: + symbol = result.get('symbol') + if symbol in screener_dict: + result.update(screener_dict[symbol]) + + except Exception as e: + print(f"An error occurred while merging data: {e}") + print(combined_results) + return [combined_results, combined_news] + def process_option_activity(item): item['put_call'] = 'Calls' if item['put_call'] == 'CALL' else 'Puts' @@ -3207,7 +3208,6 @@ async def get_all_politician(api_key: str = Security(get_api_key)): headers={"Content-Encoding": "gzip"} ) - try: with open(f"json/congress-trading/search_list.json", 'rb') as file: res_list = orjson.loads(file.read()) diff --git a/app/primary_cron_job.py b/app/primary_cron_job.py index b2cf2bd..1ee3b17 100755 --- a/app/primary_cron_job.py +++ b/app/primary_cron_job.py @@ -371,7 +371,7 @@ schedule.every(10).minutes.do(run_threaded, run_tracker).tag('tracker_job') schedule.every(1).minutes.do(run_threaded, run_cron_quote).tag('quote_job') schedule.every(15).minutes.do(run_threaded, run_market_moods).tag('market_moods_job') -schedule.every(30).minutes.do(run_threaded, run_earnings).tag('earnings_job') +schedule.every(10).minutes.do(run_threaded, run_earnings).tag('earnings_job') #schedule.every(10).minutes.do(run_threaded, run_dark_pool_flow).tag('dark_pool_flow_job') schedule.every(2).hours.do(run_threaded, run_fda_calendar).tag('fda_calendar_job') diff --git a/fastify/get-post/server.js b/fastify/get-post/server.js index 4a0ca22..e239c9f 100755 --- a/fastify/get-post/server.js +++ b/fastify/get-post/server.js @@ -1,189 +1,171 @@ // Optimized postHotness function function postHotness(upvotes, created) { - let s = 0; - for (let i = 1; i <= upvotes; i++) { - if (i <= 3) { - s += 1; - } else if (i <= 6) { - s += 3; - } else if (i <= 10) { - s += 3; - } else if (i <= 20) { - s += 4; - } else if (i <= 40) { - s += 5; - } else { - s += 6; - } + let s = 0; + for (let i = 1; i <= upvotes; i++) { + if (i <= 3) { + s += 1; + } else if (i <= 6) { + s += 3; + } else if (i <= 10) { + s += 3; + } else if (i <= 20) { + s += 4; + } else if (i <= 40) { + s += 5; + } else { + s += 6; } + } - const order = Math.log10(Math.max(Math.abs(s), 1)); - let sign = 0; - if (s > 0) { - sign = 1; - } else if (s < 0) { - sign = -1; - } + const order = Math.log10(Math.max(Math.abs(s), 1)); + let sign = 0; + if (s > 0) { + sign = 1; + } else if (s < 0) { + sign = -1; + } - const interval = 45000; // or 69000 - const createdDate = new Date(created); - const seconds = (createdDate.getTime() / 1000); - const hotness = order + (sign * seconds) / interval; - return Math.round(hotness * 10000000); + const interval = 45000; // or 69000 + const createdDate = new Date(created); + const seconds = createdDate.getTime() / 1000; + const hotness = order + (sign * seconds) / interval; + return Math.round(hotness * 10000000); } - module.exports = function (fastify, opts, done) { - - const pb = opts.pb; + const pb = opts.pb; - fastify.post('/get-post', async (request, reply) => { + fastify.post("/get-post", async (request, reply) => { const data = request.body; let filter; - const sort = data?.sortingPosts === 'hot' ? '-upvote' : '-created'; + const sort = data?.sortingPosts === "hot" ? "-upvote" : "-created"; let pinnedPost; let posts; try { + if (data?.seenPostId.length !== 0) { + filter = data?.seenPostId?.map((id) => `id!="${id}"`).join("&&"); - - if (data?.seenPostId.length !==0) - { - filter = data?.seenPostId?.map((id) => `id!="${id}"`).join("&&"); - - //applies only for profile and user directory - if (data?.userId) { - filter += `&& user="${data?.userId}" && pinned=false`; - } - - if (data?.filterTicker) { - filter += `&& tagline="${data?.filterTicker}" && pinned=false`; - } - - if(data?.sortingPosts === 'hot') { - //community page code space - // In case of sort === 'hot' show the most recent post up to 7 week by ranking them with the function postHotness - - let endDate = new Date(); - // Get the date one week earlier - let startDate = new Date(); - startDate.setDate(endDate.getDate() - 30); - endDate.setDate(endDate.getDate() + 1) - - // Format the dates as needed (e.g., "YYYY-MM-DD") - let endDateStr = endDate.toISOString().split('T')[0]; - let startDateStr = startDate.toISOString().split('T')[0]; - - filter += `&& created >= "${startDateStr}" && created <= "${endDateStr}" && pinned = false` - } - - posts = (await pb.collection('posts').getList(data?.startPage, 5, { - sort: '-created', - filter: filter, - expand: 'user,comments(post),alreadyVoted(post)', - fields: "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type" - - }))?.items; - - if(data?.sortingPosts === 'hot') { - // Add hotness property to each post - posts?.forEach(post => { - post.hotness = postHotness(post?.upvote, post?.created); - }); - posts?.sort((a, b) => b?.hotness - a?.hotness); - } - + //applies only for profile and user directory + if (data?.userId) { + filter += `&& user="${data?.userId}" && pinned=false`; } - else { - if (data?.userId) { - - posts = (await pb.collection('posts').getList(data?.startPage, 5, { - sort: sort, - filter: `user="${data?.userId}" && pinned=false`, - expand: `user,comments(post),alreadyVoted(post)`, - fields: "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type" - }))?.items; + if (data?.filterTicker) { + filter += `&& tagline="${data?.filterTicker}" && pinned=false`; + } + if (data?.sortingPosts === "hot") { + //community page code space + // In case of sort === 'hot' show the most recent post up to 7 week by ranking them with the function postHotness - } + let endDate = new Date(); + // Get the date one week earlier + let startDate = new Date(); + startDate.setDate(endDate.getDate() - 180); + endDate.setDate(endDate.getDate() + 1); - else if (data?.filterTicker) { - - posts = await pb.collection('posts').getList(data?.startPage, 5, { - sort: sort, - filter: `tagline="${data?.filterTicker}" && pinned=false`, - expand: `user,comments(post),alreadyVoted(post)`, - fields: "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type" - }) ; - - } + // Format the dates as needed (e.g., "YYYY-MM-DD") + let endDateStr = endDate.toISOString().split("T")[0]; + let startDateStr = startDate.toISOString().split("T")[0]; - else { - //community page code space - // In case of sort === 'hot' show the most recent post up to 7 week by ranking them with the function postHotness - - if(data?.sortingPosts === 'hot') { + filter += `&& created >= "${startDateStr}" && created <= "${endDateStr}" && pinned = false`; + } - let endDate = new Date(); - // Get the date one week earlier - let startDate = new Date(); - startDate.setDate(endDate.getDate() - 30); - endDate.setDate(endDate.getDate() + 1) + posts = ( + await pb.collection("posts").getList(data?.startPage, 10, { + sort: "-created", + filter: filter, + expand: "user,comments(post),alreadyVoted(post)", + fields: + "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type", + }) + )?.items; - // Format the dates as needed (e.g., "YYYY-MM-DD") - let endDateStr = endDate.toISOString().split('T')[0]; - let startDateStr = startDate.toISOString().split('T')[0]; + if (data?.sortingPosts === "hot") { + // Add hotness property to each post + posts?.forEach((post) => { + post.hotness = postHotness(post?.upvote, post?.created); + }); + posts?.sort((a, b) => b?.hotness - a?.hotness); + } + } else { + if (data?.userId) { + posts = ( + await pb.collection("posts").getList(data?.startPage, 10, { + sort: sort, + filter: `user="${data?.userId}" && pinned=false`, + expand: `user,comments(post),alreadyVoted(post)`, + fields: + "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type", + }) + )?.items; + } else if (data?.filterTicker) { + posts = await pb.collection("posts").getList(data?.startPage, 10, { + sort: sort, + filter: `tagline="${data?.filterTicker}" && pinned=false`, + expand: `user,comments(post),alreadyVoted(post)`, + fields: + "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type", + }); + } else { + //community page code space + // In case of sort === 'hot' show the most recent post up to 7 week by ranking them with the function postHotness - filter = `created >= "${startDateStr}" && created <= "${endDateStr}" && pinned = false` - } - else { - filter = `pinned=false`; - } - posts = await pb.collection('posts').getList(1, 5, { - sort: '-created', - filter: filter, - expand: 'user, comments(post), alreadyVoted(post)', - fields: "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type" - - }); + if (data?.sortingPosts === "hot") { + let endDate = new Date(); + // Get the date one week earlier + let startDate = new Date(); + startDate.setDate(endDate.getDate() - 30); + endDate.setDate(endDate.getDate() + 1); - posts = posts.items - // Add hotness property to each post - if(data?.sortingPosts === 'hot') { - posts?.forEach(post => { - post.hotness = postHotness(post?.upvote, post?.created); - }); + // Format the dates as needed (e.g., "YYYY-MM-DD") + let endDateStr = endDate.toISOString().split("T")[0]; + let startDateStr = startDate.toISOString().split("T")[0]; - posts?.sort((a, b) => b?.hotness - a?.hotness); - } + filter = `created >= "${startDateStr}" && created <= "${endDateStr}" && pinned = false`; + } else { + filter = `pinned=false`; + } + posts = await pb.collection("posts").getList(1, 10, { + sort: "-created", + filter: filter, + expand: "user, comments(post), alreadyVoted(post)", + fields: + "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type", + }); - pinnedPost = await pb.collection('posts').getFullList({ - filter: `pinned=true`, - sort: '-created', - expand: `user,comments(post),alreadyVoted(post)`, - fields: "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type" - }); - - for (let i = pinnedPost?.length - 1; i >= 0; i--) { - posts?.unshift(pinnedPost[i]); - } + posts = posts.items; + // Add hotness property to each post + if (data?.sortingPosts === "hot") { + posts?.forEach((post) => { + post.hotness = postHotness(post?.upvote, post?.created); + }); - } + posts?.sort((a, b) => b?.hotness - a?.hotness); + } - } + pinnedPost = await pb.collection("posts").getFullList({ + filter: `pinned=true`, + sort: "-created", + expand: `user,comments(post),alreadyVoted(post)`, + fields: + "*,expand.user,expand.comments(post), expand.alreadyVoted(post).user,expand.alreadyVoted(post).type", + }); - } - catch(e) - { - //console.log(e) - posts = []; + for (let i = pinnedPost?.length - 1; i >= 0; i--) { + posts?.unshift(pinnedPost[i]); + } + } + } + } catch (e) { + //console.log(e) + posts = []; } + reply.send({ items: posts }); + }); - reply.send({ items: posts }) - }) - - done(); + done(); };