diff --git a/static/robots.txt b/static/robots.txt index 2d1aa039..be5c9a31 100644 --- a/static/robots.txt +++ b/static/robots.txt @@ -1,3 +1,6 @@ -# https://www.robotstxt.org/robotstxt.html +# Allow all web crawlers full access to all content User-agent: * -Disallow: / \ No newline at end of file +Disallow: + +# Sitemap for better indexing +Sitemap: https://stocknear.com/sitemap.xml