diff --git a/docs/static/robots.txt b/docs/static/robots.txt index da3e89b6..8452acdf 100644 --- a/docs/static/robots.txt +++ b/docs/static/robots.txt @@ -1,11 +1,7 @@ # Robots.txt for OpenRAG Documentation -# Block all crawlers by default +# Allow all crawlers User-agent: * -Disallow: / - -# Allow specific crawlers -User-agent: Googlebot Allow: / # Sitemap location