diff --git a/docs/static/robots.txt b/docs/static/robots.txt new file mode 100644 index 00000000..783e3e94 --- /dev/null +++ b/docs/static/robots.txt @@ -0,0 +1,12 @@ +# Robots.txt for OpenRAG Documentation + +# Block all crawlers by default +User-agent: * +Disallow: / + +# Allow specific crawlers if needed (uncomment when ready for launch) +# User-agent: Googlebot +# Allow: / + +# Sitemap location (uncomment when ready for launch) +# Sitemap: https://docs.openr.ag/sitemap.xml