diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index ab33c338..f0767fa6 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -25,6 +25,9 @@ const config = { // For GitHub pages deployment, it is often '//' baseUrl: process.env.BASE_URL ? process.env.BASE_URL : '/', + // Control search engine indexing - set to true to prevent indexing + noIndex: true, + // GitHub pages deployment config. // If you aren't using GitHub pages, you don't need these. organizationName: 'langflow-ai', // Usually your GitHub org/user name. diff --git a/docs/static/robots.txt b/docs/static/robots.txt new file mode 100644 index 00000000..783e3e94 --- /dev/null +++ b/docs/static/robots.txt @@ -0,0 +1,12 @@ +# Robots.txt for OpenRAG Documentation + +# Block all crawlers by default +User-agent: * +Disallow: / + +# Allow specific crawlers if needed (uncomment when ready for launch) +# User-agent: Googlebot +# Allow: / + +# Sitemap location (uncomment when ready for launch) +# Sitemap: https://docs.openr.ag/sitemap.xml