diff --git a/docs/.gitignore b/docs/.gitignore index 0247a7ea..c010f4db 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -23,3 +23,4 @@ yarn-error.log* !package.json !package-lock.json !yarn.lock +!scraper.config.json diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index ab33c338..f0767fa6 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -25,6 +25,9 @@ const config = { // For GitHub pages deployment, it is often '//' baseUrl: process.env.BASE_URL ? process.env.BASE_URL : '/', + // Control search engine indexing - set to true to prevent indexing + noIndex: true, + // GitHub pages deployment config. // If you aren't using GitHub pages, you don't need these. organizationName: 'langflow-ai', // Usually your GitHub org/user name. diff --git a/docs/package.json b/docs/package.json index 0bbba668..0ca96c29 100644 --- a/docs/package.json +++ b/docs/package.json @@ -6,6 +6,7 @@ "docusaurus": "docusaurus", "start": "docusaurus start", "build": "docusaurus build", + "build:pdf": "npm run build && npm run serve & sleep 10 && npx docusaurus-to-pdf && pkill -f 'docusaurus serve'", "swizzle": "docusaurus swizzle", "deploy": "docusaurus deploy", "clear": "docusaurus clear", diff --git a/docs/pdf/openrag-documentation.pdf b/docs/pdf/openrag-documentation.pdf new file mode 100644 index 00000000..cec6d2aa Binary files /dev/null and b/docs/pdf/openrag-documentation.pdf differ diff --git a/docs/scraper.config.json b/docs/scraper.config.json new file mode 100644 index 00000000..94c57806 --- /dev/null +++ b/docs/scraper.config.json @@ -0,0 +1,7 @@ +{ + "baseUrl": "http://localhost:3000", + "entryPoint": "http://localhost:3000", + "outputDir": "./pdf/openrag-documentation.pdf", + "customStyles": "table { max-width: 3500px !important; } .navbar, .footer, .breadcrumbs { display: none !important; }", + "forceImages": true +} \ No newline at end of file diff --git a/docs/static/robots.txt b/docs/static/robots.txt new file mode 100644 index 00000000..783e3e94 --- /dev/null +++ b/docs/static/robots.txt @@ -0,0 +1,12 @@ +# Robots.txt for OpenRAG Documentation + +# Block all crawlers by default +User-agent: * +Disallow: / + +# Allow specific crawlers if needed (uncomment when ready for launch) +# User-agent: Googlebot +# Allow: / + +# Sitemap location (uncomment when ready for launch) +# Sitemap: https://docs.openr.ag/sitemap.xml