From b0c5a78e35132bfee395294d00e00ca2a086e35f Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Thu, 18 Dec 2025 15:00:01 -0800 Subject: [PATCH 1/3] prepare robots.txt for public site --- docs/static/robots.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/static/robots.txt b/docs/static/robots.txt index 783e3e94..da3e89b6 100644 --- a/docs/static/robots.txt +++ b/docs/static/robots.txt @@ -4,9 +4,9 @@ User-agent: * Disallow: / -# Allow specific crawlers if needed (uncomment when ready for launch) -# User-agent: Googlebot -# Allow: / +# Allow specific crawlers +User-agent: Googlebot +Allow: / -# Sitemap location (uncomment when ready for launch) -# Sitemap: https://docs.openr.ag/sitemap.xml +# Sitemap location +Sitemap: https://docs.openr.ag/sitemap.xml \ No newline at end of file From 296842df3423f43c82bac874e1f902d2a0cb2ede Mon Sep 17 00:00:00 2001 From: April M <36110273+aimurphy@users.noreply.github.com> Date: Thu, 18 Dec 2025 16:00:16 -0800 Subject: [PATCH 2/3] allow all crawlers --- docs/static/robots.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/static/robots.txt b/docs/static/robots.txt index da3e89b6..8452acdf 100644 --- a/docs/static/robots.txt +++ b/docs/static/robots.txt @@ -1,11 +1,7 @@ # Robots.txt for OpenRAG Documentation -# Block all crawlers by default +# Allow all crawlers User-agent: * -Disallow: / - -# Allow specific crawlers -User-agent: Googlebot Allow: / # Sitemap location From 13d1c8bb45052263a880ac343f0723e4c6839db1 Mon Sep 17 00:00:00 2001 From: Eric Hare Date: Mon, 22 Dec 2025 13:54:39 -0800 Subject: [PATCH 3/3] fix: Follow redirects for OneDrive + SharePoint --- src/connectors/onedrive/connector.py | 4 ++-- src/connectors/sharepoint/connector.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/connectors/onedrive/connector.py b/src/connectors/onedrive/connector.py index dcca775d..a902f577 100644 --- a/src/connectors/onedrive/connector.py +++ b/src/connectors/onedrive/connector.py @@ -394,7 +394,7 @@ class OneDriveConnector(BaseConnector): headers = {"Authorization": f"Bearer {token}"} async with httpx.AsyncClient() as client: - response = await client.get(url, headers=headers, timeout=60) + response = await client.get(url, headers=headers, timeout=60, follow_redirects=True) response.raise_for_status() return response.content @@ -406,7 +406,7 @@ class OneDriveConnector(BaseConnector): """Download file content from direct download URL.""" try: async with httpx.AsyncClient() as client: - response = await client.get(download_url, timeout=60) + response = await client.get(download_url, timeout=60, follow_redirects=True) response.raise_for_status() return response.content except Exception as e: diff --git a/src/connectors/sharepoint/connector.py b/src/connectors/sharepoint/connector.py index f9482d9c..1235a103 100644 --- a/src/connectors/sharepoint/connector.py +++ b/src/connectors/sharepoint/connector.py @@ -462,7 +462,7 @@ class SharePointConnector(BaseConnector): headers = {"Authorization": f"Bearer {token}"} async with httpx.AsyncClient() as client: - response = await client.get(url, headers=headers, timeout=60) + response = await client.get(url, headers=headers, timeout=60, follow_redirects=True) response.raise_for_status() return response.content @@ -535,7 +535,7 @@ class SharePointConnector(BaseConnector): """Download file content from direct download URL""" try: async with httpx.AsyncClient() as client: - response = await client.get(download_url, timeout=60) + response = await client.get(download_url, timeout=60, follow_redirects=True) response.raise_for_status() return response.content except Exception as e: