Merge branch 'release-0.2.0' of github.com:phact/openrag into release-0.2.0
This commit is contained in:
commit
f3575a2bff
3 changed files with 8 additions and 12 deletions
12
docs/static/robots.txt
vendored
12
docs/static/robots.txt
vendored
|
|
@ -1,12 +1,8 @@
|
||||||
# Robots.txt for OpenRAG Documentation
|
# Robots.txt for OpenRAG Documentation
|
||||||
|
|
||||||
# Block all crawlers by default
|
# Allow all crawlers
|
||||||
User-agent: *
|
User-agent: *
|
||||||
Disallow: /
|
Allow: /
|
||||||
|
|
||||||
# Allow specific crawlers if needed (uncomment when ready for launch)
|
# Sitemap location
|
||||||
# User-agent: Googlebot
|
Sitemap: https://docs.openr.ag/sitemap.xml
|
||||||
# Allow: /
|
|
||||||
|
|
||||||
# Sitemap location (uncomment when ready for launch)
|
|
||||||
# Sitemap: https://docs.openr.ag/sitemap.xml
|
|
||||||
|
|
@ -394,7 +394,7 @@ class OneDriveConnector(BaseConnector):
|
||||||
headers = {"Authorization": f"Bearer {token}"}
|
headers = {"Authorization": f"Bearer {token}"}
|
||||||
|
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
response = await client.get(url, headers=headers, timeout=60)
|
response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return response.content
|
return response.content
|
||||||
|
|
||||||
|
|
@ -406,7 +406,7 @@ class OneDriveConnector(BaseConnector):
|
||||||
"""Download file content from direct download URL."""
|
"""Download file content from direct download URL."""
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
response = await client.get(download_url, timeout=60)
|
response = await client.get(download_url, timeout=60, follow_redirects=True)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return response.content
|
return response.content
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
|
|
@ -462,7 +462,7 @@ class SharePointConnector(BaseConnector):
|
||||||
headers = {"Authorization": f"Bearer {token}"}
|
headers = {"Authorization": f"Bearer {token}"}
|
||||||
|
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
response = await client.get(url, headers=headers, timeout=60)
|
response = await client.get(url, headers=headers, timeout=60, follow_redirects=True)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return response.content
|
return response.content
|
||||||
|
|
||||||
|
|
@ -535,7 +535,7 @@ class SharePointConnector(BaseConnector):
|
||||||
"""Download file content from direct download URL"""
|
"""Download file content from direct download URL"""
|
||||||
try:
|
try:
|
||||||
async with httpx.AsyncClient() as client:
|
async with httpx.AsyncClient() as client:
|
||||||
response = await client.get(download_url, timeout=60)
|
response = await client.get(download_url, timeout=60, follow_redirects=True)
|
||||||
response.raise_for_status()
|
response.raise_for_status()
|
||||||
return response.content
|
return response.content
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue