gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ruff formatted

Browse source
This commit is contained in:
Geoff-Robin 2025-10-07 22:16:02 +05:30
parent fc660e4027
commit 0fd55a737f
2 changed files with 6 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
cognee/tasks/web_scraper/utils.py
View file

@ -68,7 +68,7 @@ async def fetch_page_content(
max_retries=soup_crawler_config.max_retries, max_retries=soup_crawler_config.max_retries,
retry_delay_factor=soup_crawler_config.retry_delay_factor, retry_delay_factor=soup_crawler_config.retry_delay_factor,
headers=soup_crawler_config.headers, headers=soup_crawler_config.headers,
robots_cache_ttl=soup_crawler_config.robots_cache_ttl robots_cache_ttl=soup_crawler_config.robots_cache_ttl,
) )
try: try:
results = await crawler.fetch_with_bs4( results = await crawler.fetch_with_bs4(
@ -110,13 +110,13 @@ async def fetch_with_tavily(
raise raise
client = AsyncTavilyClient( client = AsyncTavilyClient(
api_key=tavily_config.api_key if tavily_config else None, api_key=tavily_config.api_key if tavily_config else None,
proxies=tavily_config.proxies if tavily_config else None proxies=tavily_config.proxies if tavily_config else None,
) )
results = await client.extract( results = await client.extract(
urls, urls,
format="text", format="text",
extract_depth=tavily_config.extract_depth if tavily_config else "basic", extract_depth=tavily_config.extract_depth if tavily_config else "basic",
timeout=tavily_config.timeout if tavily_config else 10 timeout=tavily_config.timeout if tavily_config else 10,
) )
for failed_result in results.get("failed_results", []): for failed_result in results.get("failed_results", []):
logger.warning(f"Failed to fetch {failed_result}") logger.warning(f"Failed to fetch {failed_result}")