gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Solved more nitpick comments

Browse source
This commit is contained in:
Geoff-Robin 2025-10-06 18:16:31 +05:30
parent 791e38b2c0
commit 3c9e5f830b
2 changed files with 4 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cognee/tasks/web_scraper/utils.py
View file

@ -57,7 +57,10 @@ async def fetch_page_content(
logger.error( logger.error(
"Failed to import bs4, make sure to install using pip install beautifulsoup4>=4.13.1" "Failed to import bs4, make sure to install using pip install beautifulsoup4>=4.13.1"
) )
raise raise ImportError
if not soup_crawler_config or soup_crawler_config.extraction_rules is None:
raise ValueError("extraction_rules must be provided when not using Tavily")
extraction_rules = soup_crawler_config.extraction_rules
crawler = BeautifulSoupCrawler( crawler = BeautifulSoupCrawler(
concurrency=soup_crawler_config.concurrency, concurrency=soup_crawler_config.concurrency,
crawl_delay=soup_crawler_config.crawl_delay, crawl_delay=soup_crawler_config.crawl_delay,
@ -66,9 +69,6 @@ async def fetch_page_content(
retry_delay_factor=soup_crawler_config.retry_delay_factor, retry_delay_factor=soup_crawler_config.retry_delay_factor,
headers=soup_crawler_config.headers, headers=soup_crawler_config.headers,
) )
if not soup_crawler_config or soup_crawler_config.extraction_rules is None:
raise ValueError("extraction_rules must be provided when not using Tavily")
extraction_rules = soup_crawler_config.extraction_rules
try: try:
results = await crawler.fetch_with_bs4( results = await crawler.fetch_with_bs4(
urls, urls,

1
cognee/tests/tasks/web_scraping/web_scraping_test.py
View file

@ -28,7 +28,6 @@ async def test_web_scraping_using_bs4():
retry_delay_factor=0.5, retry_delay_factor=0.5,
extraction_rules=rules, extraction_rules=rules,
use_playwright=False, use_playwright=False,
structured=True,
) )
await cognee.add( await cognee.add(