diff --git a/cognee/tests/integration/web_url_crawler/test_add.py b/cognee/tests/integration/web_url_crawler/test_add.py new file mode 100644 index 000000000..b9840df3d --- /dev/null +++ b/cognee/tests/integration/web_url_crawler/test_add.py @@ -0,0 +1,26 @@ +import pytest +import cognee + + +@pytest.mark.asyncio +async def test_add_fails_when_preferred_loader_not_specified(): + await cognee.prune.prune_data() + await cognee.prune.prune_system(metadata=True) + with pytest.raises: + await cognee.add( + "https://en.wikipedia.org/wiki/Large_language_model", + preferred_loaders=["web_url_loader"], + ) + + +@pytest.mark.asyncio +async def test_add_succesfully_adds_url_when_preferred_loader_specified(): + await cognee.prune.prune_data() + await cognee.prune.prune_system(metadata=True) + try: + await cognee.add( + "https://en.wikipedia.org/wiki/Large_language_model", + preferred_loaders=["web_url_loader"], + ) + except Exception as e: + pytest.fail(f"Failed to add url: {e}") diff --git a/cognee/tests/integration/web_url_crawler/test_loader_engine.py b/cognee/tests/integration/web_url_crawler/test_loader_engine.py new file mode 100644 index 000000000..018c034e1 --- /dev/null +++ b/cognee/tests/integration/web_url_crawler/test_loader_engine.py @@ -0,0 +1,20 @@ +import pytest + +from cognee.infrastructure.loaders import get_loader_engine +from cognee.infrastructure.loaders.external.web_url_loader import WebUrlLoader + + +def test_get_loader_returns_none_by_default_for_web_urls(): + loader_engine = get_loader_engine() + urls = ["https://cognee.ai", "http://cognee.ai"] + for url in urls: + loader = loader_engine.get_loader(url) + assert loader is None + + +def test_get_loader_returns_valid_loader_when_preferred_loaders_specified(): + loader_engine = get_loader_engine() + urls = ["https://cognee.ai", "http://cognee.ai"] + for url in urls: + loader = loader_engine.get_loader(url, preferred_loaders=["web_url_loader"]) + assert isinstance(loader, WebUrlLoader)