tests: add some tests to assert behaviour is as expected

cognee/tests/integration/web_url_crawler/test_add.py

@@ -0,0 +1,26 @@
+import pytest
+import cognee
+
+
+@pytest.mark.asyncio
+async def test_add_fails_when_preferred_loader_not_specified():
+    await cognee.prune.prune_data()
+    await cognee.prune.prune_system(metadata=True)
+    with pytest.raises:
+        await cognee.add(
+            "https://en.wikipedia.org/wiki/Large_language_model",
+            preferred_loaders=["web_url_loader"],
+        )
+
+
+@pytest.mark.asyncio
+async def test_add_succesfully_adds_url_when_preferred_loader_specified():
+    await cognee.prune.prune_data()
+    await cognee.prune.prune_system(metadata=True)
+    try:
+        await cognee.add(
+            "https://en.wikipedia.org/wiki/Large_language_model",
+            preferred_loaders=["web_url_loader"],
+        )
+    except Exception as e:
+        pytest.fail(f"Failed to add url: {e}")

cognee/tests/integration/web_url_crawler/test_loader_engine.py

@@ -0,0 +1,20 @@
+import pytest
+
+from cognee.infrastructure.loaders import get_loader_engine
+from cognee.infrastructure.loaders.external.web_url_loader import WebUrlLoader
+
+
+def test_get_loader_returns_none_by_default_for_web_urls():
+    loader_engine = get_loader_engine()
+    urls = ["https://cognee.ai", "http://cognee.ai"]
+    for url in urls:
+        loader = loader_engine.get_loader(url)
+        assert loader is None
+
+
+def test_get_loader_returns_valid_loader_when_preferred_loaders_specified():
+    loader_engine = get_loader_engine()
+    urls = ["https://cognee.ai", "http://cognee.ai"]
+    for url in urls:
+        loader = loader_engine.get_loader(url, preferred_loaders=["web_url_loader"])
+        assert isinstance(loader, WebUrlLoader)