From 9a9f9f6836859db629024ffedce34cefba8700a9 Mon Sep 17 00:00:00 2001
From: Daulet Amirkhanov <damirkhanov01@gmail.com>
Date: Thu, 16 Oct 2025 13:58:38 +0100
Subject: [PATCH] tests: add some tests to assert behaviour is as expected

---
 .../integration/web_url_crawler/test_add.py   | 26 +++++++++++++++++++
 .../web_url_crawler/test_loader_engine.py     | 20 ++++++++++++++
 2 files changed, 46 insertions(+)
 create mode 100644 cognee/tests/integration/web_url_crawler/test_add.py
 create mode 100644 cognee/tests/integration/web_url_crawler/test_loader_engine.py

diff --git a/cognee/tests/integration/web_url_crawler/test_add.py b/cognee/tests/integration/web_url_crawler/test_add.py
new file mode 100644
index 000000000..b9840df3d
--- /dev/null
+++ b/cognee/tests/integration/web_url_crawler/test_add.py
@@ -0,0 +1,26 @@
+import pytest
+import cognee
+
+
+@pytest.mark.asyncio
+async def test_add_fails_when_preferred_loader_not_specified():
+    await cognee.prune.prune_data()
+    await cognee.prune.prune_system(metadata=True)
+    with pytest.raises:
+        await cognee.add(
+            "https://en.wikipedia.org/wiki/Large_language_model",
+            preferred_loaders=["web_url_loader"],
+        )
+
+
+@pytest.mark.asyncio
+async def test_add_succesfully_adds_url_when_preferred_loader_specified():
+    await cognee.prune.prune_data()
+    await cognee.prune.prune_system(metadata=True)
+    try:
+        await cognee.add(
+            "https://en.wikipedia.org/wiki/Large_language_model",
+            preferred_loaders=["web_url_loader"],
+        )
+    except Exception as e:
+        pytest.fail(f"Failed to add url: {e}")
diff --git a/cognee/tests/integration/web_url_crawler/test_loader_engine.py b/cognee/tests/integration/web_url_crawler/test_loader_engine.py
new file mode 100644
index 000000000..018c034e1
--- /dev/null
+++ b/cognee/tests/integration/web_url_crawler/test_loader_engine.py
@@ -0,0 +1,20 @@
+import pytest
+
+from cognee.infrastructure.loaders import get_loader_engine
+from cognee.infrastructure.loaders.external.web_url_loader import WebUrlLoader
+
+
+def test_get_loader_returns_none_by_default_for_web_urls():
+    loader_engine = get_loader_engine()
+    urls = ["https://cognee.ai", "http://cognee.ai"]
+    for url in urls:
+        loader = loader_engine.get_loader(url)
+        assert loader is None
+
+
+def test_get_loader_returns_valid_loader_when_preferred_loaders_specified():
+    loader_engine = get_loader_engine()
+    urls = ["https://cognee.ai", "http://cognee.ai"]
+    for url in urls:
+        loader = loader_engine.get_loader(url, preferred_loaders=["web_url_loader"])
+        assert isinstance(loader, WebUrlLoader)