gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

lazy load cron_web_scraper_task and web_scraper_task

Browse source
This commit is contained in:
Daulet Amirkhanov 2025-10-21 23:11:01 +01:00
parent f02aa1abfc
commit 3f5c09eb45
1 changed files with 16 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
cognee/tasks/web_scraper/__init__.py
View file

@ -6,9 +6,24 @@ BeautifulSoup or Tavily, defining data models, and handling scraping configurati
"""
from .utils import fetch_page_content
from .web_scraper_task import cron_web_scraper_task, web_scraper_task
from .default_url_crawler import DefaultUrlCrawler
# Lazy import for web_scraper_task to avoid requiring apscheduler
# Import these directly if needed: from cognee.tasks.web_scraper.web_scraper_task import ...
def __getattr__(name):
"""Lazy load web scraper task functions that require apscheduler."""
if name == "cron_web_scraper_task":
from .web_scraper_task import cron_web_scraper_task
return cron_web_scraper_task
elif name == "web_scraper_task":
from .web_scraper_task import web_scraper_task
return web_scraper_task
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
__all__ = [
"BeautifulSoupCrawler",