lazy load cron_web_scraper_task and web_scraper_task
This commit is contained in:
parent
f02aa1abfc
commit
3f5c09eb45
1 changed files with 16 additions and 1 deletions
|
|
@ -6,9 +6,24 @@ BeautifulSoup or Tavily, defining data models, and handling scraping configurati
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from .utils import fetch_page_content
|
from .utils import fetch_page_content
|
||||||
from .web_scraper_task import cron_web_scraper_task, web_scraper_task
|
|
||||||
from .default_url_crawler import DefaultUrlCrawler
|
from .default_url_crawler import DefaultUrlCrawler
|
||||||
|
|
||||||
|
# Lazy import for web_scraper_task to avoid requiring apscheduler
|
||||||
|
# Import these directly if needed: from cognee.tasks.web_scraper.web_scraper_task import ...
|
||||||
|
|
||||||
|
|
||||||
|
def __getattr__(name):
|
||||||
|
"""Lazy load web scraper task functions that require apscheduler."""
|
||||||
|
if name == "cron_web_scraper_task":
|
||||||
|
from .web_scraper_task import cron_web_scraper_task
|
||||||
|
|
||||||
|
return cron_web_scraper_task
|
||||||
|
elif name == "web_scraper_task":
|
||||||
|
from .web_scraper_task import web_scraper_task
|
||||||
|
|
||||||
|
return web_scraper_task
|
||||||
|
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||||
|
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"BeautifulSoupCrawler",
|
"BeautifulSoupCrawler",
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue