gmakstutis/cognee
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

solved nitpick comments

Browse source
This commit is contained in:
Geoff-Robin 2025-10-08 16:10:35 +05:30
parent a3fbbdf8eb
commit 599ef4a141
1 changed files with 11 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
cognee/tasks/web_scraper/web_scraper_task.py
View file

@ -25,14 +25,22 @@ from .utils import fetch_page_content
try: try:
from apscheduler.triggers.cron import CronTrigger from apscheduler.triggers.cron import CronTrigger
from apscheduler.schedulers.asyncio import AsyncIOScheduler from apscheduler.schedulers.asyncio import AsyncIOScheduler
scheduler = AsyncIOScheduler()
except ImportError: except ImportError:
raise ImportError("Please install apscheduler by pip install APScheduler>=3.10") raise ImportError("Please install apscheduler by pip install APScheduler>=3.10")
logger = get_logger(__name__) logger = get_logger(__name__)
_scheduler = None
def get_scheduler():
global _scheduler
if _scheduler is None:
_scheduler = AsyncIOScheduler()
return _scheduler
async def cron_web_scraper_task( async def cron_web_scraper_task(
url: Union[str, List[str]], url: Union[str, List[str]],
*, *,
@ -73,6 +81,7 @@ async def cron_web_scraper_task(
except ValueError as e: except ValueError as e:
raise ValueError(f"Invalid cron string '{schedule}': {e}") raise ValueError(f"Invalid cron string '{schedule}': {e}")
scheduler = get_scheduler()
scheduler.add_job( scheduler.add_job(
web_scraper_task, web_scraper_task,
kwargs={ kwargs={