From 549ae82c8de8f38922ae8052c7bce6986c83d4c1 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Mon, 13 Feb 2023 17:48:47 +0000 Subject: [PATCH] respect last scrape date in DB (XXX: need to make this work if DB is empty) --- scripts/layer_map_scraper/scraper.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/scripts/layer_map_scraper/scraper.py b/scripts/layer_map_scraper/scraper.py index 4ad82b8185..d9772573b7 100644 --- a/scripts/layer_map_scraper/scraper.py +++ b/scripts/layer_map_scraper/scraper.py @@ -131,6 +131,19 @@ async def timeline_task( Task loop that is responsible for scraping one timeline """ + last_scrape_at = await db.fetchval("select max(scrape_ts) from scrapes where tenant_id = $1 and timeline_id = $2", tenant_id, timeline_id) + logging.info(f"last scrape at: {last_scrape_at}") + next_scrape_at = datetime.datetime.now(tz=last_scrape_at.tzinfo) + datetime.timedelta(seconds=interval) + logging.info(f"next scrape at: {next_scrape_at}") + now = datetime.datetime.now(tz=last_scrape_at.tzinfo) + logging.info(f"now is: {now}") + sleep_secs = (now - next_scrape_at).total_seconds() + if sleep_secs < 0: + logging.warning(f"timeline has had scraping from before this script started running") + else: + logging.info(f"sleeping remaining {sleep_secs} seconds since last scrape") + await asyncio.sleep(sleep_secs) + while not stop_var.is_set(): try: logging.info(f"begin scraping timeline {tenant_id}/{timeline_id}")