rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-05-24 00:20:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

respect last scrape date in DB (XXX: need to make this work if DB is empty)

Browse Source
This commit is contained in:
Christian Schwarz
2023-02-13 17:48:47 +00:00
parent f170aad625
commit 549ae82c8d
1 changed files with 13 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
scripts/layer_map_scraper/scraper.py
View File

@@ -131,6 +131,19 @@ async def timeline_task(
Task loop that is responsible for scraping one timeline
"""
last_scrape_at = await db.fetchval("select max(scrape_ts) from scrapes where tenant_id = $1 and timeline_id = $2", tenant_id, timeline_id)
logging.info(f"last scrape at: {last_scrape_at}")
next_scrape_at = datetime.datetime.now(tz=last_scrape_at.tzinfo) + datetime.timedelta(seconds=interval)
logging.info(f"next scrape at: {next_scrape_at}")
now = datetime.datetime.now(tz=last_scrape_at.tzinfo)
logging.info(f"now is: {now}")
sleep_secs = (now - next_scrape_at).total_seconds()
if sleep_secs < 0:
logging.warning(f"timeline has had scraping from before this script started running")
else:
logging.info(f"sleeping remaining {sleep_secs} seconds since last scrape")
await asyncio.sleep(sleep_secs)
while not stop_var.is_set():
try:
logging.info(f"begin scraping timeline {tenant_id}/{timeline_id}")