rust/neon
1
0
Fork 0
mirror of https://github.com/neondatabase/neon.git synced 2026-01-17 10:22:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

handle situation correctly if no previous scrape exists for a given timeline

Browse Source
This commit is contained in:
Christian Schwarz
2023-02-13 17:53:57 +00:00
parent 66220f7a6d
commit cc1f4a691b
1 changed files with 12 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
scripts/layer_map_scraper/scraper.py
View File

@@ -132,17 +132,18 @@ async def timeline_task(
"""
last_scrape_at = await db.fetchval("select max(scrape_ts) from scrapes where tenant_id = $1 and timeline_id = $2", tenant_id, timeline_id)
logging.info(f"last scrape at: {last_scrape_at}")
next_scrape_at = datetime.datetime.now(tz=last_scrape_at.tzinfo) + datetime.timedelta(seconds=interval)
logging.info(f"next scrape at: {next_scrape_at}")
now = datetime.datetime.now(tz=last_scrape_at.tzinfo)
logging.info(f"now is: {now}")
sleep_secs = (now - next_scrape_at).total_seconds()
if sleep_secs < 0:
logging.warning(f"timeline has had scraping from before this script started running")
else:
logging.info(f"sleeping remaining {sleep_secs} seconds since last scrape")
await asyncio.sleep(sleep_secs)
if last_scrape_at is not None:
logging.info(f"last scrape at: {last_scrape_at}")
next_scrape_at = datetime.datetime.now(tz=last_scrape_at.tzinfo) + datetime.timedelta(seconds=interval)
logging.info(f"next scrape at: {next_scrape_at}")
now = datetime.datetime.now(tz=last_scrape_at.tzinfo)
logging.info(f"now is: {now}")
sleep_secs = (now - next_scrape_at).total_seconds()
if sleep_secs < 0:
logging.warning(f"timeline has had scraping from before this script started running")
else:
logging.info(f"sleeping remaining {sleep_secs} seconds since last scrape")
await asyncio.sleep(sleep_secs)
while not stop_var.is_set():
try: