mirror of
https://github.com/neondatabase/neon.git
synced 2026-06-02 13:00:37 +00:00
new schema, remove old views, add new example queries, parse pageserver startup timestamp
This commit is contained in:
Christian Schwarz
2707
poetry.lock
generated
2707
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,8 @@ types-psutil = "^5.9.5.4"
|
||||
types-toml = "^0.10.8"
|
||||
pytest-httpserver = "^1.0.6"
|
||||
aiohttp = "3.7.4"
|
||||
python-dateutil = "^2.8.2"
|
||||
types-python-dateutil = "^2.8.19.6"
|
||||
|
||||
[tool.poetry.dev-dependencies]
|
||||
flake8 = "^5.0.4"
|
||||
|
||||
@@ -1,55 +1,51 @@
|
||||
|
||||
CREATE TABLE layer_map (
|
||||
CREATE TABLE scrapes (
|
||||
scrape_ts timestamp with time zone,
|
||||
pageserver_id text,
|
||||
launch_id text,
|
||||
pageserver_launch_timestamp timestamp with time zone,
|
||||
tenant_id text,
|
||||
timeline_id text,
|
||||
layer_map jsonb
|
||||
layer_map_dump jsonb
|
||||
);
|
||||
|
||||
CREATE VIEW last_access_ts AS
|
||||
SELECT layer_map.scrape_ts,
|
||||
layer_map.tenant_id,
|
||||
layer_map.timeline_id,
|
||||
layer_info.remote,
|
||||
layer_info.layer_file_name,
|
||||
layer_info.layer_file_size,
|
||||
to_timestamp(((max(most_recent_rec.when_millis_since_epoch) / (1000)::numeric))::double precision) AS last_access
|
||||
FROM layer_map,
|
||||
LATERAL jsonb_to_recordset(layer_map.layer_map['historic_layers'::text]) layer_info(kind text, remote boolean, layer_file_name text, layer_file_size numeric, access_stats jsonb),
|
||||
LATERAL jsonb_to_record(layer_info.access_stats) access_stats_rec(most_recent jsonb),
|
||||
LATERAL jsonb_to_recordset(
|
||||
CASE
|
||||
WHEN (jsonb_array_length(access_stats_rec.most_recent) > 0) THEN access_stats_rec.most_recent
|
||||
ELSE '[{"when_millis_since_epoch": 0}]'::jsonb
|
||||
END) most_recent_rec(when_millis_since_epoch numeric)
|
||||
GROUP BY layer_map.scrape_ts, layer_map.tenant_id, layer_map.timeline_id, layer_info.layer_file_name, layer_info.remote, layer_info.layer_file_size;
|
||||
--- what follows are example queries ---
|
||||
|
||||
--- how many layer accesses did we have per layers/timeline/tenant in the last 30 seconds
|
||||
with flattened_to_access_count as (
|
||||
select *
|
||||
from scrapes as scrapes
|
||||
cross join jsonb_to_recordset(scrapes.layer_map_dump -> 'historic_layers') historic_layer(layer_file_name text, access_stats jsonb)
|
||||
cross join jsonb_to_record(historic_layer.access_stats) access_stats(access_count_by_access_kind jsonb)
|
||||
cross join LATERAL (select key as access_kind, value::numeric as access_count from jsonb_each(access_count_by_access_kind)) access_count
|
||||
)
|
||||
select tenant_id, timeline_id, layer_file_name, access_kind, SUM(access_count) access_count_sum
|
||||
from flattened_to_access_count
|
||||
where scrape_ts > (clock_timestamp() - '30 second'::interval)
|
||||
group by rollup(tenant_id, timeline_id, layer_file_name, access_kind)
|
||||
having SUM(access_count) > 0
|
||||
order by access_count_sum desc, tenant_id desc, timeline_id desc, layer_file_name, access_kind;
|
||||
|
||||
CREATE VIEW last_access AS
|
||||
SELECT last_access_ts.tenant_id,
|
||||
last_access_ts.timeline_id,
|
||||
last_access_ts.layer_file_name,
|
||||
max(last_access_ts.last_access) AS last_access,
|
||||
max(last_access_ts.layer_file_size) AS layer_file_size
|
||||
FROM last_access_ts
|
||||
GROUP BY last_access_ts.tenant_id, last_access_ts.timeline_id, last_access_ts.layer_file_name;
|
||||
|
||||
|
||||
CREATE VIEW layer_files AS
|
||||
SELECT DISTINCT last_access_ts.tenant_id,
|
||||
last_access_ts.timeline_id,
|
||||
last_access_ts.layer_file_name
|
||||
FROM last_access_ts;
|
||||
|
||||
|
||||
|
||||
CREATE VIEW most_recent_scrape AS
|
||||
SELECT last_access_ts.tenant_id,
|
||||
last_access_ts.timeline_id,
|
||||
last_access_ts.layer_file_name,
|
||||
max(last_access_ts.scrape_ts) AS most_recent_scrape_ts
|
||||
FROM last_access_ts
|
||||
GROUP BY last_access_ts.tenant_id, last_access_ts.timeline_id, last_access_ts.layer_file_name;
|
||||
|
||||
--- residence change events in the last 30 minutes
|
||||
-- (precise, unless more residence changes happen between scrapes than layer access stats buffer)
|
||||
with flattened_to_residence_changes as (select *
|
||||
from scrapes as scrapes
|
||||
cross join jsonb_to_recordset(scrapes.layer_map_dump -> 'historic_layers') historic_layer(layer_file_name text, access_stats jsonb)
|
||||
cross join jsonb_to_record(historic_layer.access_stats) access_stats(residence_events_history jsonb)
|
||||
cross join jsonb_to_record(access_stats.residence_events_history) residence_events_history(buffer jsonb, drop_count numeric)
|
||||
cross join jsonb_to_recordset(residence_events_history.buffer) residence_events_buffer(status text, reason text, timestamp_millis_since_epoch numeric)
|
||||
)
|
||||
, renamed as (
|
||||
select
|
||||
scrape_ts,
|
||||
pageserver_launch_timestamp,
|
||||
layer_file_name,
|
||||
tenant_id,
|
||||
timeline_id,
|
||||
to_timestamp(timestamp_millis_since_epoch/1000) as residence_change_ts,
|
||||
status,
|
||||
reason
|
||||
from flattened_to_residence_changes
|
||||
)
|
||||
select distinct residence_change_ts, status, reason, tenant_id, timeline_id, layer_file_name
|
||||
from renamed
|
||||
where residence_change_ts > (clock_timestamp() - '30 min'::interval)
|
||||
order by residence_change_ts desc, layer_file_name;
|
||||
|
||||
16
scraper.py
16
scraper.py
@@ -14,6 +14,7 @@ import datetime
|
||||
|
||||
import aiohttp
|
||||
import asyncpg
|
||||
import dateutil.parser
|
||||
|
||||
|
||||
class ClientException(Exception):
|
||||
@@ -61,21 +62,24 @@ class Client:
|
||||
raise ClientException("expecting list")
|
||||
return [t["timeline_id"] for t in body]
|
||||
|
||||
async def get_layer_map(self, tenant_id, timeline_id, reset) -> Tuple[Optional[str], Any]:
|
||||
async def get_layer_map(
|
||||
self, tenant_id, timeline_id, reset
|
||||
) -> Tuple[Optional[datetime.datetime], Any]:
|
||||
resp = await self.sess.get(
|
||||
f"{self.endpoint}/v1/tenant/{tenant_id}/timeline/{timeline_id}/layer",
|
||||
params={"reset": reset},
|
||||
)
|
||||
if not resp.ok:
|
||||
raise ClientException(f"{resp}")
|
||||
launch_ts = resp.headers.get("PAGESERVER_LAUNCH_TIMESTAMP", None)
|
||||
launch_ts_str = resp.headers["PAGESERVER_LAUNCH_TIMESTAMP"]
|
||||
launch_ts = dateutil.parser.parse(launch_ts_str)
|
||||
body = await resp.json()
|
||||
return (launch_ts, body)
|
||||
|
||||
|
||||
async def scrape_timeline(ps_id: str, ps_client: Client, db: asyncpg.Pool, tenant_id, timeline_id):
|
||||
now = datetime.datetime.utcnow()
|
||||
launch_ts, layer_map = await ps_client.get_layer_map(
|
||||
now = datetime.datetime.now()
|
||||
launch_ts, layer_map_dump = await ps_client.get_layer_map(
|
||||
tenant_id,
|
||||
timeline_id,
|
||||
# Reset the stats on every access to get max resolution on the task kind bitmap.
|
||||
@@ -87,14 +91,14 @@ async def scrape_timeline(ps_id: str, ps_client: Client, db: asyncpg.Pool, tenan
|
||||
)
|
||||
await db.execute(
|
||||
"""
|
||||
insert into layer_map (scrape_ts, pageserver_id, launch_id, tenant_id, timeline_id, layer_map)
|
||||
insert into scrapes (scrape_ts, pageserver_id, pageserver_launch_timestamp, tenant_id, timeline_id, layer_map_dump)
|
||||
values ($1, $2, $3, $4, $5, $6::jsonb);""",
|
||||
now,
|
||||
ps_id,
|
||||
launch_ts,
|
||||
tenant_id,
|
||||
timeline_id,
|
||||
json.dumps(layer_map),
|
||||
json.dumps(layer_map_dump),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user