From 1360361f602f4e253257033e21ccdbfd8bcc542e Mon Sep 17 00:00:00 2001 From: sharnoff Date: Tue, 28 Feb 2023 21:11:00 -0800 Subject: [PATCH 01/14] Fix missing VM cgconfig.conf (#3718) It was being added to the wrong stage in the dockerfile. This should fix it, and resolves an ongoing issue on staging. --- Dockerfile.vm-compute-node | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.vm-compute-node b/Dockerfile.vm-compute-node index af3bfb3590..9929c9b675 100644 --- a/Dockerfile.vm-compute-node +++ b/Dockerfile.vm-compute-node @@ -10,7 +10,6 @@ RUN set -e \ && rm -f /etc/inittab \ && touch /etc/inittab -ADD vm-cgconfig.conf /etc/cgconfig.conf RUN set -e \ && echo "::sysinit:cgconfigparser -l /etc/cgconfig.conf -s 1664" >> /etc/inittab \ && echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart --cgroup=neon-postgres'" >> /etc/inittab @@ -26,6 +25,7 @@ RUN apt update && \ RUN adduser vm-informant --disabled-password --no-create-home USER postgres +ADD vm-cgconfig.conf /etc/cgconfig.conf COPY --from=informant /etc/inittab /etc/inittab COPY --from=informant /usr/bin/vm-informant /usr/local/bin/vm-informant From d19c5248c960baffcd5ae096e8f18bbd5dfd45e0 Mon Sep 17 00:00:00 2001 From: Shany Pozin Date: Wed, 1 Mar 2023 18:09:08 +0200 Subject: [PATCH 02/14] Add UUID header to mgmt API (#3708) ## Describe your changes ## Issue ticket number and link #3479 ## Checklist before requesting a review - [x] I have performed a self-review of my code. - [ ] If it is a core feature, I have added thorough tests. - [ ] Do we need to implement analytics? if so did you add the relevant metrics to the dashboard? - [ ] If this PR requires public announcement, mark it with /release-notes label and add several sentences in this section. --- Cargo.lock | 3 +- libs/utils/Cargo.toml | 3 +- libs/utils/src/http/endpoint.rs | 116 ++++++++++++++++++++++++++++++-- workspace_hack/Cargo.toml | 1 - 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6b23144182..02b03e02fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4505,6 +4505,7 @@ dependencies = [ "byteorder", "bytes", "criterion", + "futures", "git-version", "heapless", "hex", @@ -4534,6 +4535,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "uuid", "workspace_hack", ] @@ -4840,7 +4842,6 @@ dependencies = [ "either", "fail", "futures", - "futures-channel", "futures-executor", "futures-util", "hashbrown 0.12.3", diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 92e805ac58..6acdb6fa53 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -13,6 +13,7 @@ bincode.workspace = true bytes.workspace = true heapless.workspace = true hyper = { workspace = true, features = ["full"] } +futures = { workspace = true} routerify.workspace = true serde.workspace = true serde_json.workspace = true @@ -39,7 +40,7 @@ pq_proto.workspace = true workspace_hack.workspace = true url.workspace = true - +uuid = { version = "1.2", features = ["v4", "serde"] } [dev-dependencies] byteorder.workspace = true bytes.workspace = true diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index 9c300de7a7..41975f6944 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -8,8 +8,7 @@ use hyper::{Method, StatusCode}; use metrics::{register_int_counter, Encoder, IntCounter, TextEncoder}; use once_cell::sync::Lazy; use routerify::ext::RequestExt; -use routerify::RequestInfo; -use routerify::{Middleware, Router, RouterBuilder, RouterService}; +use routerify::{Middleware, RequestInfo, Router, RouterBuilder, RouterService}; use tokio::task::JoinError; use tracing; @@ -27,14 +26,35 @@ static SERVE_METRICS_COUNT: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +static X_REQUEST_ID_HEADER_STR: &str = "x-request-id"; + +static X_REQUEST_ID_HEADER: HeaderName = HeaderName::from_static(X_REQUEST_ID_HEADER_STR); +#[derive(Debug, Default, Clone)] +struct RequestId(String); + async fn logger(res: Response, info: RequestInfo) -> Result, ApiError> { + let request_id = info.context::().unwrap_or_default().0; + // cannot factor out the Level to avoid the repetition // because tracing can only work with const Level // which is not the case here + if info.method() == Method::GET && res.status() == StatusCode::OK { - tracing::debug!("{} {} {}", info.method(), info.uri().path(), res.status()); + tracing::debug!( + "{} {} {} {}", + info.method(), + info.uri().path(), + request_id, + res.status() + ); } else { - tracing::info!("{} {} {}", info.method(), info.uri().path(), res.status()); + tracing::info!( + "{} {} {} {}", + info.method(), + info.uri().path(), + request_id, + res.status() + ); } Ok(res) } @@ -63,9 +83,52 @@ async fn prometheus_metrics_handler(_req: Request) -> Result( +) -> Middleware { + Middleware::pre(move |req| async move { + let request_id = match req.headers().get(&X_REQUEST_ID_HEADER) { + Some(request_id) => request_id + .to_str() + .expect("extract request id value") + .to_owned(), + None => { + let request_id = uuid::Uuid::new_v4(); + request_id.to_string() + } + }; + + if req.method() == Method::GET { + tracing::debug!("{} {} {}", req.method(), req.uri().path(), request_id); + } else { + tracing::info!("{} {} {}", req.method(), req.uri().path(), request_id); + } + req.set_context(RequestId(request_id)); + + Ok(req) + }) +} + +async fn add_request_id_header_to_response( + mut res: Response, + req_info: RequestInfo, +) -> Result, ApiError> { + if let Some(request_id) = req_info.context::() { + if let Ok(request_header_value) = HeaderValue::from_str(&request_id.0) { + res.headers_mut() + .insert(&X_REQUEST_ID_HEADER, request_header_value); + }; + }; + + Ok(res) +} + pub fn make_router() -> RouterBuilder { Router::builder() + .middleware(add_request_id_middleware()) .middleware(Middleware::post_with_info(logger)) + .middleware(Middleware::post_with_info( + add_request_id_header_to_response, + )) .get("/metrics", prometheus_metrics_handler) .err_handler(error::handler) } @@ -231,3 +294,48 @@ where Ok(()) } +#[cfg(test)] +mod tests { + use super::*; + use futures::future::poll_fn; + use hyper::service::Service; + use routerify::RequestServiceBuilder; + use std::net::{IpAddr, SocketAddr}; + + #[tokio::test] + async fn test_request_id_returned() { + let builder = RequestServiceBuilder::new(make_router().build().unwrap()).unwrap(); + let remote_addr = SocketAddr::new(IpAddr::from_str("127.0.0.1").unwrap(), 80); + let mut service = builder.build(remote_addr); + if let Err(e) = poll_fn(|ctx| service.poll_ready(ctx)).await { + panic!("request service is not ready: {:?}", e); + } + + let mut req: Request = Request::default(); + req.headers_mut() + .append(&X_REQUEST_ID_HEADER, HeaderValue::from_str("42").unwrap()); + + let resp: Response = service.call(req).await.unwrap(); + + let header_val = resp.headers().get(&X_REQUEST_ID_HEADER).unwrap(); + + assert!(header_val == "42", "response header mismatch"); + } + + #[tokio::test] + async fn test_request_id_empty() { + let builder = RequestServiceBuilder::new(make_router().build().unwrap()).unwrap(); + let remote_addr = SocketAddr::new(IpAddr::from_str("127.0.0.1").unwrap(), 80); + let mut service = builder.build(remote_addr); + if let Err(e) = poll_fn(|ctx| service.poll_ready(ctx)).await { + panic!("request service is not ready: {:?}", e); + } + + let req: Request = Request::default(); + let resp: Response = service.call(req).await.unwrap(); + + let header_val = resp.headers().get(&X_REQUEST_ID_HEADER); + + assert_ne!(header_val, None, "response header should NOT be empty"); + } +} diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index c0cf3c5611..bd21095fff 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -21,7 +21,6 @@ crossbeam-utils = { version = "0.8" } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures = { version = "0.3" } -futures-channel = { version = "0.3", features = ["sink"] } futures-executor = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } hashbrown = { version = "0.12", features = ["raw"] } From 8dae87999465898e3e3c39d38358b2f562282c81 Mon Sep 17 00:00:00 2001 From: sharnoff Date: Wed, 1 Mar 2023 23:58:43 -0800 Subject: [PATCH 03/14] Disable VM cgroup shenanigans (#3730) As discussed - temporary, so it can unblock releasing autoscaling. Cleaner to fully remove, then add back rather than commenting it out. --- Dockerfile.vm-compute-node | 11 ++--------- vm-cgconfig.conf | 12 ------------ 2 files changed, 2 insertions(+), 21 deletions(-) delete mode 100644 vm-cgconfig.conf diff --git a/Dockerfile.vm-compute-node b/Dockerfile.vm-compute-node index 9929c9b675..dff40485de 100644 --- a/Dockerfile.vm-compute-node +++ b/Dockerfile.vm-compute-node @@ -11,22 +11,15 @@ RUN set -e \ && touch /etc/inittab RUN set -e \ - && echo "::sysinit:cgconfigparser -l /etc/cgconfig.conf -s 1664" >> /etc/inittab \ - && echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart --cgroup=neon-postgres'" >> /etc/inittab + && echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart'" >> /etc/inittab # Combine, starting from non-VM compute node image. FROM $SRC_IMAGE as base -# Temporarily set user back to root so we can run apt update and adduser +# Temporarily set user back to root so we can run adduser USER root -RUN apt update && \ - apt install --no-install-recommends -y \ - cgroup-tools RUN adduser vm-informant --disabled-password --no-create-home USER postgres -ADD vm-cgconfig.conf /etc/cgconfig.conf COPY --from=informant /etc/inittab /etc/inittab COPY --from=informant /usr/bin/vm-informant /usr/local/bin/vm-informant - -ENTRYPOINT ["/usr/sbin/cgexec", "-g", "*:neon-postgres", "/usr/local/bin/compute_ctl"] diff --git a/vm-cgconfig.conf b/vm-cgconfig.conf deleted file mode 100644 index a2e201708e..0000000000 --- a/vm-cgconfig.conf +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration for cgroups in VM compute nodes -group neon-postgres { - perm { - admin { - uid = vm-informant; - } - task { - gid = users; - } - } - memory {} -} From a60f687ce2b9022f10943b537daf05b09a9d48fc Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Thu, 2 Mar 2023 11:35:25 +0100 Subject: [PATCH 04/14] Compile `rum` extension --- Dockerfile.compute-node | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 68278ed07b..4abc8e9cc5 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -225,6 +225,21 @@ RUN wget https://github.com/iCyberon/pg_hashids/archive/refs/tags/v1.2.1.tar.gz make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_hashids.control +######################################################################################### +# +# Layer "rum-pg-build" +# compile rum extension +# +######################################################################################### +FROM build-deps AS rum-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +RUN wget https://github.com/postgrespro/rum/archive/refs/tags/1.3.13.tar.gz -O rum.tar.gz && \ + mkdir rum-src && cd rum-src && tar xvzf ../rum.tar.gz --strip-components=1 -C . && \ + make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/rum.control + ######################################################################################### # # Layer "rust extensions" @@ -304,6 +319,7 @@ COPY --from=pg-jsonschema-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-graphql-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=hypopg-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-hashids-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=rum-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ From 5e514b8465fa4b470946945ac8887f5ff7803130 Mon Sep 17 00:00:00 2001 From: Vadim Kharitonov Date: Thu, 2 Mar 2023 14:33:29 +0100 Subject: [PATCH 05/14] Compile pgTAP extension --- Dockerfile.compute-node | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 4abc8e9cc5..1a2cd9fb77 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -240,6 +240,21 @@ RUN wget https://github.com/postgrespro/rum/archive/refs/tags/1.3.13.tar.gz -O r make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/rum.control +######################################################################################### +# +# Layer "pgtap-pg-build" +# compile pgTAP extension +# +######################################################################################### +FROM build-deps AS pgtap-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +RUN wget https://github.com/theory/pgtap/archive/refs/tags/v1.2.0.tar.gz -O pgtap.tar.gz && \ + mkdir pgtap-src && cd pgtap-src && tar xvzf ../pgtap.tar.gz --strip-components=1 -C . && \ + make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgtap.control + ######################################################################################### # # Layer "rust extensions" @@ -320,6 +335,7 @@ COPY --from=pg-graphql-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=hypopg-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-hashids-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=rum-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=pgtap-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ From b23742e09c24b77217c9556699fcdd909332554e Mon Sep 17 00:00:00 2001 From: Arthur Petukhovsky Date: Fri, 3 Mar 2023 14:01:05 +0300 Subject: [PATCH 06/14] Create `/v1/debug_dump` safekeepers endpoint (#3710) Add HTTP endpoint to get full safekeeper state of all existing timelines (all in-memory values and info about all files stored on disk). Example: https://gist.github.com/petuhovskiy/3cbb8f870401e9f486731d145161c286 --- Cargo.lock | 1 + libs/utils/src/http/json.rs | 13 +- safekeeper/Cargo.toml | 1 + safekeeper/src/debug_dump.rs | 264 +++++++++++++++++++++++ safekeeper/src/http/routes.rs | 79 ++++++- safekeeper/src/lib.rs | 1 + safekeeper/src/safekeeper.rs | 3 +- safekeeper/src/timeline.rs | 95 +++++--- safekeeper/src/timelines_global_map.rs | 10 + safekeeper/src/wal_storage.rs | 10 + test_runner/fixtures/neon_fixtures.py | 7 + test_runner/regress/test_wal_acceptor.py | 32 +++ 12 files changed, 468 insertions(+), 48 deletions(-) create mode 100644 safekeeper/src/debug_dump.rs diff --git a/Cargo.lock b/Cargo.lock index 02b03e02fb..fe5aae6ae8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3307,6 +3307,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", + "chrono", "clap 4.1.4", "const_format", "crc32c", diff --git a/libs/utils/src/http/json.rs b/libs/utils/src/http/json.rs index 8981fdd1dd..40e61e3d0c 100644 --- a/libs/utils/src/http/json.rs +++ b/libs/utils/src/http/json.rs @@ -1,7 +1,9 @@ +use std::fmt::Display; + use anyhow::Context; use bytes::Buf; use hyper::{header, Body, Request, Response, StatusCode}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Serializer}; use super::error::ApiError; @@ -31,3 +33,12 @@ pub fn json_response( .map_err(|e| ApiError::InternalServerError(e.into()))?; Ok(response) } + +/// Serialize through Display trait. +pub fn display_serialize(z: &F, s: S) -> Result +where + S: Serializer, + F: Display, +{ + s.serialize_str(&format!("{}", z)) +} diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index 4ee8d82203..2424509477 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -10,6 +10,7 @@ anyhow.workspace = true async-trait.workspace = true byteorder.workspace = true bytes.workspace = true +chrono.workspace = true clap = { workspace = true, features = ["derive"] } const_format.workspace = true crc32c.workspace = true diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs new file mode 100644 index 0000000000..674cf9f6eb --- /dev/null +++ b/safekeeper/src/debug_dump.rs @@ -0,0 +1,264 @@ +//! Utils for dumping full state of the safekeeper. + +use std::fs; +use std::fs::DirEntry; +use std::io::BufReader; +use std::io::Read; +use std::path::PathBuf; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use postgres_ffi::XLogSegNo; +use serde::Serialize; + +use utils::http::json::display_serialize; +use utils::id::NodeId; +use utils::id::TenantTimelineId; +use utils::id::{TenantId, TimelineId}; +use utils::lsn::Lsn; + +use crate::safekeeper::SafeKeeperState; +use crate::safekeeper::SafekeeperMemState; +use crate::safekeeper::TermHistory; +use crate::SafeKeeperConf; + +use crate::timeline::ReplicaState; +use crate::GlobalTimelines; + +/// Various filters that influence the resulting JSON output. +#[derive(Debug, Serialize)] +pub struct Args { + /// Dump all available safekeeper state. False by default. + pub dump_all: bool, + + /// Dump control_file content. Uses value of `dump_all` by default. + pub dump_control_file: bool, + + /// Dump in-memory state. Uses value of `dump_all` by default. + pub dump_memory: bool, + + /// Dump all disk files in a timeline directory. Uses value of `dump_all` by default. + pub dump_disk_content: bool, + + /// Dump full term history. True by default. + pub dump_term_history: bool, + + /// Filter timelines by tenant_id. + pub tenant_id: Option, + + /// Filter timelines by timeline_id. + pub timeline_id: Option, +} + +/// Response for debug dump request. +#[derive(Debug, Serialize)] +pub struct Response { + pub start_time: DateTime, + pub finish_time: DateTime, + pub timelines: Vec, + pub timelines_count: usize, + pub config: Config, +} + +/// Safekeeper configuration. +#[derive(Debug, Serialize)] +pub struct Config { + pub id: NodeId, + pub workdir: PathBuf, + pub listen_pg_addr: String, + pub listen_http_addr: String, + pub no_sync: bool, + pub max_offloader_lag_bytes: u64, + pub wal_backup_enabled: bool, +} + +#[derive(Debug, Serialize)] +pub struct Timeline { + #[serde(serialize_with = "display_serialize")] + pub tenant_id: TenantId, + #[serde(serialize_with = "display_serialize")] + pub timeline_id: TimelineId, + pub control_file: Option, + pub memory: Option, + pub disk_content: Option, +} + +#[derive(Debug, Serialize)] +pub struct Memory { + pub is_cancelled: bool, + pub peers_info_len: usize, + pub replicas: Vec>, + pub wal_backup_active: bool, + pub active: bool, + pub num_computes: u32, + pub last_removed_segno: XLogSegNo, + pub epoch_start_lsn: Lsn, + pub mem_state: SafekeeperMemState, + + // PhysicalStorage state. + pub write_lsn: Lsn, + pub write_record_lsn: Lsn, + pub flush_lsn: Lsn, + pub file_open: bool, +} + +#[derive(Debug, Serialize)] +pub struct DiskContent { + pub files: Vec, +} + +#[derive(Debug, Serialize)] +pub struct FileInfo { + pub name: String, + pub size: u64, + pub created: DateTime, + pub modified: DateTime, + pub start_zeroes: u64, + pub end_zeroes: u64, + // TODO: add sha256 checksum +} + +/// Build debug dump response, using the provided [`Args`] filters. +pub fn build(args: Args) -> Result { + let start_time = Utc::now(); + let timelines_count = GlobalTimelines::timelines_count(); + + let ptrs_snapshot = if args.tenant_id.is_some() && args.timeline_id.is_some() { + // If both tenant_id and timeline_id are specified, we can just get the + // timeline directly, without taking a snapshot of the whole list. + let ttid = TenantTimelineId::new(args.tenant_id.unwrap(), args.timeline_id.unwrap()); + if let Ok(tli) = GlobalTimelines::get(ttid) { + vec![tli] + } else { + vec![] + } + } else { + // Otherwise, take a snapshot of the whole list. + GlobalTimelines::get_all() + }; + + // TODO: return Stream instead of Vec + let mut timelines = Vec::new(); + for tli in ptrs_snapshot { + let ttid = tli.ttid; + if let Some(tenant_id) = args.tenant_id { + if tenant_id != ttid.tenant_id { + continue; + } + } + if let Some(timeline_id) = args.timeline_id { + if timeline_id != ttid.timeline_id { + continue; + } + } + + let control_file = if args.dump_control_file { + let mut state = tli.get_state().1; + if !args.dump_term_history { + state.acceptor_state.term_history = TermHistory(vec![]); + } + Some(state) + } else { + None + }; + + let memory = if args.dump_memory { + Some(tli.memory_dump()) + } else { + None + }; + + let disk_content = if args.dump_disk_content { + // build_disk_content can fail, but we don't want to fail the whole + // request because of that. + build_disk_content(&tli.timeline_dir).ok() + } else { + None + }; + + let timeline = Timeline { + tenant_id: ttid.tenant_id, + timeline_id: ttid.timeline_id, + control_file, + memory, + disk_content, + }; + timelines.push(timeline); + } + + let config = GlobalTimelines::get_global_config(); + + Ok(Response { + start_time, + finish_time: Utc::now(), + timelines, + timelines_count, + config: build_config(config), + }) +} + +/// Builds DiskContent from a directory path. It can fail if the directory +/// is deleted between the time we get the path and the time we try to open it. +fn build_disk_content(path: &std::path::Path) -> Result { + let mut files = Vec::new(); + for entry in fs::read_dir(path)? { + if entry.is_err() { + continue; + } + let file = build_file_info(entry?); + if file.is_err() { + continue; + } + files.push(file?); + } + + Ok(DiskContent { files }) +} + +/// Builds FileInfo from DirEntry. Sometimes it can return an error +/// if the file is deleted between the time we get the DirEntry +/// and the time we try to open it. +fn build_file_info(entry: DirEntry) -> Result { + let metadata = entry.metadata()?; + let path = entry.path(); + let name = path + .file_name() + .and_then(|x| x.to_str()) + .unwrap_or("") + .to_owned(); + let mut file = fs::File::open(path)?; + let mut reader = BufReader::new(&mut file).bytes().filter_map(|x| x.ok()); + + let start_zeroes = reader.by_ref().take_while(|&x| x == 0).count() as u64; + let mut end_zeroes = 0; + for b in reader { + if b == 0 { + end_zeroes += 1; + } else { + end_zeroes = 0; + } + } + + Ok(FileInfo { + name, + size: metadata.len(), + created: DateTime::from(metadata.created()?), + modified: DateTime::from(metadata.modified()?), + start_zeroes, + end_zeroes, + }) +} + +/// Converts SafeKeeperConf to Config, filtering out the fields that are not +/// supposed to be exposed. +fn build_config(config: SafeKeeperConf) -> Config { + Config { + id: config.my_id, + workdir: config.workdir, + listen_pg_addr: config.listen_pg_addr, + listen_http_addr: config.listen_http_addr, + no_sync: config.no_sync, + max_offloader_lag_bytes: config.max_offloader_lag_bytes, + wal_backup_enabled: config.wal_backup_enabled, + } +} diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index a917d61678..ced9599b36 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -5,14 +5,16 @@ use once_cell::sync::Lazy; use postgres_ffi::WAL_SEGMENT_SIZE; use safekeeper_api::models::SkTimelineInfo; use serde::Serialize; -use serde::Serializer; use std::collections::{HashMap, HashSet}; -use std::fmt::Display; +use std::fmt; +use std::str::FromStr; use std::sync::Arc; use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use tokio::task::JoinError; +use utils::http::json::display_serialize; +use crate::debug_dump; use crate::safekeeper::ServerInfo; use crate::safekeeper::Term; @@ -54,15 +56,6 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { .as_ref() } -/// Serialize through Display trait. -fn display_serialize(z: &F, s: S) -> Result -where - S: Serializer, - F: Display, -{ - s.serialize_str(&format!("{}", z)) -} - /// Same as TermSwitchEntry, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. #[derive(Debug, Serialize)] @@ -276,6 +269,69 @@ async fn record_safekeeper_info(mut request: Request) -> Result>(k: &str, v: &str) -> Result { + v.parse() + .map_err(|e| ApiError::BadRequest(anyhow::anyhow!("cannot parse {k}: {e}"))) +} + +/// Dump debug info about all available safekeeper state. +async fn dump_debug_handler(mut request: Request) -> Result, ApiError> { + check_permission(&request, None)?; + ensure_no_body(&mut request).await?; + + let mut dump_all: Option = None; + let mut dump_control_file: Option = None; + let mut dump_memory: Option = None; + let mut dump_disk_content: Option = None; + let mut dump_term_history: Option = None; + let mut tenant_id: Option = None; + let mut timeline_id: Option = None; + + let query = request.uri().query().unwrap_or(""); + let mut values = url::form_urlencoded::parse(query.as_bytes()); + + for (k, v) in &mut values { + match k.as_ref() { + "dump_all" => dump_all = Some(parse_kv_str(&k, &v)?), + "dump_control_file" => dump_control_file = Some(parse_kv_str(&k, &v)?), + "dump_memory" => dump_memory = Some(parse_kv_str(&k, &v)?), + "dump_disk_content" => dump_disk_content = Some(parse_kv_str(&k, &v)?), + "dump_term_history" => dump_term_history = Some(parse_kv_str(&k, &v)?), + "tenant_id" => tenant_id = Some(parse_kv_str(&k, &v)?), + "timeline_id" => timeline_id = Some(parse_kv_str(&k, &v)?), + _ => Err(ApiError::BadRequest(anyhow::anyhow!( + "Unknown query parameter: {}", + k + )))?, + } + } + + let dump_all = dump_all.unwrap_or(false); + let dump_control_file = dump_control_file.unwrap_or(dump_all); + let dump_memory = dump_memory.unwrap_or(dump_all); + let dump_disk_content = dump_disk_content.unwrap_or(dump_all); + let dump_term_history = dump_term_history.unwrap_or(true); + + let args = debug_dump::Args { + dump_all, + dump_control_file, + dump_memory, + dump_disk_content, + dump_term_history, + tenant_id, + timeline_id, + }; + + let resp = tokio::task::spawn_blocking(move || { + debug_dump::build(args).map_err(ApiError::InternalServerError) + }) + .await + .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??; + + // TODO: use streaming response + json_response(StatusCode::OK, resp) +} + /// Safekeeper http router. pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder { let mut router = endpoint::make_router(); @@ -316,6 +372,7 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder "/v1/record_safekeeper_info/:tenant_id/:timeline_id", record_safekeeper_info, ) + .get("/v1/debug_dump", dump_debug_handler) } #[cfg(test)] diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 891d73533f..6ab108ceb0 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -10,6 +10,7 @@ mod auth; pub mod broker; pub mod control_file; pub mod control_file_upgrade; +pub mod debug_dump; pub mod handler; pub mod http; pub mod json_ctrl; diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index fa973a3ede..c37411d667 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -204,7 +204,7 @@ pub struct SafeKeeperState { pub peers: PersistedPeers, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] // In memory safekeeper state. Fields mirror ones in `SafeKeeperState`; values // are not flushed yet. pub struct SafekeeperMemState { @@ -212,6 +212,7 @@ pub struct SafekeeperMemState { pub backup_lsn: Lsn, pub peer_horizon_lsn: Lsn, pub remote_consistent_lsn: Lsn, + #[serde(with = "hex")] pub proposer_uuid: PgUuid, } diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 43c395574f..7479741774 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Result}; use parking_lot::{Mutex, MutexGuard}; use postgres_ffi::XLogSegNo; use pq_proto::ReplicationFeedback; +use serde::Serialize; use std::cmp::{max, min}; use std::path::PathBuf; use tokio::{ @@ -28,9 +29,9 @@ use crate::send_wal::HotStandbyFeedback; use crate::{control_file, safekeeper::UNKNOWN_SERVER_VERSION}; use crate::metrics::FullTimelineInfo; -use crate::wal_storage; use crate::wal_storage::Storage as wal_storage_iface; use crate::SafeKeeperConf; +use crate::{debug_dump, wal_storage}; /// Things safekeeper should know about timeline state on peers. #[derive(Debug, Clone)] @@ -80,7 +81,7 @@ impl PeersInfo { } /// Replica status update + hot standby feedback -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Serialize)] pub struct ReplicaState { /// last known lsn received by replica pub last_received_lsn: Lsn, // None means we don't know @@ -381,7 +382,7 @@ pub struct Timeline { cancellation_rx: watch::Receiver, /// Directory where timeline state is stored. - timeline_dir: PathBuf, + pub timeline_dir: PathBuf, } impl Timeline { @@ -588,38 +589,6 @@ impl Timeline { self.write_shared_state().wal_backup_attend() } - /// Returns full timeline info, required for the metrics. If the timeline is - /// not active, returns None instead. - pub fn info_for_metrics(&self) -> Option { - if self.is_cancelled() { - return None; - } - - let state = self.write_shared_state(); - if state.active { - Some(FullTimelineInfo { - ttid: self.ttid, - replicas: state - .replicas - .iter() - .filter_map(|r| r.as_ref()) - .copied() - .collect(), - wal_backup_active: state.wal_backup_active, - timeline_is_active: state.active, - num_computes: state.num_computes, - last_removed_segno: state.last_removed_segno, - epoch_start_lsn: state.sk.epoch_start_lsn, - mem_state: state.sk.inmem.clone(), - persisted_state: state.sk.state.clone(), - flush_lsn: state.sk.wal_store.flush_lsn(), - wal_storage: state.sk.wal_store.get_metrics(), - }) - } else { - None - } - } - /// Returns commit_lsn watch channel. pub fn get_commit_lsn_watch_rx(&self) -> watch::Receiver { self.commit_lsn_watch_rx.clone() @@ -784,6 +753,62 @@ impl Timeline { shared_state.last_removed_segno = horizon_segno; Ok(()) } + + /// Returns full timeline info, required for the metrics. If the timeline is + /// not active, returns None instead. + pub fn info_for_metrics(&self) -> Option { + if self.is_cancelled() { + return None; + } + + let state = self.write_shared_state(); + if state.active { + Some(FullTimelineInfo { + ttid: self.ttid, + replicas: state + .replicas + .iter() + .filter_map(|r| r.as_ref()) + .copied() + .collect(), + wal_backup_active: state.wal_backup_active, + timeline_is_active: state.active, + num_computes: state.num_computes, + last_removed_segno: state.last_removed_segno, + epoch_start_lsn: state.sk.epoch_start_lsn, + mem_state: state.sk.inmem.clone(), + persisted_state: state.sk.state.clone(), + flush_lsn: state.sk.wal_store.flush_lsn(), + wal_storage: state.sk.wal_store.get_metrics(), + }) + } else { + None + } + } + + /// Returns in-memory timeline state to build a full debug dump. + pub fn memory_dump(&self) -> debug_dump::Memory { + let state = self.write_shared_state(); + + let (write_lsn, write_record_lsn, flush_lsn, file_open) = + state.sk.wal_store.internal_state(); + + debug_dump::Memory { + is_cancelled: self.is_cancelled(), + peers_info_len: state.peers_info.0.len(), + replicas: state.replicas.clone(), + wal_backup_active: state.wal_backup_active, + active: state.active, + num_computes: state.num_computes, + last_removed_segno: state.last_removed_segno, + epoch_start_lsn: state.sk.epoch_start_lsn, + mem_state: state.sk.inmem.clone(), + write_lsn, + write_record_lsn, + flush_lsn, + file_open, + } + } } /// Deletes directory and it's contents. Returns false if directory does not exist. diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs index 66e0145042..baef17ffa8 100644 --- a/safekeeper/src/timelines_global_map.rs +++ b/safekeeper/src/timelines_global_map.rs @@ -159,6 +159,16 @@ impl GlobalTimelines { Ok(()) } + /// Get the number of timelines in the map. + pub fn timelines_count() -> usize { + TIMELINES_STATE.lock().unwrap().timelines.len() + } + + /// Get the global safekeeper config. + pub fn get_global_config() -> SafeKeeperConf { + TIMELINES_STATE.lock().unwrap().get_conf().clone() + } + /// Create a new timeline with the given id. If the timeline already exists, returns /// an existing timeline. pub fn create( diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 561104bd27..ae02b3c7bc 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -165,6 +165,16 @@ impl PhysicalStorage { }) } + /// Get all known state of the storage. + pub fn internal_state(&self) -> (Lsn, Lsn, Lsn, bool) { + ( + self.write_lsn, + self.write_record_lsn, + self.flush_record_lsn, + self.file.is_some(), + ) + } + /// Call fdatasync if config requires so. fn fdatasync_file(&mut self, file: &mut File) -> Result<()> { if !self.conf.no_sync { diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index c4b3d057f8..56b56b8578 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2988,6 +2988,13 @@ class SafekeeperHttpClient(requests.Session): def check_status(self): self.get(f"http://localhost:{self.port}/v1/status").raise_for_status() + def debug_dump(self, params: Dict[str, str] = {}) -> Dict[str, Any]: + res = self.get(f"http://localhost:{self.port}/v1/debug_dump", params=params) + res.raise_for_status() + res_json = res.json() + assert isinstance(res_json, dict) + return res_json + def timeline_create( self, tenant_id: TenantId, timeline_id: TimelineId, pg_version: int, commit_lsn: Lsn ): diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 9e3b0ec02f..0ac9127c6b 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -775,6 +775,9 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): if not auth_enabled: wa_http_cli = wa.http_client() wa_http_cli.check_status() + + wa_http_cli_debug = wa.http_client() + wa_http_cli_debug.check_status() else: wa_http_cli = wa.http_client(auth_token=env.auth_keys.generate_tenant_token(tenant_id)) wa_http_cli.check_status() @@ -785,6 +788,10 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): wa_http_cli_noauth = wa.http_client() wa_http_cli_noauth.check_status() + # debug endpoint requires safekeeper scope + wa_http_cli_debug = wa.http_client(auth_token=env.auth_keys.generate_safekeeper_token()) + wa_http_cli_debug.check_status() + # fetch something sensible from status tli_status = wa_http_cli.timeline_status(tenant_id, timeline_id) epoch = tli_status.acceptor_epoch @@ -795,6 +802,12 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): with pytest.raises(cli.HTTPError, match="Forbidden|Unauthorized"): cli.timeline_status(tenant_id, timeline_id) + # fetch debug_dump endpoint + debug_dump_0 = wa_http_cli_debug.debug_dump({"dump_all": "true"}) + log.info(f"debug_dump before reboot {debug_dump_0}") + assert debug_dump_0["timelines_count"] == 1 + assert debug_dump_0["timelines"][0]["timeline_id"] == str(timeline_id) + pg.safe_psql("create table t(i int)") # ensure epoch goes up after reboot @@ -808,6 +821,25 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): # and timeline_start_lsn stays the same assert tli_status.timeline_start_lsn == timeline_start_lsn + # fetch debug_dump after reboot + debug_dump_1 = wa_http_cli_debug.debug_dump({"dump_all": "true"}) + log.info(f"debug_dump after reboot {debug_dump_1}") + assert debug_dump_1["timelines_count"] == 1 + assert debug_dump_1["timelines"][0]["timeline_id"] == str(timeline_id) + + # check that commit_lsn and flush_lsn not decreased + assert ( + debug_dump_1["timelines"][0]["memory"]["mem_state"]["commit_lsn"] + >= debug_dump_0["timelines"][0]["memory"]["mem_state"]["commit_lsn"] + ) + assert ( + debug_dump_1["timelines"][0]["memory"]["flush_lsn"] + >= debug_dump_0["timelines"][0]["memory"]["flush_lsn"] + ) + + # check .config in response + assert debug_dump_1["config"]["id"] == env.safekeepers[0].id + class SafekeeperEnv: def __init__( From 764d27f6960c4eb48335c5576f601dbb1524a59f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 1 Mar 2023 18:42:14 +0100 Subject: [PATCH 07/14] fix checkpoint_timeout serialization in TenantConf Without this change, when actually setting this conf opt, the tenant would become Broken next time we load it. Why? The serde_toml representation that persist_tenant_conf would write out would be a TOML inline table of `secs` and `nsecs`. But our hand-rolled TenantConf parser expects a TOML string. I checked that all other `Duration` values in TenantConfOpt use the humantime serialization. Issues like this would likely be systematically prevent by https://github.com/neondatabase/neon/issues/3682 --- pageserver/src/tenant/config.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index fca08dd51a..1a52b26ae7 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -103,6 +103,7 @@ pub struct TenantConfOpt { pub checkpoint_distance: Option, #[serde(skip_serializing_if = "Option::is_none")] + #[serde(with = "humantime_serde")] #[serde(default)] pub checkpoint_timeout: Option, From 68141a924da9442e4cf7a0bad2bb5153fe41d76f Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 1 Mar 2023 12:09:02 +0100 Subject: [PATCH 08/14] eviction: remove needless if-let around resident size decrement The branch was always taken at runtime, so, this should not constitute a behavioral change. refs https://github.com/neondatabase/neon/issues/3722 --- pageserver/src/tenant/timeline.rs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index d46ac26e7d..3926249572 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1047,11 +1047,12 @@ impl Timeline { return Ok(false); } - let layer_metadata = LayerFileMetadata::new( - local_layer - .file_size() - .expect("Local layer should have a file size"), - ); + let layer_file_size = local_layer + .file_size() + .expect("Local layer should have a file size"); + + let layer_metadata = LayerFileMetadata::new(layer_file_size); + let new_remote_layer = Arc::new(match local_layer.filename() { LayerFileName::Image(image_name) => RemoteLayer::new_img( self.tenant_id, @@ -1075,15 +1076,13 @@ impl Timeline { let replaced = match batch_updates.replace_historic(local_layer, new_remote_layer)? { Replacement::Replaced { .. } => { - let layer_size = local_layer.file_size(); - if let Err(e) = local_layer.delete() { error!("failed to remove layer file on evict after replacement: {e:#?}"); } - if let Some(layer_size) = layer_size { - self.metrics.resident_physical_size_gauge.sub(layer_size); - } + self.metrics + .resident_physical_size_gauge + .sub(layer_file_size); true } From 1b9b9d60d462afd893cf989ca6a6cc55039b7e15 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 1 Mar 2023 13:06:43 +0100 Subject: [PATCH 09/14] eviction: add comment explaining resident size decrement on error https://github.com/neondatabase/neon/issues/3722 --- pageserver/src/tenant/timeline.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 3926249572..ca0cd04bd0 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1079,7 +1079,16 @@ impl Timeline { if let Err(e) = local_layer.delete() { error!("failed to remove layer file on evict after replacement: {e:#?}"); } - + // Always decrement the physical size gauge, even if we failed to delete the file. + // Rationale: we already replaced the layer with a remote layer in the layer map, + // and any subsequent download_remote_layer will + // 1. overwrite the file on disk and + // 2. add the downloaded size to the resident size gauge. + // + // If there is no re-download, and we restart the pageserver, then load_layer_map + // will treat the file as a local layer again, count it towards resident size, + // and it'll be like the layer removal never happened. + // The bump in resident size is perhaps unexpected but overall a robust behavior. self.metrics .resident_physical_size_gauge .sub(layer_file_size); From 38022ff11c1307d36b83070612d705f07f5f3437 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 1 Mar 2023 17:54:59 +0100 Subject: [PATCH 10/14] gc: only decrement resident size if GC'd layer is resident Before this patch, GC would call PersistentLayer::delete() on every GC'ed layer. RemoteLayer::delete() returned Ok(()) unconditionally. GC would then proceed by decrementing the resident size metric, even though the layer is a RemoteLayer. This patch makes the following changes: - Rename PersistentLayer::delete() to delete_resident_layer_file(). That name is unambiguous. - Make RemoteLayer::delete_resident_layer_file return an Err(). We would have uncovered this bug if we had done that from the start. - Change GC / Timeline::delete_historic_layer check whether the layer is remote or not, and only call delete_resident_layer_file() if it's not remote. This brings us in line with how eviction does it. - Add a regression test. fixes https://github.com/neondatabase/neon/issues/3722 --- pageserver/src/tenant/storage_layer.rs | 2 +- .../src/tenant/storage_layer/delta_layer.rs | 2 +- .../src/tenant/storage_layer/image_layer.rs | 2 +- .../src/tenant/storage_layer/remote_layer.rs | 4 +- pageserver/src/tenant/timeline.rs | 15 +- test_runner/fixtures/neon_fixtures.py | 16 +- test_runner/regress/test_layer_eviction.py | 166 ++++++++++++++++++ 7 files changed, 195 insertions(+), 12 deletions(-) diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 9198cfd1df..52ce2cab42 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -364,7 +364,7 @@ pub trait PersistentLayer: Layer { } /// Permanently remove this layer from disk. - fn delete(&self) -> Result<()>; + fn delete_resident_layer_file(&self) -> Result<()>; fn downcast_remote_layer(self: Arc) -> Option> { None diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 4d1e08322d..37719dfce5 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -438,7 +438,7 @@ impl PersistentLayer for DeltaLayer { )) } - fn delete(&self) -> Result<()> { + fn delete_resident_layer_file(&self) -> Result<()> { // delete underlying file fs::remove_file(self.path())?; Ok(()) diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index e48abd38dd..e37e001eda 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -252,7 +252,7 @@ impl PersistentLayer for ImageLayer { unimplemented!(); } - fn delete(&self) -> Result<()> { + fn delete_resident_layer_file(&self) -> Result<()> { // delete underlying file fs::remove_file(self.path())?; Ok(()) diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 8465a99339..dbce2e7888 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -155,8 +155,8 @@ impl PersistentLayer for RemoteLayer { bail!("cannot iterate a remote layer"); } - fn delete(&self) -> Result<()> { - Ok(()) + fn delete_resident_layer_file(&self) -> Result<()> { + bail!("remote layer has no layer file"); } fn downcast_remote_layer<'a>(self: Arc) -> Option> { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index ca0cd04bd0..101b27bb97 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -1076,7 +1076,7 @@ impl Timeline { let replaced = match batch_updates.replace_historic(local_layer, new_remote_layer)? { Replacement::Replaced { .. } => { - if let Err(e) = local_layer.delete() { + if let Err(e) = local_layer.delete_resident_layer_file() { error!("failed to remove layer file on evict after replacement: {e:#?}"); } // Always decrement the physical size gauge, even if we failed to delete the file. @@ -1950,11 +1950,14 @@ impl Timeline { layer: Arc, updates: &mut BatchedUpdates<'_, dyn PersistentLayer>, ) -> anyhow::Result<()> { - let layer_size = layer.file_size(); - - layer.delete()?; - if let Some(layer_size) = layer_size { - self.metrics.resident_physical_size_gauge.sub(layer_size); + if !layer.is_remote_layer() { + layer.delete_resident_layer_file()?; + let layer_file_size = layer + .file_size() + .expect("Local layer should have a file size"); + self.metrics + .resident_physical_size_gauge + .sub(layer_file_size); } // TODO Removing from the bottom of the layer map is expensive. diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 56b56b8578..49218f3c98 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -14,6 +14,7 @@ import tempfile import textwrap import time import uuid +from collections import defaultdict from contextlib import closing, contextmanager from dataclasses import dataclass, field from enum import Flag, auto @@ -1516,6 +1517,11 @@ class PageserverHttpClient(requests.Session): assert res.status_code == 200 + def evict_all_layers(self, tenant_id: TenantId, timeline_id: TimelineId): + info = self.layer_map_info(tenant_id, timeline_id) + for layer in info.historic_layers: + self.evict_layer(tenant_id, timeline_id, layer.layer_file_name) + @dataclass class TenantConfig: @@ -1551,6 +1557,14 @@ class LayerMapInfo: return info + def kind_count(self) -> Dict[str, int]: + counts: Dict[str, int] = defaultdict(int) + for inmem_layer in self.in_memory_layers: + counts[inmem_layer.kind] += 1 + for hist_layer in self.historic_layers: + counts[hist_layer.kind] += 1 + return counts + @dataclass class InMemoryLayerInfo: @@ -1567,7 +1581,7 @@ class InMemoryLayerInfo: ) -@dataclass +@dataclass(frozen=True) class HistoricLayerInfo: kind: str layer_file_name: str diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index a03dd88c41..404bd67050 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -1,8 +1,13 @@ +import time + import pytest +from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, RemoteStorageKind, + wait_for_last_flush_lsn, wait_for_last_record_lsn, + wait_for_sk_commit_lsn_to_reach_remote_storage, wait_for_upload, ) from fixtures.types import Lsn, TenantId, TimelineId @@ -138,3 +143,164 @@ def test_basic_eviction( assert ( redownloaded_layer_map_info == initial_layer_map_info ), "Should have the same layer map after redownloading the evicted layers" + + +def test_gc_of_remote_layers(neon_env_builder: NeonEnvBuilder): + + neon_env_builder.enable_remote_storage( + remote_storage_kind=RemoteStorageKind.LOCAL_FS, + test_name="test_gc_of_remote_layers", + ) + + env = neon_env_builder.init_start() + + tenant_config = { + "pitr_interval": "1s", # set to non-zero, so GC actually does something + "gc_period": "0s", # we want to control when GC runs + "compaction_period": "0s", # we want to control when compaction runs + "checkpoint_timeout": "24h", # something we won't reach + "checkpoint_distance": f"{50 * (1024**2)}", # something we won't reach, we checkpoint manually + "compaction_threshold": "3", + # "image_creation_threshold": set at runtime + "compaction_target_size": f"{128 * (1024**2)}", # make it so that we only have 1 partition => image coverage for delta layers => enables gc of delta layers + } + + def tenant_update_config(changes): + tenant_config.update(changes) + env.neon_cli.config_tenant(tenant_id, tenant_config) + + tenant_id, timeline_id = env.neon_cli.create_tenant(conf=tenant_config) + log.info("tenant id is %s", tenant_id) + env.initial_tenant = tenant_id # update_and_gc relies on this + ps_http = env.pageserver.http_client() + + pg = env.postgres.create_start("main") + + log.info("fill with data, creating delta & image layers, some of which are GC'able after") + # no particular reason to create the layers like this, but we are sure + # not to hit the image_creation_threshold here. + with pg.cursor() as cur: + cur.execute("create table a (id bigserial primary key, some_value bigint not null)") + cur.execute("insert into a(some_value) select i from generate_series(1, 10000) s(i)") + wait_for_last_flush_lsn(env, pg, tenant_id, timeline_id) + ps_http.timeline_checkpoint(tenant_id, timeline_id) + + # Create delta layers, then turn them into image layers. + # Do it multiple times so that there's something to GC. + for k in range(0, 2): + # produce delta layers => disable image layer creation by setting high threshold + tenant_update_config({"image_creation_threshold": "100"}) + for i in range(0, 2): + for j in range(0, 3): + # create a minimal amount of "delta difficulty" for this table + with pg.cursor() as cur: + cur.execute("update a set some_value = -some_value + %s", (j,)) + + with pg.cursor() as cur: + # vacuuming should aid to reuse keys, though it's not really important + # with image_creation_threshold=1 which we will use on the last compaction + cur.execute("vacuum") + + wait_for_last_flush_lsn(env, pg, tenant_id, timeline_id) + + if i == 1 and j == 2 and k == 1: + # last iteration; stop before checkpoint to avoid leaving an inmemory layer + pg.stop_and_destroy() + + ps_http.timeline_checkpoint(tenant_id, timeline_id) + + # images should not yet be created, because threshold is too high, + # but these will be reshuffled to L1 layers + ps_http.timeline_compact(tenant_id, timeline_id) + + for _ in range(0, 20): + # loop in case flushing is still in progress + layers = ps_http.layer_map_info(tenant_id, timeline_id) + if not layers.in_memory_layers: + break + time.sleep(0.2) + + # now that we've grown some delta layers, turn them into image layers + tenant_update_config({"image_creation_threshold": "1"}) + ps_http.timeline_compact(tenant_id, timeline_id) + + # wait for all uploads to finish + wait_for_sk_commit_lsn_to_reach_remote_storage( + tenant_id, timeline_id, env.safekeepers, env.pageserver + ) + + # shutdown safekeepers to avoid on-demand downloads from walreceiver + for sk in env.safekeepers: + sk.stop() + + ps_http.timeline_checkpoint(tenant_id, timeline_id) + + log.info("ensure the code above produced image and delta layers") + pre_evict_info = ps_http.layer_map_info(tenant_id, timeline_id) + log.info("layer map dump: %s", pre_evict_info) + by_kind = pre_evict_info.kind_count() + log.info("by kind: %s", by_kind) + assert by_kind["Image"] > 0 + assert by_kind["Delta"] > 0 + assert by_kind["InMemory"] == 0 + resident_layers = list(env.timeline_dir(tenant_id, timeline_id).glob("*-*_*")) + log.info("resident layers count before eviction: %s", len(resident_layers)) + + log.info("evict all layers") + ps_http.evict_all_layers(tenant_id, timeline_id) + + def ensure_resident_and_remote_size_metrics(): + log.info("ensure that all the layers are gone") + resident_layers = list(env.timeline_dir(tenant_id, timeline_id).glob("*-*_*")) + # we have disabled all background loops, so, this should hold + assert len(resident_layers) == 0 + + info = ps_http.layer_map_info(tenant_id, timeline_id) + log.info("layer map dump: %s", info) + + log.info("ensure that resident_physical_size metric is zero") + resident_physical_size_metric = ps_http.get_timeline_metric( + tenant_id, timeline_id, "pageserver_resident_physical_size" + ) + assert resident_physical_size_metric == 0 + log.info("ensure that resident_physical_size metric corresponds to layer map dump") + assert resident_physical_size_metric == sum( + [layer.layer_file_size or 0 for layer in info.historic_layers if not layer.remote] + ) + + # TODO: the following would be nice to assert, but for some reason, the commented-out + # assert below fails with 113401856.0 != 140427264 + # => https://github.com/neondatabase/neon/issues/3738 + # + # log.info("ensure that remote_physical_size metric matches layer map") + # remote_physical_size_metric = ps_http.get_timeline_metric( + # tenant_id, timeline_id, "pageserver_remote_physical_size" + # ) + # log.info("ensure that remote_physical_size metric corresponds to layer map dump") + # assert remote_physical_size_metric == sum( + # [layer.layer_file_size or 0 for layer in info.historic_layers if layer.remote] + # ) + + log.info("before runnning GC, ensure that remote_physical size is zero") + ensure_resident_and_remote_size_metrics() + + log.info("run GC") + time.sleep(2) # let pitr_interval + 1 second pass + ps_http.timeline_gc(tenant_id, timeline_id, 0) + time.sleep(1) + assert not env.pageserver.log_contains("Nothing to GC") + + log.info("ensure GC deleted some layers, otherwise this test is pointless") + post_gc_info = ps_http.layer_map_info(tenant_id, timeline_id) + log.info("layer map dump: %s", post_gc_info) + log.info("by kind: %s", post_gc_info.kind_count()) + pre_evict_layers = set([layer.layer_file_name for layer in pre_evict_info.historic_layers]) + post_gc_layers = set([layer.layer_file_name for layer in post_gc_info.historic_layers]) + assert post_gc_layers.issubset(pre_evict_layers) + assert len(post_gc_layers) < len(pre_evict_layers) + + log.info("update_gc_info might download some layers. Evict them again.") + ps_http.evict_all_layers(tenant_id, timeline_id) + + log.info("after running GC, ensure that resident size is still zero") + ensure_resident_and_remote_size_metrics() From 1b780fa752ee44af27c88a2bed352ee9554b5e01 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 2 Mar 2023 17:05:54 +0100 Subject: [PATCH 11/14] timeline_checkpoint_handler: add span with tenant and timeline id Before this patch, the logs written by freeze_and_flush() and compact() didn't have any span, which made the test logs annoying to read. --- pageserver/src/http/routes.rs | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index d2d9f24efb..3cfc68c1eb 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -971,19 +971,22 @@ async fn timeline_checkpoint_handler(request: Request) -> Result Date: Fri, 3 Mar 2023 14:53:27 +0100 Subject: [PATCH 12/14] tests: use `parse_metrics` everywhere (#3737) - use parse_metrics() in all places where we parse Prometheus metrics - query_all: make `filter` argument optional - encourage using properly parsed, typed metrics by changing get_metrics() to return already-parsed metrics. The new get_metric_str() method, like in the Safekeeper type, returns the raw text response. --- test_runner/fixtures/benchmark_fixture.py | 12 +---- test_runner/fixtures/metrics.py | 3 +- test_runner/fixtures/neon_fixtures.py | 47 ++++++++++--------- test_runner/regress/test_build_info_metric.py | 2 +- test_runner/regress/test_gc_aggressive.py | 3 +- test_runner/regress/test_metric_collection.py | 3 +- test_runner/regress/test_ondemand_download.py | 12 ++++- test_runner/regress/test_tenant_detach.py | 5 +- test_runner/regress/test_tenant_tasks.py | 14 ++++-- test_runner/regress/test_tenants.py | 6 +-- test_runner/regress/test_timeline_size.py | 33 +++++++------ 11 files changed, 74 insertions(+), 66 deletions(-) diff --git a/test_runner/fixtures/benchmark_fixture.py b/test_runner/fixtures/benchmark_fixture.py index b1489b7ab1..3428e6db8a 100644 --- a/test_runner/fixtures/benchmark_fixture.py +++ b/test_runner/fixtures/benchmark_fixture.py @@ -366,17 +366,9 @@ class NeonBenchmarker: def get_int_counter_value(self, pageserver: NeonPageserver, metric_name: str) -> int: """Fetch the value of given int counter from pageserver metrics.""" - # TODO: If we start to collect more of the prometheus metrics in the - # performance test suite like this, we should refactor this to load and - # parse all the metrics into a more convenient structure in one go. - # - # The metric should be an integer, as it's a number of bytes. But in general - # all prometheus metrics are floats. So to be pedantic, read it as a float - # and round to integer. all_metrics = pageserver.http_client().get_metrics() - matches = re.search(rf"^{metric_name} (\S+)$", all_metrics, re.MULTILINE) - assert matches, f"metric {metric_name} not found" - return int(round(float(matches.group(1)))) + sample = all_metrics.query_one(metric_name) + return int(round(sample.value)) def get_timeline_size( self, repo_dir: Path, tenant_id: TenantId, timeline_id: TimelineId diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index ba0d325c39..450c02735a 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -13,7 +13,8 @@ class Metrics: self.metrics = defaultdict(list) self.name = name - def query_all(self, name: str, filter: Dict[str, str]) -> List[Sample]: + def query_all(self, name: str, filter: Optional[Dict[str, str]] = None) -> List[Sample]: + filter = filter or {} res = [] for sample in self.metrics[name]: try: diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 49218f3c98..94ee1d50f7 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -29,7 +29,6 @@ import asyncpg import backoff # type: ignore import boto3 import jwt -import prometheus_client import psycopg2 import pytest import requests @@ -37,7 +36,7 @@ from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import FixtureRequest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics +from fixtures.metrics import Metrics, parse_metrics from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import ( ATTACHMENT_NAME_REGEX, @@ -46,7 +45,6 @@ from fixtures.utils import ( get_self_dir, subprocess_capture, ) -from prometheus_client.parser import text_string_to_metric_families # Type-related stuff from psycopg2.extensions import connection as PgConnection @@ -1437,22 +1435,27 @@ class PageserverHttpClient(requests.Session): assert completed["successful_download_count"] > 0 return completed - def get_metrics(self) -> str: + def get_metrics_str(self) -> str: + """You probably want to use get_metrics() instead.""" res = self.get(f"http://localhost:{self.port}/metrics") self.verbose_error(res) return res.text - def get_timeline_metric(self, tenant_id: TenantId, timeline_id: TimelineId, metric_name: str): - raw = self.get_metrics() - family: List[prometheus_client.Metric] = list(text_string_to_metric_families(raw)) - [metric] = [m for m in family if m.name == metric_name] - [sample] = [ - s - for s in metric.samples - if s.labels["tenant_id"] == str(tenant_id) - and s.labels["timeline_id"] == str(timeline_id) - ] - return sample.value + def get_metrics(self) -> Metrics: + res = self.get_metrics_str() + return parse_metrics(res) + + def get_timeline_metric( + self, tenant_id: TenantId, timeline_id: TimelineId, metric_name: str + ) -> float: + metrics = self.get_metrics() + return metrics.query_one( + metric_name, + filter={ + "tenant_id": str(tenant_id), + "timeline_id": str(timeline_id), + }, + ).value def get_remote_timeline_client_metric( self, @@ -1462,7 +1465,7 @@ class PageserverHttpClient(requests.Session): file_kind: str, op_kind: str, ) -> Optional[float]: - metrics = parse_metrics(self.get_metrics(), "pageserver") + metrics = self.get_metrics() matches = metrics.query_all( name=metric_name, filter={ @@ -1481,14 +1484,16 @@ class PageserverHttpClient(requests.Session): assert len(matches) < 2, "above filter should uniquely identify metric" return value - def get_metric_value(self, name: str) -> Optional[str]: + def get_metric_value( + self, name: str, filter: Optional[Dict[str, str]] = None + ) -> Optional[float]: metrics = self.get_metrics() - relevant = [line for line in metrics.splitlines() if line.startswith(name)] - if len(relevant) == 0: + results = metrics.query_all(name, filter=filter) + if not results: log.info(f'could not find metric "{name}"') return None - assert len(relevant) == 1 - return relevant[0].lstrip(name).strip() + assert len(results) == 1, f"metric {name} with given filters is not unique, got: {results}" + return results[0].value def layer_map_info( self, diff --git a/test_runner/regress/test_build_info_metric.py b/test_runner/regress/test_build_info_metric.py index b75b5bd775..c622d562fd 100644 --- a/test_runner/regress/test_build_info_metric.py +++ b/test_runner/regress/test_build_info_metric.py @@ -8,7 +8,7 @@ def test_build_info_metric(neon_env_builder: NeonEnvBuilder, link_proxy: NeonPro parsed_metrics = {} - parsed_metrics["pageserver"] = parse_metrics(env.pageserver.http_client().get_metrics()) + parsed_metrics["pageserver"] = parse_metrics(env.pageserver.http_client().get_metrics_str()) parsed_metrics["safekeeper"] = parse_metrics(env.safekeepers[0].http_client().get_metrics_str()) parsed_metrics["proxy"] = parse_metrics(link_proxy.get_metrics()) diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py index 5f052bf81a..77438e1b64 100644 --- a/test_runner/regress/test_gc_aggressive.py +++ b/test_runner/regress/test_gc_aggressive.py @@ -4,7 +4,6 @@ import random import pytest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, @@ -134,7 +133,7 @@ def test_gc_index_upload(neon_env_builder: NeonEnvBuilder, remote_storage_kind: # Helper function that gets the number of given kind of remote ops from the metrics def get_num_remote_ops(file_kind: str, op_kind: str) -> int: - ps_metrics = parse_metrics(env.pageserver.http_client().get_metrics(), "pageserver") + ps_metrics = env.pageserver.http_client().get_metrics() total = 0.0 for sample in ps_metrics.query_all( name="pageserver_remote_operation_seconds_count", diff --git a/test_runner/regress/test_metric_collection.py b/test_runner/regress/test_metric_collection.py index 3f252992f5..a33af9a3b2 100644 --- a/test_runner/regress/test_metric_collection.py +++ b/test_runner/regress/test_metric_collection.py @@ -9,7 +9,6 @@ from typing import Iterator import pytest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( PSQL, NeonEnvBuilder, @@ -143,7 +142,7 @@ def test_metric_collection( # Helper function that gets the number of given kind of remote ops from the metrics def get_num_remote_ops(file_kind: str, op_kind: str) -> int: - ps_metrics = parse_metrics(env.pageserver.http_client().get_metrics(), "pageserver") + ps_metrics = env.pageserver.http_client().get_metrics() total = 0.0 for sample in ps_metrics.query_all( name="pageserver_remote_operation_seconds_count", diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index 5ee94de32d..e6c580c37c 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -11,6 +11,7 @@ from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, PageserverApiException, + PageserverHttpClient, RemoteStorageKind, assert_tenant_status, available_remote_storages, @@ -25,9 +26,16 @@ from fixtures.types import Lsn from fixtures.utils import query_scalar -def get_num_downloaded_layers(client, tenant_id, timeline_id): +def get_num_downloaded_layers(client: PageserverHttpClient, tenant_id, timeline_id): value = client.get_metric_value( - f'pageserver_remote_operation_seconds_count{{file_kind="layer",op_kind="download",status="success",tenant_id="{tenant_id}",timeline_id="{timeline_id}"}}' + "pageserver_remote_operation_seconds_count", + { + "file_kind": "layer", + "op_kind": "download", + "status": "success", + "tenant_id": tenant_id, + "timeline_id": timeline_id, + }, ) if value is None: return 0 diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 6c3454b79b..ac1f7b2891 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -6,7 +6,6 @@ from threading import Thread import asyncpg import pytest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, @@ -79,7 +78,7 @@ def test_tenant_reattach( ".*failed to perform remote task UploadMetadata.*, will retry.*" ) - ps_metrics = parse_metrics(pageserver_http.get_metrics(), "pageserver") + ps_metrics = pageserver_http.get_metrics() tenant_metric_filter = { "tenant_id": str(tenant_id), "timeline_id": str(timeline_id), @@ -93,7 +92,7 @@ def test_tenant_reattach( time.sleep(1) # for metrics propagation - ps_metrics = parse_metrics(pageserver_http.get_metrics(), "pageserver") + ps_metrics = pageserver_http.get_metrics() pageserver_last_record_lsn = int( ps_metrics.query_one("pageserver_last_record_lsn", filter=tenant_metric_filter).value ) diff --git a/test_runner/regress/test_tenant_tasks.py b/test_runner/regress/test_tenant_tasks.py index 4eba4ce942..24b211e368 100644 --- a/test_runner/regress/test_tenant_tasks.py +++ b/test_runner/regress/test_tenant_tasks.py @@ -50,16 +50,22 @@ def test_tenant_tasks(neon_env_builder: NeonEnvBuilder): wait_until(10, 0.2, lambda: assert_active(tenant_id)) # Assert that all tasks finish quickly after tenant is detached - task_starts = client.get_metric_value('pageserver_tenant_task_events{event="start"}') + task_starts = client.get_metric_value("pageserver_tenant_task_events_total", {"event": "start"}) assert task_starts is not None assert int(task_starts) > 0 client.tenant_detach(tenant) client.tenant_detach(env.initial_tenant) def assert_tasks_finish(): - tasks_started = client.get_metric_value('pageserver_tenant_task_events{event="start"}') - tasks_ended = client.get_metric_value('pageserver_tenant_task_events{event="stop"}') - tasks_panicked = client.get_metric_value('pageserver_tenant_task_events{event="panic"}') + tasks_started = client.get_metric_value( + "pageserver_tenant_task_events_total", {"event": "start"} + ) + tasks_ended = client.get_metric_value( + "pageserver_tenant_task_events_total", {"event": "stop"} + ) + tasks_panicked = client.get_metric_value( + "pageserver_tenant_task_events_total", {"event": "panic"} + ) log.info(f"started {tasks_started}, ended {tasks_ended}, panicked {tasks_panicked}") assert tasks_started == tasks_ended assert tasks_panicked is None or int(tasks_panicked) == 0 diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index e56bb1b469..bf87cb3ad4 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -107,7 +107,7 @@ def test_metrics_normal_work(neon_env_builder: NeonEnvBuilder): assert cur.fetchone() == (5000050000,) collected_metrics = { - "pageserver": env.pageserver.http_client().get_metrics(), + "pageserver": env.pageserver.http_client().get_metrics_str(), } for sk in env.safekeepers: collected_metrics[f"safekeeper{sk.id}"] = sk.http_client().get_metrics_str() @@ -207,7 +207,7 @@ def test_pageserver_metrics_removed_after_detach( assert cur.fetchone() == (5000050000,) def get_ps_metric_samples_for_tenant(tenant_id: TenantId) -> List[Sample]: - ps_metrics = parse_metrics(env.pageserver.http_client().get_metrics(), "pageserver") + ps_metrics = env.pageserver.http_client().get_metrics() samples = [] for metric_name in ps_metrics.metrics: for sample in ps_metrics.query_all( @@ -307,7 +307,7 @@ def test_pageserver_with_empty_tenants( time.sleep(1) # to allow metrics propagation - ps_metrics = parse_metrics(client.get_metrics(), "pageserver") + ps_metrics = client.get_metrics() broken_tenants_metric_filter = { "tenant_id": str(tenant_without_timelines_dir), "state": "broken", diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index 3b41cc5c90..ca4f32fff9 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -1,7 +1,6 @@ import math import queue import random -import re import threading import time from contextlib import closing @@ -465,26 +464,26 @@ def test_timeline_size_metrics( # get the metrics and parse the metric for the current timeline's physical size metrics = env.pageserver.http_client().get_metrics() - matches = re.search( - f'^pageserver_resident_physical_size{{tenant_id="{env.initial_tenant}",timeline_id="{new_timeline_id}"}} (\\S+)$', - metrics, - re.MULTILINE, - ) - assert matches - tl_physical_size_metric = int(matches.group(1)) + tl_physical_size_metric = metrics.query_one( + name="pageserver_resident_physical_size", + filter={ + "tenant_id": str(env.initial_tenant), + "timeline_id": str(new_timeline_id), + }, + ).value # assert that the physical size metric matches the actual physical size on disk timeline_path = env.timeline_dir(env.initial_tenant, new_timeline_id) assert tl_physical_size_metric == get_timeline_dir_size(timeline_path) # Check that the logical size metric is sane, and matches - matches = re.search( - f'^pageserver_current_logical_size{{tenant_id="{env.initial_tenant}",timeline_id="{new_timeline_id}"}} (\\S+)$', - metrics, - re.MULTILINE, - ) - assert matches - tl_logical_size_metric = int(matches.group(1)) + tl_logical_size_metric = metrics.query_one( + name="pageserver_current_logical_size", + filter={ + "tenant_id": str(env.initial_tenant), + "timeline_id": str(new_timeline_id), + }, + ).value pgdatadir = test_output_dir / "pgdata-vanilla" pg_bin = PgBin(test_output_dir, pg_distrib_dir, pg_version) @@ -575,8 +574,8 @@ def get_physical_size_values( client = env.pageserver.http_client() - res.prometheus_resident_physical = client.get_timeline_metric( - tenant_id, timeline_id, "pageserver_resident_physical_size" + res.prometheus_resident_physical = int( + client.get_timeline_metric(tenant_id, timeline_id, "pageserver_resident_physical_size") ) detail = client.timeline_detail( From 66a5159511ec80e032824d85b6eafca9f01b31a6 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Thu, 2 Mar 2023 19:04:54 +0100 Subject: [PATCH 13/14] fix: compaction: no index upload scheduled if no on-demand downloads Commit 0cf7fd0fb82b082d02dfadd9d6a488a7f799d72f Compaction with on-demand download (#3598) introduced a subtle bug: if we don't have to do on-demand downloads, we only take one ROUND in fn compact() and exit early. Thereby, we miss scheduling the index part upload for any layers created by fn compact_inner(). Before that commit, we didn't have this problem. So, this patch fixes it. Since no regression test caught this, I went ahead and extended the timeline size tests to assert that, if remote storage is configured, 1. pageserver_remote_physical_size matches the other physical sizes 2. file sizes reported by the layer map info endpoint match the other physical size metrics Without the pageserver code fix, the regression test would fail at the physical size assertion, complaining that any of the resident physical size != remote physical size metric 50790400.0 != 18399232.0 I figured out what the problem is by comparing the remote storage and local directories like so, and noticed that the image layer in the local directory wasn't present on the remote side. It's size was exactly the difference 50790400.0 - 18399232.0 =32391168.0 fixes https://github.com/neondatabase/neon/issues/3738 --- pageserver/src/tenant/timeline.rs | 4 +- test_runner/fixtures/neon_fixtures.py | 20 +++ test_runner/regress/test_layer_eviction.py | 20 ++- test_runner/regress/test_timeline_size.py | 134 ++++++++++++++++++--- 4 files changed, 145 insertions(+), 33 deletions(-) diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 101b27bb97..8b24fd6ecd 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -662,8 +662,8 @@ impl Timeline { // update the index file on next flush iteration too. But it // could take a while until that happens. // - // Additionally, only do this on the terminal round before sleeping. - if last_round { + // Additionally, only do this once before we return from this function. + if last_round || res.is_ok() { if let Some(remote_client) = &self.remote_client { remote_client.schedule_index_upload_for_file_changes()?; } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 94ee1d50f7..ba98563693 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3542,3 +3542,23 @@ def wait_for_sk_commit_lsn_to_reach_remote_storage( ps_http.timeline_checkpoint(tenant_id, timeline_id) wait_for_upload(ps_http, tenant_id, timeline_id, lsn) return lsn + + +def wait_for_upload_queue_empty( + pageserver: NeonPageserver, tenant_id: TenantId, timeline_id: TimelineId +): + ps_http = pageserver.http_client() + while True: + all_metrics = ps_http.get_metrics() + tl = all_metrics.query_all( + "pageserver_remote_timeline_client_calls_unfinished", + { + "tenant_id": str(tenant_id), + "timeline_id": str(timeline_id), + }, + ) + assert len(tl) > 0 + log.info(f"upload queue for {tenant_id}/{timeline_id}: {tl}") + if all(m.value == 0 for m in tl): + return + time.sleep(0.2) diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index 404bd67050..e7c9713f98 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -268,18 +268,14 @@ def test_gc_of_remote_layers(neon_env_builder: NeonEnvBuilder): [layer.layer_file_size or 0 for layer in info.historic_layers if not layer.remote] ) - # TODO: the following would be nice to assert, but for some reason, the commented-out - # assert below fails with 113401856.0 != 140427264 - # => https://github.com/neondatabase/neon/issues/3738 - # - # log.info("ensure that remote_physical_size metric matches layer map") - # remote_physical_size_metric = ps_http.get_timeline_metric( - # tenant_id, timeline_id, "pageserver_remote_physical_size" - # ) - # log.info("ensure that remote_physical_size metric corresponds to layer map dump") - # assert remote_physical_size_metric == sum( - # [layer.layer_file_size or 0 for layer in info.historic_layers if layer.remote] - # ) + log.info("ensure that remote_physical_size metric matches layer map") + remote_physical_size_metric = ps_http.get_timeline_metric( + tenant_id, timeline_id, "pageserver_remote_physical_size" + ) + log.info("ensure that remote_physical_size metric corresponds to layer map dump") + assert remote_physical_size_metric == sum( + layer.layer_file_size or 0 for layer in info.historic_layers if layer.remote + ) log.info("before runnning GC, ensure that remote_physical size is zero") ensure_resident_and_remote_size_metrics() diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index ca4f32fff9..ea4b65c9a8 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -5,6 +5,7 @@ import threading import time from contextlib import closing from pathlib import Path +from typing import Optional import psycopg2.errors import psycopg2.extras @@ -18,9 +19,11 @@ from fixtures.neon_fixtures import ( PgBin, PortDistributor, Postgres, + RemoteStorageKind, VanillaPostgres, assert_tenant_status, wait_for_last_flush_lsn, + wait_for_upload_queue_empty, wait_until, ) from fixtures.types import TenantId, TimelineId @@ -301,8 +304,18 @@ def test_timeline_initial_logical_size_calculation_cancellation( # message emitted by the code behind failpoint "timeline-calculate-logical-size-check-dir-exists" -def test_timeline_physical_size_init(neon_simple_env: NeonEnv): - env = neon_simple_env +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_init( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + + env = neon_env_builder.init_start() + new_timeline_id = env.neon_cli.create_branch("test_timeline_physical_size_init") pg = env.postgres.create_start("test_timeline_physical_size_init") @@ -330,12 +343,22 @@ def test_timeline_physical_size_init(neon_simple_env: NeonEnv): ) assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) -def test_timeline_physical_size_post_checkpoint(neon_simple_env: NeonEnv): - env = neon_simple_env +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_post_checkpoint( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + + env = neon_env_builder.init_start() + pageserver_http = env.pageserver.http_client() new_timeline_id = env.neon_cli.create_branch("test_timeline_physical_size_post_checkpoint") pg = env.postgres.create_start("test_timeline_physical_size_post_checkpoint") @@ -353,11 +376,21 @@ def test_timeline_physical_size_post_checkpoint(neon_simple_env: NeonEnv): pageserver_http.timeline_checkpoint(env.initial_tenant, new_timeline_id) assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) -def test_timeline_physical_size_post_compaction(neon_env_builder: NeonEnvBuilder): +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_post_compaction( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + # Disable background compaction as we don't want it to happen after `get_physical_size` request # and before checking the expected size on disk, which makes the assertion failed neon_env_builder.pageserver_config_override = ( @@ -386,15 +419,33 @@ def test_timeline_physical_size_post_compaction(neon_env_builder: NeonEnvBuilder ) wait_for_last_flush_lsn(env, pg, env.initial_tenant, new_timeline_id) + + # shutdown safekeepers to prevent new data from coming in + for sk in env.safekeepers: + sk.stop() + pageserver_http.timeline_checkpoint(env.initial_tenant, new_timeline_id) pageserver_http.timeline_compact(env.initial_tenant, new_timeline_id) + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, env.initial_tenant, new_timeline_id) + assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) -def test_timeline_physical_size_post_gc(neon_env_builder: NeonEnvBuilder): +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_post_gc( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + # Disable background compaction and GC as we don't want it to happen after `get_physical_size` request # and before checking the expected size on disk, which makes the assertion failed neon_env_builder.pageserver_config_override = "tenant_config={checkpoint_distance=100000, compaction_period='0s', gc_period='0s', pitr_interval='1s'}" @@ -430,8 +481,12 @@ def test_timeline_physical_size_post_gc(neon_env_builder: NeonEnvBuilder): pageserver_http.timeline_checkpoint(env.initial_tenant, new_timeline_id) pageserver_http.timeline_gc(env.initial_tenant, new_timeline_id, gc_horizon=None) + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, env.initial_tenant, new_timeline_id) + assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) @@ -515,18 +570,29 @@ def test_timeline_size_metrics( assert math.isclose(dbsize_sum, tl_logical_size_metric, abs_tol=2 * 1024 * 1024) -def test_tenant_physical_size(neon_simple_env: NeonEnv): +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_tenant_physical_size( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): random.seed(100) - env = neon_simple_env + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + + env = neon_env_builder.init_start() + pageserver_http = env.pageserver.http_client() client = env.pageserver.http_client() tenant, timeline = env.neon_cli.create_tenant() + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, tenant, timeline) def get_timeline_resident_physical_size(timeline: TimelineId): - sizes = get_physical_size_values(env, tenant, timeline) - assert_physical_size_invariants(sizes) + sizes = get_physical_size_values(env, tenant, timeline, remote_storage_kind) + assert_physical_size_invariants(sizes, remote_storage_kind) return sizes.prometheus_resident_physical timeline_total_resident_physical_size = get_timeline_resident_physical_size(timeline) @@ -546,6 +612,9 @@ def test_tenant_physical_size(neon_simple_env: NeonEnv): wait_for_last_flush_lsn(env, pg, tenant, timeline) pageserver_http.timeline_checkpoint(tenant, timeline) + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, tenant, timeline) + timeline_total_resident_physical_size += get_timeline_resident_physical_size(timeline) pg.stop() @@ -563,21 +632,39 @@ def test_tenant_physical_size(neon_simple_env: NeonEnv): class TimelinePhysicalSizeValues: api_current_physical: int - prometheus_resident_physical: int + prometheus_resident_physical: float + prometheus_remote_physical: Optional[float] = None python_timelinedir_layerfiles_physical: int + layer_map_file_size_sum: int def get_physical_size_values( - env: NeonEnv, tenant_id: TenantId, timeline_id: TimelineId + env: NeonEnv, + tenant_id: TenantId, + timeline_id: TimelineId, + remote_storage_kind: Optional[RemoteStorageKind], ) -> TimelinePhysicalSizeValues: res = TimelinePhysicalSizeValues() client = env.pageserver.http_client() - res.prometheus_resident_physical = int( - client.get_timeline_metric(tenant_id, timeline_id, "pageserver_resident_physical_size") + res.layer_map_file_size_sum = sum( + layer.layer_file_size or 0 + for layer in client.layer_map_info(tenant_id, timeline_id).historic_layers ) + metrics = client.get_metrics() + metrics_filter = {"tenant_id": str(tenant_id), "timeline_id": str(timeline_id)} + res.prometheus_resident_physical = metrics.query_one( + "pageserver_resident_physical_size", metrics_filter + ).value + if remote_storage_kind is not None: + res.prometheus_remote_physical = metrics.query_one( + "pageserver_remote_physical_size", metrics_filter + ).value + else: + res.prometheus_remote_physical = None + detail = client.timeline_detail( tenant_id, timeline_id, include_timeline_dir_layer_file_size_sum=True ) @@ -589,11 +676,20 @@ def get_physical_size_values( return res -def assert_physical_size_invariants(sizes: TimelinePhysicalSizeValues): +def assert_physical_size_invariants( + sizes: TimelinePhysicalSizeValues, remote_storage_kind: Optional[RemoteStorageKind] +): # resident phyiscal size is defined as assert sizes.python_timelinedir_layerfiles_physical == sizes.prometheus_resident_physical + assert sizes.python_timelinedir_layerfiles_physical == sizes.layer_map_file_size_sum + # we don't do layer eviction, so, all layers are resident assert sizes.api_current_physical == sizes.prometheus_resident_physical + if remote_storage_kind is not None: + assert sizes.prometheus_resident_physical == sizes.prometheus_remote_physical + # XXX would be nice to assert layer file physical storage utilization here as well, but we can only do that for LocalFS + else: + assert sizes.prometheus_remote_physical is None # Timeline logical size initialization is an asynchronous background task that runs once, From 9cada8b59d26b97ad361b51c2e2dbf3fc89bd892 Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Fri, 3 Mar 2023 17:53:43 +0100 Subject: [PATCH 14/14] fix benchmarks, broken by PR #3737 Benchmarks only run on `main` branch, so, the pre-commit tests didn't catch these. --- test_runner/fixtures/benchmark_fixture.py | 17 +++++++++++------ test_runner/fixtures/compare_fixtures.py | 6 +++--- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/test_runner/fixtures/benchmark_fixture.py b/test_runner/fixtures/benchmark_fixture.py index 3428e6db8a..a39aaf8241 100644 --- a/test_runner/fixtures/benchmark_fixture.py +++ b/test_runner/fixtures/benchmark_fixture.py @@ -354,20 +354,25 @@ class NeonBenchmarker: """ Fetch the "cumulative # of bytes written" metric from the pageserver """ - metric_name = r'libmetrics_disk_io_bytes_total{io_operation="write"}' - return self.get_int_counter_value(pageserver, metric_name) + return self.get_int_counter_value( + pageserver, "libmetrics_disk_io_bytes_total", {"io_operation": "write"} + ) def get_peak_mem(self, pageserver: NeonPageserver) -> int: """ Fetch the "maxrss" metric from the pageserver """ - metric_name = r"libmetrics_maxrss_kb" - return self.get_int_counter_value(pageserver, metric_name) + return self.get_int_counter_value(pageserver, "libmetrics_maxrss_kb") - def get_int_counter_value(self, pageserver: NeonPageserver, metric_name: str) -> int: + def get_int_counter_value( + self, + pageserver: NeonPageserver, + metric_name: str, + label_filters: Optional[Dict[str, str]] = None, + ) -> int: """Fetch the value of given int counter from pageserver metrics.""" all_metrics = pageserver.http_client().get_metrics() - sample = all_metrics.query_one(metric_name) + sample = all_metrics.query_one(metric_name, label_filters) return int(round(sample.value)) def get_timeline_size( diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 17c0b19447..0ba926c8d2 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -144,12 +144,12 @@ class NeonCompare(PgCompare): "size", timeline_size / (1024 * 1024), "MB", report=MetricReport.LOWER_IS_BETTER ) - params = f'{{tenant_id="{self.tenant}",timeline_id="{self.timeline}"}}' + metric_filters = {"tenant_id": str(self.tenant), "timeline_id": str(self.timeline)} total_files = self.zenbenchmark.get_int_counter_value( - self.env.pageserver, "pageserver_created_persistent_files_total" + params + self.env.pageserver, "pageserver_created_persistent_files_total", metric_filters ) total_bytes = self.zenbenchmark.get_int_counter_value( - self.env.pageserver, "pageserver_written_persistent_bytes_total" + params + self.env.pageserver, "pageserver_written_persistent_bytes_total", metric_filters ) self.zenbenchmark.record( "data_uploaded", total_bytes / (1024 * 1024), "MB", report=MetricReport.LOWER_IS_BETTER