diff --git a/Cargo.lock b/Cargo.lock index 6b23144182..fe5aae6ae8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3307,6 +3307,7 @@ dependencies = [ "async-trait", "byteorder", "bytes", + "chrono", "clap 4.1.4", "const_format", "crc32c", @@ -4505,6 +4506,7 @@ dependencies = [ "byteorder", "bytes", "criterion", + "futures", "git-version", "heapless", "hex", @@ -4534,6 +4536,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "uuid", "workspace_hack", ] @@ -4840,7 +4843,6 @@ dependencies = [ "either", "fail", "futures", - "futures-channel", "futures-executor", "futures-util", "hashbrown 0.12.3", diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 68278ed07b..1a2cd9fb77 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -225,6 +225,36 @@ RUN wget https://github.com/iCyberon/pg_hashids/archive/refs/tags/v1.2.1.tar.gz make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_hashids.control +######################################################################################### +# +# Layer "rum-pg-build" +# compile rum extension +# +######################################################################################### +FROM build-deps AS rum-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +RUN wget https://github.com/postgrespro/rum/archive/refs/tags/1.3.13.tar.gz -O rum.tar.gz && \ + mkdir rum-src && cd rum-src && tar xvzf ../rum.tar.gz --strip-components=1 -C . && \ + make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ + make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config USE_PGXS=1 && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/rum.control + +######################################################################################### +# +# Layer "pgtap-pg-build" +# compile pgTAP extension +# +######################################################################################### +FROM build-deps AS pgtap-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +RUN wget https://github.com/theory/pgtap/archive/refs/tags/v1.2.0.tar.gz -O pgtap.tar.gz && \ + mkdir pgtap-src && cd pgtap-src && tar xvzf ../pgtap.tar.gz --strip-components=1 -C . && \ + make -j $(getconf _NPROCESSORS_ONLN) PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + make -j $(getconf _NPROCESSORS_ONLN) install PG_CONFIG=/usr/local/pgsql/bin/pg_config && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/pgtap.control + ######################################################################################### # # Layer "rust extensions" @@ -304,6 +334,8 @@ COPY --from=pg-jsonschema-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-graphql-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=hypopg-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-hashids-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=rum-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=pgtap-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ diff --git a/Dockerfile.vm-compute-node b/Dockerfile.vm-compute-node index af3bfb3590..dff40485de 100644 --- a/Dockerfile.vm-compute-node +++ b/Dockerfile.vm-compute-node @@ -10,23 +10,16 @@ RUN set -e \ && rm -f /etc/inittab \ && touch /etc/inittab -ADD vm-cgconfig.conf /etc/cgconfig.conf RUN set -e \ - && echo "::sysinit:cgconfigparser -l /etc/cgconfig.conf -s 1664" >> /etc/inittab \ - && echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart --cgroup=neon-postgres'" >> /etc/inittab + && echo "::respawn:su vm-informant -c '/usr/local/bin/vm-informant --auto-restart'" >> /etc/inittab # Combine, starting from non-VM compute node image. FROM $SRC_IMAGE as base -# Temporarily set user back to root so we can run apt update and adduser +# Temporarily set user back to root so we can run adduser USER root -RUN apt update && \ - apt install --no-install-recommends -y \ - cgroup-tools RUN adduser vm-informant --disabled-password --no-create-home USER postgres COPY --from=informant /etc/inittab /etc/inittab COPY --from=informant /usr/bin/vm-informant /usr/local/bin/vm-informant - -ENTRYPOINT ["/usr/sbin/cgexec", "-g", "*:neon-postgres", "/usr/local/bin/compute_ctl"] diff --git a/libs/utils/Cargo.toml b/libs/utils/Cargo.toml index 92e805ac58..6acdb6fa53 100644 --- a/libs/utils/Cargo.toml +++ b/libs/utils/Cargo.toml @@ -13,6 +13,7 @@ bincode.workspace = true bytes.workspace = true heapless.workspace = true hyper = { workspace = true, features = ["full"] } +futures = { workspace = true} routerify.workspace = true serde.workspace = true serde_json.workspace = true @@ -39,7 +40,7 @@ pq_proto.workspace = true workspace_hack.workspace = true url.workspace = true - +uuid = { version = "1.2", features = ["v4", "serde"] } [dev-dependencies] byteorder.workspace = true bytes.workspace = true diff --git a/libs/utils/src/http/endpoint.rs b/libs/utils/src/http/endpoint.rs index 9c300de7a7..41975f6944 100644 --- a/libs/utils/src/http/endpoint.rs +++ b/libs/utils/src/http/endpoint.rs @@ -8,8 +8,7 @@ use hyper::{Method, StatusCode}; use metrics::{register_int_counter, Encoder, IntCounter, TextEncoder}; use once_cell::sync::Lazy; use routerify::ext::RequestExt; -use routerify::RequestInfo; -use routerify::{Middleware, Router, RouterBuilder, RouterService}; +use routerify::{Middleware, RequestInfo, Router, RouterBuilder, RouterService}; use tokio::task::JoinError; use tracing; @@ -27,14 +26,35 @@ static SERVE_METRICS_COUNT: Lazy = Lazy::new(|| { .expect("failed to define a metric") }); +static X_REQUEST_ID_HEADER_STR: &str = "x-request-id"; + +static X_REQUEST_ID_HEADER: HeaderName = HeaderName::from_static(X_REQUEST_ID_HEADER_STR); +#[derive(Debug, Default, Clone)] +struct RequestId(String); + async fn logger(res: Response, info: RequestInfo) -> Result, ApiError> { + let request_id = info.context::().unwrap_or_default().0; + // cannot factor out the Level to avoid the repetition // because tracing can only work with const Level // which is not the case here + if info.method() == Method::GET && res.status() == StatusCode::OK { - tracing::debug!("{} {} {}", info.method(), info.uri().path(), res.status()); + tracing::debug!( + "{} {} {} {}", + info.method(), + info.uri().path(), + request_id, + res.status() + ); } else { - tracing::info!("{} {} {}", info.method(), info.uri().path(), res.status()); + tracing::info!( + "{} {} {} {}", + info.method(), + info.uri().path(), + request_id, + res.status() + ); } Ok(res) } @@ -63,9 +83,52 @@ async fn prometheus_metrics_handler(_req: Request) -> Result( +) -> Middleware { + Middleware::pre(move |req| async move { + let request_id = match req.headers().get(&X_REQUEST_ID_HEADER) { + Some(request_id) => request_id + .to_str() + .expect("extract request id value") + .to_owned(), + None => { + let request_id = uuid::Uuid::new_v4(); + request_id.to_string() + } + }; + + if req.method() == Method::GET { + tracing::debug!("{} {} {}", req.method(), req.uri().path(), request_id); + } else { + tracing::info!("{} {} {}", req.method(), req.uri().path(), request_id); + } + req.set_context(RequestId(request_id)); + + Ok(req) + }) +} + +async fn add_request_id_header_to_response( + mut res: Response, + req_info: RequestInfo, +) -> Result, ApiError> { + if let Some(request_id) = req_info.context::() { + if let Ok(request_header_value) = HeaderValue::from_str(&request_id.0) { + res.headers_mut() + .insert(&X_REQUEST_ID_HEADER, request_header_value); + }; + }; + + Ok(res) +} + pub fn make_router() -> RouterBuilder { Router::builder() + .middleware(add_request_id_middleware()) .middleware(Middleware::post_with_info(logger)) + .middleware(Middleware::post_with_info( + add_request_id_header_to_response, + )) .get("/metrics", prometheus_metrics_handler) .err_handler(error::handler) } @@ -231,3 +294,48 @@ where Ok(()) } +#[cfg(test)] +mod tests { + use super::*; + use futures::future::poll_fn; + use hyper::service::Service; + use routerify::RequestServiceBuilder; + use std::net::{IpAddr, SocketAddr}; + + #[tokio::test] + async fn test_request_id_returned() { + let builder = RequestServiceBuilder::new(make_router().build().unwrap()).unwrap(); + let remote_addr = SocketAddr::new(IpAddr::from_str("127.0.0.1").unwrap(), 80); + let mut service = builder.build(remote_addr); + if let Err(e) = poll_fn(|ctx| service.poll_ready(ctx)).await { + panic!("request service is not ready: {:?}", e); + } + + let mut req: Request = Request::default(); + req.headers_mut() + .append(&X_REQUEST_ID_HEADER, HeaderValue::from_str("42").unwrap()); + + let resp: Response = service.call(req).await.unwrap(); + + let header_val = resp.headers().get(&X_REQUEST_ID_HEADER).unwrap(); + + assert!(header_val == "42", "response header mismatch"); + } + + #[tokio::test] + async fn test_request_id_empty() { + let builder = RequestServiceBuilder::new(make_router().build().unwrap()).unwrap(); + let remote_addr = SocketAddr::new(IpAddr::from_str("127.0.0.1").unwrap(), 80); + let mut service = builder.build(remote_addr); + if let Err(e) = poll_fn(|ctx| service.poll_ready(ctx)).await { + panic!("request service is not ready: {:?}", e); + } + + let req: Request = Request::default(); + let resp: Response = service.call(req).await.unwrap(); + + let header_val = resp.headers().get(&X_REQUEST_ID_HEADER); + + assert_ne!(header_val, None, "response header should NOT be empty"); + } +} diff --git a/libs/utils/src/http/json.rs b/libs/utils/src/http/json.rs index 8981fdd1dd..40e61e3d0c 100644 --- a/libs/utils/src/http/json.rs +++ b/libs/utils/src/http/json.rs @@ -1,7 +1,9 @@ +use std::fmt::Display; + use anyhow::Context; use bytes::Buf; use hyper::{header, Body, Request, Response, StatusCode}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Serialize, Serializer}; use super::error::ApiError; @@ -31,3 +33,12 @@ pub fn json_response( .map_err(|e| ApiError::InternalServerError(e.into()))?; Ok(response) } + +/// Serialize through Display trait. +pub fn display_serialize(z: &F, s: S) -> Result +where + S: Serializer, + F: Display, +{ + s.serialize_str(&format!("{}", z)) +} diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index d2d9f24efb..3cfc68c1eb 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -971,19 +971,22 @@ async fn timeline_checkpoint_handler(request: Request) -> Result, #[serde(skip_serializing_if = "Option::is_none")] + #[serde(with = "humantime_serde")] #[serde(default)] pub checkpoint_timeout: Option, diff --git a/pageserver/src/tenant/storage_layer.rs b/pageserver/src/tenant/storage_layer.rs index 9198cfd1df..52ce2cab42 100644 --- a/pageserver/src/tenant/storage_layer.rs +++ b/pageserver/src/tenant/storage_layer.rs @@ -364,7 +364,7 @@ pub trait PersistentLayer: Layer { } /// Permanently remove this layer from disk. - fn delete(&self) -> Result<()>; + fn delete_resident_layer_file(&self) -> Result<()>; fn downcast_remote_layer(self: Arc) -> Option> { None diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index 4d1e08322d..37719dfce5 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -438,7 +438,7 @@ impl PersistentLayer for DeltaLayer { )) } - fn delete(&self) -> Result<()> { + fn delete_resident_layer_file(&self) -> Result<()> { // delete underlying file fs::remove_file(self.path())?; Ok(()) diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index e48abd38dd..e37e001eda 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -252,7 +252,7 @@ impl PersistentLayer for ImageLayer { unimplemented!(); } - fn delete(&self) -> Result<()> { + fn delete_resident_layer_file(&self) -> Result<()> { // delete underlying file fs::remove_file(self.path())?; Ok(()) diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index 8465a99339..dbce2e7888 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -155,8 +155,8 @@ impl PersistentLayer for RemoteLayer { bail!("cannot iterate a remote layer"); } - fn delete(&self) -> Result<()> { - Ok(()) + fn delete_resident_layer_file(&self) -> Result<()> { + bail!("remote layer has no layer file"); } fn downcast_remote_layer<'a>(self: Arc) -> Option> { diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index d46ac26e7d..8b24fd6ecd 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -662,8 +662,8 @@ impl Timeline { // update the index file on next flush iteration too. But it // could take a while until that happens. // - // Additionally, only do this on the terminal round before sleeping. - if last_round { + // Additionally, only do this once before we return from this function. + if last_round || res.is_ok() { if let Some(remote_client) = &self.remote_client { remote_client.schedule_index_upload_for_file_changes()?; } @@ -1047,11 +1047,12 @@ impl Timeline { return Ok(false); } - let layer_metadata = LayerFileMetadata::new( - local_layer - .file_size() - .expect("Local layer should have a file size"), - ); + let layer_file_size = local_layer + .file_size() + .expect("Local layer should have a file size"); + + let layer_metadata = LayerFileMetadata::new(layer_file_size); + let new_remote_layer = Arc::new(match local_layer.filename() { LayerFileName::Image(image_name) => RemoteLayer::new_img( self.tenant_id, @@ -1075,15 +1076,22 @@ impl Timeline { let replaced = match batch_updates.replace_historic(local_layer, new_remote_layer)? { Replacement::Replaced { .. } => { - let layer_size = local_layer.file_size(); - - if let Err(e) = local_layer.delete() { + if let Err(e) = local_layer.delete_resident_layer_file() { error!("failed to remove layer file on evict after replacement: {e:#?}"); } - - if let Some(layer_size) = layer_size { - self.metrics.resident_physical_size_gauge.sub(layer_size); - } + // Always decrement the physical size gauge, even if we failed to delete the file. + // Rationale: we already replaced the layer with a remote layer in the layer map, + // and any subsequent download_remote_layer will + // 1. overwrite the file on disk and + // 2. add the downloaded size to the resident size gauge. + // + // If there is no re-download, and we restart the pageserver, then load_layer_map + // will treat the file as a local layer again, count it towards resident size, + // and it'll be like the layer removal never happened. + // The bump in resident size is perhaps unexpected but overall a robust behavior. + self.metrics + .resident_physical_size_gauge + .sub(layer_file_size); true } @@ -1942,11 +1950,14 @@ impl Timeline { layer: Arc, updates: &mut BatchedUpdates<'_, dyn PersistentLayer>, ) -> anyhow::Result<()> { - let layer_size = layer.file_size(); - - layer.delete()?; - if let Some(layer_size) = layer_size { - self.metrics.resident_physical_size_gauge.sub(layer_size); + if !layer.is_remote_layer() { + layer.delete_resident_layer_file()?; + let layer_file_size = layer + .file_size() + .expect("Local layer should have a file size"); + self.metrics + .resident_physical_size_gauge + .sub(layer_file_size); } // TODO Removing from the bottom of the layer map is expensive. diff --git a/safekeeper/Cargo.toml b/safekeeper/Cargo.toml index 4ee8d82203..2424509477 100644 --- a/safekeeper/Cargo.toml +++ b/safekeeper/Cargo.toml @@ -10,6 +10,7 @@ anyhow.workspace = true async-trait.workspace = true byteorder.workspace = true bytes.workspace = true +chrono.workspace = true clap = { workspace = true, features = ["derive"] } const_format.workspace = true crc32c.workspace = true diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs new file mode 100644 index 0000000000..674cf9f6eb --- /dev/null +++ b/safekeeper/src/debug_dump.rs @@ -0,0 +1,264 @@ +//! Utils for dumping full state of the safekeeper. + +use std::fs; +use std::fs::DirEntry; +use std::io::BufReader; +use std::io::Read; +use std::path::PathBuf; + +use anyhow::Result; +use chrono::{DateTime, Utc}; +use postgres_ffi::XLogSegNo; +use serde::Serialize; + +use utils::http::json::display_serialize; +use utils::id::NodeId; +use utils::id::TenantTimelineId; +use utils::id::{TenantId, TimelineId}; +use utils::lsn::Lsn; + +use crate::safekeeper::SafeKeeperState; +use crate::safekeeper::SafekeeperMemState; +use crate::safekeeper::TermHistory; +use crate::SafeKeeperConf; + +use crate::timeline::ReplicaState; +use crate::GlobalTimelines; + +/// Various filters that influence the resulting JSON output. +#[derive(Debug, Serialize)] +pub struct Args { + /// Dump all available safekeeper state. False by default. + pub dump_all: bool, + + /// Dump control_file content. Uses value of `dump_all` by default. + pub dump_control_file: bool, + + /// Dump in-memory state. Uses value of `dump_all` by default. + pub dump_memory: bool, + + /// Dump all disk files in a timeline directory. Uses value of `dump_all` by default. + pub dump_disk_content: bool, + + /// Dump full term history. True by default. + pub dump_term_history: bool, + + /// Filter timelines by tenant_id. + pub tenant_id: Option, + + /// Filter timelines by timeline_id. + pub timeline_id: Option, +} + +/// Response for debug dump request. +#[derive(Debug, Serialize)] +pub struct Response { + pub start_time: DateTime, + pub finish_time: DateTime, + pub timelines: Vec, + pub timelines_count: usize, + pub config: Config, +} + +/// Safekeeper configuration. +#[derive(Debug, Serialize)] +pub struct Config { + pub id: NodeId, + pub workdir: PathBuf, + pub listen_pg_addr: String, + pub listen_http_addr: String, + pub no_sync: bool, + pub max_offloader_lag_bytes: u64, + pub wal_backup_enabled: bool, +} + +#[derive(Debug, Serialize)] +pub struct Timeline { + #[serde(serialize_with = "display_serialize")] + pub tenant_id: TenantId, + #[serde(serialize_with = "display_serialize")] + pub timeline_id: TimelineId, + pub control_file: Option, + pub memory: Option, + pub disk_content: Option, +} + +#[derive(Debug, Serialize)] +pub struct Memory { + pub is_cancelled: bool, + pub peers_info_len: usize, + pub replicas: Vec>, + pub wal_backup_active: bool, + pub active: bool, + pub num_computes: u32, + pub last_removed_segno: XLogSegNo, + pub epoch_start_lsn: Lsn, + pub mem_state: SafekeeperMemState, + + // PhysicalStorage state. + pub write_lsn: Lsn, + pub write_record_lsn: Lsn, + pub flush_lsn: Lsn, + pub file_open: bool, +} + +#[derive(Debug, Serialize)] +pub struct DiskContent { + pub files: Vec, +} + +#[derive(Debug, Serialize)] +pub struct FileInfo { + pub name: String, + pub size: u64, + pub created: DateTime, + pub modified: DateTime, + pub start_zeroes: u64, + pub end_zeroes: u64, + // TODO: add sha256 checksum +} + +/// Build debug dump response, using the provided [`Args`] filters. +pub fn build(args: Args) -> Result { + let start_time = Utc::now(); + let timelines_count = GlobalTimelines::timelines_count(); + + let ptrs_snapshot = if args.tenant_id.is_some() && args.timeline_id.is_some() { + // If both tenant_id and timeline_id are specified, we can just get the + // timeline directly, without taking a snapshot of the whole list. + let ttid = TenantTimelineId::new(args.tenant_id.unwrap(), args.timeline_id.unwrap()); + if let Ok(tli) = GlobalTimelines::get(ttid) { + vec![tli] + } else { + vec![] + } + } else { + // Otherwise, take a snapshot of the whole list. + GlobalTimelines::get_all() + }; + + // TODO: return Stream instead of Vec + let mut timelines = Vec::new(); + for tli in ptrs_snapshot { + let ttid = tli.ttid; + if let Some(tenant_id) = args.tenant_id { + if tenant_id != ttid.tenant_id { + continue; + } + } + if let Some(timeline_id) = args.timeline_id { + if timeline_id != ttid.timeline_id { + continue; + } + } + + let control_file = if args.dump_control_file { + let mut state = tli.get_state().1; + if !args.dump_term_history { + state.acceptor_state.term_history = TermHistory(vec![]); + } + Some(state) + } else { + None + }; + + let memory = if args.dump_memory { + Some(tli.memory_dump()) + } else { + None + }; + + let disk_content = if args.dump_disk_content { + // build_disk_content can fail, but we don't want to fail the whole + // request because of that. + build_disk_content(&tli.timeline_dir).ok() + } else { + None + }; + + let timeline = Timeline { + tenant_id: ttid.tenant_id, + timeline_id: ttid.timeline_id, + control_file, + memory, + disk_content, + }; + timelines.push(timeline); + } + + let config = GlobalTimelines::get_global_config(); + + Ok(Response { + start_time, + finish_time: Utc::now(), + timelines, + timelines_count, + config: build_config(config), + }) +} + +/// Builds DiskContent from a directory path. It can fail if the directory +/// is deleted between the time we get the path and the time we try to open it. +fn build_disk_content(path: &std::path::Path) -> Result { + let mut files = Vec::new(); + for entry in fs::read_dir(path)? { + if entry.is_err() { + continue; + } + let file = build_file_info(entry?); + if file.is_err() { + continue; + } + files.push(file?); + } + + Ok(DiskContent { files }) +} + +/// Builds FileInfo from DirEntry. Sometimes it can return an error +/// if the file is deleted between the time we get the DirEntry +/// and the time we try to open it. +fn build_file_info(entry: DirEntry) -> Result { + let metadata = entry.metadata()?; + let path = entry.path(); + let name = path + .file_name() + .and_then(|x| x.to_str()) + .unwrap_or("") + .to_owned(); + let mut file = fs::File::open(path)?; + let mut reader = BufReader::new(&mut file).bytes().filter_map(|x| x.ok()); + + let start_zeroes = reader.by_ref().take_while(|&x| x == 0).count() as u64; + let mut end_zeroes = 0; + for b in reader { + if b == 0 { + end_zeroes += 1; + } else { + end_zeroes = 0; + } + } + + Ok(FileInfo { + name, + size: metadata.len(), + created: DateTime::from(metadata.created()?), + modified: DateTime::from(metadata.modified()?), + start_zeroes, + end_zeroes, + }) +} + +/// Converts SafeKeeperConf to Config, filtering out the fields that are not +/// supposed to be exposed. +fn build_config(config: SafeKeeperConf) -> Config { + Config { + id: config.my_id, + workdir: config.workdir, + listen_pg_addr: config.listen_pg_addr, + listen_http_addr: config.listen_http_addr, + no_sync: config.no_sync, + max_offloader_lag_bytes: config.max_offloader_lag_bytes, + wal_backup_enabled: config.wal_backup_enabled, + } +} diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index a917d61678..ced9599b36 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -5,14 +5,16 @@ use once_cell::sync::Lazy; use postgres_ffi::WAL_SEGMENT_SIZE; use safekeeper_api::models::SkTimelineInfo; use serde::Serialize; -use serde::Serializer; use std::collections::{HashMap, HashSet}; -use std::fmt::Display; +use std::fmt; +use std::str::FromStr; use std::sync::Arc; use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use tokio::task::JoinError; +use utils::http::json::display_serialize; +use crate::debug_dump; use crate::safekeeper::ServerInfo; use crate::safekeeper::Term; @@ -54,15 +56,6 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { .as_ref() } -/// Serialize through Display trait. -fn display_serialize(z: &F, s: S) -> Result -where - S: Serializer, - F: Display, -{ - s.serialize_str(&format!("{}", z)) -} - /// Same as TermSwitchEntry, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. #[derive(Debug, Serialize)] @@ -276,6 +269,69 @@ async fn record_safekeeper_info(mut request: Request) -> Result>(k: &str, v: &str) -> Result { + v.parse() + .map_err(|e| ApiError::BadRequest(anyhow::anyhow!("cannot parse {k}: {e}"))) +} + +/// Dump debug info about all available safekeeper state. +async fn dump_debug_handler(mut request: Request) -> Result, ApiError> { + check_permission(&request, None)?; + ensure_no_body(&mut request).await?; + + let mut dump_all: Option = None; + let mut dump_control_file: Option = None; + let mut dump_memory: Option = None; + let mut dump_disk_content: Option = None; + let mut dump_term_history: Option = None; + let mut tenant_id: Option = None; + let mut timeline_id: Option = None; + + let query = request.uri().query().unwrap_or(""); + let mut values = url::form_urlencoded::parse(query.as_bytes()); + + for (k, v) in &mut values { + match k.as_ref() { + "dump_all" => dump_all = Some(parse_kv_str(&k, &v)?), + "dump_control_file" => dump_control_file = Some(parse_kv_str(&k, &v)?), + "dump_memory" => dump_memory = Some(parse_kv_str(&k, &v)?), + "dump_disk_content" => dump_disk_content = Some(parse_kv_str(&k, &v)?), + "dump_term_history" => dump_term_history = Some(parse_kv_str(&k, &v)?), + "tenant_id" => tenant_id = Some(parse_kv_str(&k, &v)?), + "timeline_id" => timeline_id = Some(parse_kv_str(&k, &v)?), + _ => Err(ApiError::BadRequest(anyhow::anyhow!( + "Unknown query parameter: {}", + k + )))?, + } + } + + let dump_all = dump_all.unwrap_or(false); + let dump_control_file = dump_control_file.unwrap_or(dump_all); + let dump_memory = dump_memory.unwrap_or(dump_all); + let dump_disk_content = dump_disk_content.unwrap_or(dump_all); + let dump_term_history = dump_term_history.unwrap_or(true); + + let args = debug_dump::Args { + dump_all, + dump_control_file, + dump_memory, + dump_disk_content, + dump_term_history, + tenant_id, + timeline_id, + }; + + let resp = tokio::task::spawn_blocking(move || { + debug_dump::build(args).map_err(ApiError::InternalServerError) + }) + .await + .map_err(|e: JoinError| ApiError::InternalServerError(e.into()))??; + + // TODO: use streaming response + json_response(StatusCode::OK, resp) +} + /// Safekeeper http router. pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder { let mut router = endpoint::make_router(); @@ -316,6 +372,7 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder "/v1/record_safekeeper_info/:tenant_id/:timeline_id", record_safekeeper_info, ) + .get("/v1/debug_dump", dump_debug_handler) } #[cfg(test)] diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 891d73533f..6ab108ceb0 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -10,6 +10,7 @@ mod auth; pub mod broker; pub mod control_file; pub mod control_file_upgrade; +pub mod debug_dump; pub mod handler; pub mod http; pub mod json_ctrl; diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index fa973a3ede..c37411d667 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -204,7 +204,7 @@ pub struct SafeKeeperState { pub peers: PersistedPeers, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize)] // In memory safekeeper state. Fields mirror ones in `SafeKeeperState`; values // are not flushed yet. pub struct SafekeeperMemState { @@ -212,6 +212,7 @@ pub struct SafekeeperMemState { pub backup_lsn: Lsn, pub peer_horizon_lsn: Lsn, pub remote_consistent_lsn: Lsn, + #[serde(with = "hex")] pub proposer_uuid: PgUuid, } diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 43c395574f..7479741774 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -5,6 +5,7 @@ use anyhow::{bail, Result}; use parking_lot::{Mutex, MutexGuard}; use postgres_ffi::XLogSegNo; use pq_proto::ReplicationFeedback; +use serde::Serialize; use std::cmp::{max, min}; use std::path::PathBuf; use tokio::{ @@ -28,9 +29,9 @@ use crate::send_wal::HotStandbyFeedback; use crate::{control_file, safekeeper::UNKNOWN_SERVER_VERSION}; use crate::metrics::FullTimelineInfo; -use crate::wal_storage; use crate::wal_storage::Storage as wal_storage_iface; use crate::SafeKeeperConf; +use crate::{debug_dump, wal_storage}; /// Things safekeeper should know about timeline state on peers. #[derive(Debug, Clone)] @@ -80,7 +81,7 @@ impl PeersInfo { } /// Replica status update + hot standby feedback -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, Copy, Serialize)] pub struct ReplicaState { /// last known lsn received by replica pub last_received_lsn: Lsn, // None means we don't know @@ -381,7 +382,7 @@ pub struct Timeline { cancellation_rx: watch::Receiver, /// Directory where timeline state is stored. - timeline_dir: PathBuf, + pub timeline_dir: PathBuf, } impl Timeline { @@ -588,38 +589,6 @@ impl Timeline { self.write_shared_state().wal_backup_attend() } - /// Returns full timeline info, required for the metrics. If the timeline is - /// not active, returns None instead. - pub fn info_for_metrics(&self) -> Option { - if self.is_cancelled() { - return None; - } - - let state = self.write_shared_state(); - if state.active { - Some(FullTimelineInfo { - ttid: self.ttid, - replicas: state - .replicas - .iter() - .filter_map(|r| r.as_ref()) - .copied() - .collect(), - wal_backup_active: state.wal_backup_active, - timeline_is_active: state.active, - num_computes: state.num_computes, - last_removed_segno: state.last_removed_segno, - epoch_start_lsn: state.sk.epoch_start_lsn, - mem_state: state.sk.inmem.clone(), - persisted_state: state.sk.state.clone(), - flush_lsn: state.sk.wal_store.flush_lsn(), - wal_storage: state.sk.wal_store.get_metrics(), - }) - } else { - None - } - } - /// Returns commit_lsn watch channel. pub fn get_commit_lsn_watch_rx(&self) -> watch::Receiver { self.commit_lsn_watch_rx.clone() @@ -784,6 +753,62 @@ impl Timeline { shared_state.last_removed_segno = horizon_segno; Ok(()) } + + /// Returns full timeline info, required for the metrics. If the timeline is + /// not active, returns None instead. + pub fn info_for_metrics(&self) -> Option { + if self.is_cancelled() { + return None; + } + + let state = self.write_shared_state(); + if state.active { + Some(FullTimelineInfo { + ttid: self.ttid, + replicas: state + .replicas + .iter() + .filter_map(|r| r.as_ref()) + .copied() + .collect(), + wal_backup_active: state.wal_backup_active, + timeline_is_active: state.active, + num_computes: state.num_computes, + last_removed_segno: state.last_removed_segno, + epoch_start_lsn: state.sk.epoch_start_lsn, + mem_state: state.sk.inmem.clone(), + persisted_state: state.sk.state.clone(), + flush_lsn: state.sk.wal_store.flush_lsn(), + wal_storage: state.sk.wal_store.get_metrics(), + }) + } else { + None + } + } + + /// Returns in-memory timeline state to build a full debug dump. + pub fn memory_dump(&self) -> debug_dump::Memory { + let state = self.write_shared_state(); + + let (write_lsn, write_record_lsn, flush_lsn, file_open) = + state.sk.wal_store.internal_state(); + + debug_dump::Memory { + is_cancelled: self.is_cancelled(), + peers_info_len: state.peers_info.0.len(), + replicas: state.replicas.clone(), + wal_backup_active: state.wal_backup_active, + active: state.active, + num_computes: state.num_computes, + last_removed_segno: state.last_removed_segno, + epoch_start_lsn: state.sk.epoch_start_lsn, + mem_state: state.sk.inmem.clone(), + write_lsn, + write_record_lsn, + flush_lsn, + file_open, + } + } } /// Deletes directory and it's contents. Returns false if directory does not exist. diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs index 66e0145042..baef17ffa8 100644 --- a/safekeeper/src/timelines_global_map.rs +++ b/safekeeper/src/timelines_global_map.rs @@ -159,6 +159,16 @@ impl GlobalTimelines { Ok(()) } + /// Get the number of timelines in the map. + pub fn timelines_count() -> usize { + TIMELINES_STATE.lock().unwrap().timelines.len() + } + + /// Get the global safekeeper config. + pub fn get_global_config() -> SafeKeeperConf { + TIMELINES_STATE.lock().unwrap().get_conf().clone() + } + /// Create a new timeline with the given id. If the timeline already exists, returns /// an existing timeline. pub fn create( diff --git a/safekeeper/src/wal_storage.rs b/safekeeper/src/wal_storage.rs index 561104bd27..ae02b3c7bc 100644 --- a/safekeeper/src/wal_storage.rs +++ b/safekeeper/src/wal_storage.rs @@ -165,6 +165,16 @@ impl PhysicalStorage { }) } + /// Get all known state of the storage. + pub fn internal_state(&self) -> (Lsn, Lsn, Lsn, bool) { + ( + self.write_lsn, + self.write_record_lsn, + self.flush_record_lsn, + self.file.is_some(), + ) + } + /// Call fdatasync if config requires so. fn fdatasync_file(&mut self, file: &mut File) -> Result<()> { if !self.conf.no_sync { diff --git a/test_runner/fixtures/benchmark_fixture.py b/test_runner/fixtures/benchmark_fixture.py index b1489b7ab1..a39aaf8241 100644 --- a/test_runner/fixtures/benchmark_fixture.py +++ b/test_runner/fixtures/benchmark_fixture.py @@ -354,29 +354,26 @@ class NeonBenchmarker: """ Fetch the "cumulative # of bytes written" metric from the pageserver """ - metric_name = r'libmetrics_disk_io_bytes_total{io_operation="write"}' - return self.get_int_counter_value(pageserver, metric_name) + return self.get_int_counter_value( + pageserver, "libmetrics_disk_io_bytes_total", {"io_operation": "write"} + ) def get_peak_mem(self, pageserver: NeonPageserver) -> int: """ Fetch the "maxrss" metric from the pageserver """ - metric_name = r"libmetrics_maxrss_kb" - return self.get_int_counter_value(pageserver, metric_name) + return self.get_int_counter_value(pageserver, "libmetrics_maxrss_kb") - def get_int_counter_value(self, pageserver: NeonPageserver, metric_name: str) -> int: + def get_int_counter_value( + self, + pageserver: NeonPageserver, + metric_name: str, + label_filters: Optional[Dict[str, str]] = None, + ) -> int: """Fetch the value of given int counter from pageserver metrics.""" - # TODO: If we start to collect more of the prometheus metrics in the - # performance test suite like this, we should refactor this to load and - # parse all the metrics into a more convenient structure in one go. - # - # The metric should be an integer, as it's a number of bytes. But in general - # all prometheus metrics are floats. So to be pedantic, read it as a float - # and round to integer. all_metrics = pageserver.http_client().get_metrics() - matches = re.search(rf"^{metric_name} (\S+)$", all_metrics, re.MULTILINE) - assert matches, f"metric {metric_name} not found" - return int(round(float(matches.group(1)))) + sample = all_metrics.query_one(metric_name, label_filters) + return int(round(sample.value)) def get_timeline_size( self, repo_dir: Path, tenant_id: TenantId, timeline_id: TimelineId diff --git a/test_runner/fixtures/compare_fixtures.py b/test_runner/fixtures/compare_fixtures.py index 17c0b19447..0ba926c8d2 100644 --- a/test_runner/fixtures/compare_fixtures.py +++ b/test_runner/fixtures/compare_fixtures.py @@ -144,12 +144,12 @@ class NeonCompare(PgCompare): "size", timeline_size / (1024 * 1024), "MB", report=MetricReport.LOWER_IS_BETTER ) - params = f'{{tenant_id="{self.tenant}",timeline_id="{self.timeline}"}}' + metric_filters = {"tenant_id": str(self.tenant), "timeline_id": str(self.timeline)} total_files = self.zenbenchmark.get_int_counter_value( - self.env.pageserver, "pageserver_created_persistent_files_total" + params + self.env.pageserver, "pageserver_created_persistent_files_total", metric_filters ) total_bytes = self.zenbenchmark.get_int_counter_value( - self.env.pageserver, "pageserver_written_persistent_bytes_total" + params + self.env.pageserver, "pageserver_written_persistent_bytes_total", metric_filters ) self.zenbenchmark.record( "data_uploaded", total_bytes / (1024 * 1024), "MB", report=MetricReport.LOWER_IS_BETTER diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index ba0d325c39..450c02735a 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -13,7 +13,8 @@ class Metrics: self.metrics = defaultdict(list) self.name = name - def query_all(self, name: str, filter: Dict[str, str]) -> List[Sample]: + def query_all(self, name: str, filter: Optional[Dict[str, str]] = None) -> List[Sample]: + filter = filter or {} res = [] for sample in self.metrics[name]: try: diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index c4b3d057f8..ba98563693 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -14,6 +14,7 @@ import tempfile import textwrap import time import uuid +from collections import defaultdict from contextlib import closing, contextmanager from dataclasses import dataclass, field from enum import Flag, auto @@ -28,7 +29,6 @@ import asyncpg import backoff # type: ignore import boto3 import jwt -import prometheus_client import psycopg2 import pytest import requests @@ -36,7 +36,7 @@ from _pytest.config import Config from _pytest.config.argparsing import Parser from _pytest.fixtures import FixtureRequest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics +from fixtures.metrics import Metrics, parse_metrics from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import ( ATTACHMENT_NAME_REGEX, @@ -45,7 +45,6 @@ from fixtures.utils import ( get_self_dir, subprocess_capture, ) -from prometheus_client.parser import text_string_to_metric_families # Type-related stuff from psycopg2.extensions import connection as PgConnection @@ -1436,22 +1435,27 @@ class PageserverHttpClient(requests.Session): assert completed["successful_download_count"] > 0 return completed - def get_metrics(self) -> str: + def get_metrics_str(self) -> str: + """You probably want to use get_metrics() instead.""" res = self.get(f"http://localhost:{self.port}/metrics") self.verbose_error(res) return res.text - def get_timeline_metric(self, tenant_id: TenantId, timeline_id: TimelineId, metric_name: str): - raw = self.get_metrics() - family: List[prometheus_client.Metric] = list(text_string_to_metric_families(raw)) - [metric] = [m for m in family if m.name == metric_name] - [sample] = [ - s - for s in metric.samples - if s.labels["tenant_id"] == str(tenant_id) - and s.labels["timeline_id"] == str(timeline_id) - ] - return sample.value + def get_metrics(self) -> Metrics: + res = self.get_metrics_str() + return parse_metrics(res) + + def get_timeline_metric( + self, tenant_id: TenantId, timeline_id: TimelineId, metric_name: str + ) -> float: + metrics = self.get_metrics() + return metrics.query_one( + metric_name, + filter={ + "tenant_id": str(tenant_id), + "timeline_id": str(timeline_id), + }, + ).value def get_remote_timeline_client_metric( self, @@ -1461,7 +1465,7 @@ class PageserverHttpClient(requests.Session): file_kind: str, op_kind: str, ) -> Optional[float]: - metrics = parse_metrics(self.get_metrics(), "pageserver") + metrics = self.get_metrics() matches = metrics.query_all( name=metric_name, filter={ @@ -1480,14 +1484,16 @@ class PageserverHttpClient(requests.Session): assert len(matches) < 2, "above filter should uniquely identify metric" return value - def get_metric_value(self, name: str) -> Optional[str]: + def get_metric_value( + self, name: str, filter: Optional[Dict[str, str]] = None + ) -> Optional[float]: metrics = self.get_metrics() - relevant = [line for line in metrics.splitlines() if line.startswith(name)] - if len(relevant) == 0: + results = metrics.query_all(name, filter=filter) + if not results: log.info(f'could not find metric "{name}"') return None - assert len(relevant) == 1 - return relevant[0].lstrip(name).strip() + assert len(results) == 1, f"metric {name} with given filters is not unique, got: {results}" + return results[0].value def layer_map_info( self, @@ -1516,6 +1522,11 @@ class PageserverHttpClient(requests.Session): assert res.status_code == 200 + def evict_all_layers(self, tenant_id: TenantId, timeline_id: TimelineId): + info = self.layer_map_info(tenant_id, timeline_id) + for layer in info.historic_layers: + self.evict_layer(tenant_id, timeline_id, layer.layer_file_name) + @dataclass class TenantConfig: @@ -1551,6 +1562,14 @@ class LayerMapInfo: return info + def kind_count(self) -> Dict[str, int]: + counts: Dict[str, int] = defaultdict(int) + for inmem_layer in self.in_memory_layers: + counts[inmem_layer.kind] += 1 + for hist_layer in self.historic_layers: + counts[hist_layer.kind] += 1 + return counts + @dataclass class InMemoryLayerInfo: @@ -1567,7 +1586,7 @@ class InMemoryLayerInfo: ) -@dataclass +@dataclass(frozen=True) class HistoricLayerInfo: kind: str layer_file_name: str @@ -2988,6 +3007,13 @@ class SafekeeperHttpClient(requests.Session): def check_status(self): self.get(f"http://localhost:{self.port}/v1/status").raise_for_status() + def debug_dump(self, params: Dict[str, str] = {}) -> Dict[str, Any]: + res = self.get(f"http://localhost:{self.port}/v1/debug_dump", params=params) + res.raise_for_status() + res_json = res.json() + assert isinstance(res_json, dict) + return res_json + def timeline_create( self, tenant_id: TenantId, timeline_id: TimelineId, pg_version: int, commit_lsn: Lsn ): @@ -3516,3 +3542,23 @@ def wait_for_sk_commit_lsn_to_reach_remote_storage( ps_http.timeline_checkpoint(tenant_id, timeline_id) wait_for_upload(ps_http, tenant_id, timeline_id, lsn) return lsn + + +def wait_for_upload_queue_empty( + pageserver: NeonPageserver, tenant_id: TenantId, timeline_id: TimelineId +): + ps_http = pageserver.http_client() + while True: + all_metrics = ps_http.get_metrics() + tl = all_metrics.query_all( + "pageserver_remote_timeline_client_calls_unfinished", + { + "tenant_id": str(tenant_id), + "timeline_id": str(timeline_id), + }, + ) + assert len(tl) > 0 + log.info(f"upload queue for {tenant_id}/{timeline_id}: {tl}") + if all(m.value == 0 for m in tl): + return + time.sleep(0.2) diff --git a/test_runner/regress/test_build_info_metric.py b/test_runner/regress/test_build_info_metric.py index b75b5bd775..c622d562fd 100644 --- a/test_runner/regress/test_build_info_metric.py +++ b/test_runner/regress/test_build_info_metric.py @@ -8,7 +8,7 @@ def test_build_info_metric(neon_env_builder: NeonEnvBuilder, link_proxy: NeonPro parsed_metrics = {} - parsed_metrics["pageserver"] = parse_metrics(env.pageserver.http_client().get_metrics()) + parsed_metrics["pageserver"] = parse_metrics(env.pageserver.http_client().get_metrics_str()) parsed_metrics["safekeeper"] = parse_metrics(env.safekeepers[0].http_client().get_metrics_str()) parsed_metrics["proxy"] = parse_metrics(link_proxy.get_metrics()) diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py index 5f052bf81a..77438e1b64 100644 --- a/test_runner/regress/test_gc_aggressive.py +++ b/test_runner/regress/test_gc_aggressive.py @@ -4,7 +4,6 @@ import random import pytest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, @@ -134,7 +133,7 @@ def test_gc_index_upload(neon_env_builder: NeonEnvBuilder, remote_storage_kind: # Helper function that gets the number of given kind of remote ops from the metrics def get_num_remote_ops(file_kind: str, op_kind: str) -> int: - ps_metrics = parse_metrics(env.pageserver.http_client().get_metrics(), "pageserver") + ps_metrics = env.pageserver.http_client().get_metrics() total = 0.0 for sample in ps_metrics.query_all( name="pageserver_remote_operation_seconds_count", diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index a03dd88c41..e7c9713f98 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -1,8 +1,13 @@ +import time + import pytest +from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, RemoteStorageKind, + wait_for_last_flush_lsn, wait_for_last_record_lsn, + wait_for_sk_commit_lsn_to_reach_remote_storage, wait_for_upload, ) from fixtures.types import Lsn, TenantId, TimelineId @@ -138,3 +143,160 @@ def test_basic_eviction( assert ( redownloaded_layer_map_info == initial_layer_map_info ), "Should have the same layer map after redownloading the evicted layers" + + +def test_gc_of_remote_layers(neon_env_builder: NeonEnvBuilder): + + neon_env_builder.enable_remote_storage( + remote_storage_kind=RemoteStorageKind.LOCAL_FS, + test_name="test_gc_of_remote_layers", + ) + + env = neon_env_builder.init_start() + + tenant_config = { + "pitr_interval": "1s", # set to non-zero, so GC actually does something + "gc_period": "0s", # we want to control when GC runs + "compaction_period": "0s", # we want to control when compaction runs + "checkpoint_timeout": "24h", # something we won't reach + "checkpoint_distance": f"{50 * (1024**2)}", # something we won't reach, we checkpoint manually + "compaction_threshold": "3", + # "image_creation_threshold": set at runtime + "compaction_target_size": f"{128 * (1024**2)}", # make it so that we only have 1 partition => image coverage for delta layers => enables gc of delta layers + } + + def tenant_update_config(changes): + tenant_config.update(changes) + env.neon_cli.config_tenant(tenant_id, tenant_config) + + tenant_id, timeline_id = env.neon_cli.create_tenant(conf=tenant_config) + log.info("tenant id is %s", tenant_id) + env.initial_tenant = tenant_id # update_and_gc relies on this + ps_http = env.pageserver.http_client() + + pg = env.postgres.create_start("main") + + log.info("fill with data, creating delta & image layers, some of which are GC'able after") + # no particular reason to create the layers like this, but we are sure + # not to hit the image_creation_threshold here. + with pg.cursor() as cur: + cur.execute("create table a (id bigserial primary key, some_value bigint not null)") + cur.execute("insert into a(some_value) select i from generate_series(1, 10000) s(i)") + wait_for_last_flush_lsn(env, pg, tenant_id, timeline_id) + ps_http.timeline_checkpoint(tenant_id, timeline_id) + + # Create delta layers, then turn them into image layers. + # Do it multiple times so that there's something to GC. + for k in range(0, 2): + # produce delta layers => disable image layer creation by setting high threshold + tenant_update_config({"image_creation_threshold": "100"}) + for i in range(0, 2): + for j in range(0, 3): + # create a minimal amount of "delta difficulty" for this table + with pg.cursor() as cur: + cur.execute("update a set some_value = -some_value + %s", (j,)) + + with pg.cursor() as cur: + # vacuuming should aid to reuse keys, though it's not really important + # with image_creation_threshold=1 which we will use on the last compaction + cur.execute("vacuum") + + wait_for_last_flush_lsn(env, pg, tenant_id, timeline_id) + + if i == 1 and j == 2 and k == 1: + # last iteration; stop before checkpoint to avoid leaving an inmemory layer + pg.stop_and_destroy() + + ps_http.timeline_checkpoint(tenant_id, timeline_id) + + # images should not yet be created, because threshold is too high, + # but these will be reshuffled to L1 layers + ps_http.timeline_compact(tenant_id, timeline_id) + + for _ in range(0, 20): + # loop in case flushing is still in progress + layers = ps_http.layer_map_info(tenant_id, timeline_id) + if not layers.in_memory_layers: + break + time.sleep(0.2) + + # now that we've grown some delta layers, turn them into image layers + tenant_update_config({"image_creation_threshold": "1"}) + ps_http.timeline_compact(tenant_id, timeline_id) + + # wait for all uploads to finish + wait_for_sk_commit_lsn_to_reach_remote_storage( + tenant_id, timeline_id, env.safekeepers, env.pageserver + ) + + # shutdown safekeepers to avoid on-demand downloads from walreceiver + for sk in env.safekeepers: + sk.stop() + + ps_http.timeline_checkpoint(tenant_id, timeline_id) + + log.info("ensure the code above produced image and delta layers") + pre_evict_info = ps_http.layer_map_info(tenant_id, timeline_id) + log.info("layer map dump: %s", pre_evict_info) + by_kind = pre_evict_info.kind_count() + log.info("by kind: %s", by_kind) + assert by_kind["Image"] > 0 + assert by_kind["Delta"] > 0 + assert by_kind["InMemory"] == 0 + resident_layers = list(env.timeline_dir(tenant_id, timeline_id).glob("*-*_*")) + log.info("resident layers count before eviction: %s", len(resident_layers)) + + log.info("evict all layers") + ps_http.evict_all_layers(tenant_id, timeline_id) + + def ensure_resident_and_remote_size_metrics(): + log.info("ensure that all the layers are gone") + resident_layers = list(env.timeline_dir(tenant_id, timeline_id).glob("*-*_*")) + # we have disabled all background loops, so, this should hold + assert len(resident_layers) == 0 + + info = ps_http.layer_map_info(tenant_id, timeline_id) + log.info("layer map dump: %s", info) + + log.info("ensure that resident_physical_size metric is zero") + resident_physical_size_metric = ps_http.get_timeline_metric( + tenant_id, timeline_id, "pageserver_resident_physical_size" + ) + assert resident_physical_size_metric == 0 + log.info("ensure that resident_physical_size metric corresponds to layer map dump") + assert resident_physical_size_metric == sum( + [layer.layer_file_size or 0 for layer in info.historic_layers if not layer.remote] + ) + + log.info("ensure that remote_physical_size metric matches layer map") + remote_physical_size_metric = ps_http.get_timeline_metric( + tenant_id, timeline_id, "pageserver_remote_physical_size" + ) + log.info("ensure that remote_physical_size metric corresponds to layer map dump") + assert remote_physical_size_metric == sum( + layer.layer_file_size or 0 for layer in info.historic_layers if layer.remote + ) + + log.info("before runnning GC, ensure that remote_physical size is zero") + ensure_resident_and_remote_size_metrics() + + log.info("run GC") + time.sleep(2) # let pitr_interval + 1 second pass + ps_http.timeline_gc(tenant_id, timeline_id, 0) + time.sleep(1) + assert not env.pageserver.log_contains("Nothing to GC") + + log.info("ensure GC deleted some layers, otherwise this test is pointless") + post_gc_info = ps_http.layer_map_info(tenant_id, timeline_id) + log.info("layer map dump: %s", post_gc_info) + log.info("by kind: %s", post_gc_info.kind_count()) + pre_evict_layers = set([layer.layer_file_name for layer in pre_evict_info.historic_layers]) + post_gc_layers = set([layer.layer_file_name for layer in post_gc_info.historic_layers]) + assert post_gc_layers.issubset(pre_evict_layers) + assert len(post_gc_layers) < len(pre_evict_layers) + + log.info("update_gc_info might download some layers. Evict them again.") + ps_http.evict_all_layers(tenant_id, timeline_id) + + log.info("after running GC, ensure that resident size is still zero") + ensure_resident_and_remote_size_metrics() diff --git a/test_runner/regress/test_metric_collection.py b/test_runner/regress/test_metric_collection.py index 3f252992f5..a33af9a3b2 100644 --- a/test_runner/regress/test_metric_collection.py +++ b/test_runner/regress/test_metric_collection.py @@ -9,7 +9,6 @@ from typing import Iterator import pytest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( PSQL, NeonEnvBuilder, @@ -143,7 +142,7 @@ def test_metric_collection( # Helper function that gets the number of given kind of remote ops from the metrics def get_num_remote_ops(file_kind: str, op_kind: str) -> int: - ps_metrics = parse_metrics(env.pageserver.http_client().get_metrics(), "pageserver") + ps_metrics = env.pageserver.http_client().get_metrics() total = 0.0 for sample in ps_metrics.query_all( name="pageserver_remote_operation_seconds_count", diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index 5ee94de32d..e6c580c37c 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -11,6 +11,7 @@ from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, PageserverApiException, + PageserverHttpClient, RemoteStorageKind, assert_tenant_status, available_remote_storages, @@ -25,9 +26,16 @@ from fixtures.types import Lsn from fixtures.utils import query_scalar -def get_num_downloaded_layers(client, tenant_id, timeline_id): +def get_num_downloaded_layers(client: PageserverHttpClient, tenant_id, timeline_id): value = client.get_metric_value( - f'pageserver_remote_operation_seconds_count{{file_kind="layer",op_kind="download",status="success",tenant_id="{tenant_id}",timeline_id="{timeline_id}"}}' + "pageserver_remote_operation_seconds_count", + { + "file_kind": "layer", + "op_kind": "download", + "status": "success", + "tenant_id": tenant_id, + "timeline_id": timeline_id, + }, ) if value is None: return 0 diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 6c3454b79b..ac1f7b2891 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -6,7 +6,6 @@ from threading import Thread import asyncpg import pytest from fixtures.log_helper import log -from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( NeonEnv, NeonEnvBuilder, @@ -79,7 +78,7 @@ def test_tenant_reattach( ".*failed to perform remote task UploadMetadata.*, will retry.*" ) - ps_metrics = parse_metrics(pageserver_http.get_metrics(), "pageserver") + ps_metrics = pageserver_http.get_metrics() tenant_metric_filter = { "tenant_id": str(tenant_id), "timeline_id": str(timeline_id), @@ -93,7 +92,7 @@ def test_tenant_reattach( time.sleep(1) # for metrics propagation - ps_metrics = parse_metrics(pageserver_http.get_metrics(), "pageserver") + ps_metrics = pageserver_http.get_metrics() pageserver_last_record_lsn = int( ps_metrics.query_one("pageserver_last_record_lsn", filter=tenant_metric_filter).value ) diff --git a/test_runner/regress/test_tenant_tasks.py b/test_runner/regress/test_tenant_tasks.py index 4eba4ce942..24b211e368 100644 --- a/test_runner/regress/test_tenant_tasks.py +++ b/test_runner/regress/test_tenant_tasks.py @@ -50,16 +50,22 @@ def test_tenant_tasks(neon_env_builder: NeonEnvBuilder): wait_until(10, 0.2, lambda: assert_active(tenant_id)) # Assert that all tasks finish quickly after tenant is detached - task_starts = client.get_metric_value('pageserver_tenant_task_events{event="start"}') + task_starts = client.get_metric_value("pageserver_tenant_task_events_total", {"event": "start"}) assert task_starts is not None assert int(task_starts) > 0 client.tenant_detach(tenant) client.tenant_detach(env.initial_tenant) def assert_tasks_finish(): - tasks_started = client.get_metric_value('pageserver_tenant_task_events{event="start"}') - tasks_ended = client.get_metric_value('pageserver_tenant_task_events{event="stop"}') - tasks_panicked = client.get_metric_value('pageserver_tenant_task_events{event="panic"}') + tasks_started = client.get_metric_value( + "pageserver_tenant_task_events_total", {"event": "start"} + ) + tasks_ended = client.get_metric_value( + "pageserver_tenant_task_events_total", {"event": "stop"} + ) + tasks_panicked = client.get_metric_value( + "pageserver_tenant_task_events_total", {"event": "panic"} + ) log.info(f"started {tasks_started}, ended {tasks_ended}, panicked {tasks_panicked}") assert tasks_started == tasks_ended assert tasks_panicked is None or int(tasks_panicked) == 0 diff --git a/test_runner/regress/test_tenants.py b/test_runner/regress/test_tenants.py index e56bb1b469..bf87cb3ad4 100644 --- a/test_runner/regress/test_tenants.py +++ b/test_runner/regress/test_tenants.py @@ -107,7 +107,7 @@ def test_metrics_normal_work(neon_env_builder: NeonEnvBuilder): assert cur.fetchone() == (5000050000,) collected_metrics = { - "pageserver": env.pageserver.http_client().get_metrics(), + "pageserver": env.pageserver.http_client().get_metrics_str(), } for sk in env.safekeepers: collected_metrics[f"safekeeper{sk.id}"] = sk.http_client().get_metrics_str() @@ -207,7 +207,7 @@ def test_pageserver_metrics_removed_after_detach( assert cur.fetchone() == (5000050000,) def get_ps_metric_samples_for_tenant(tenant_id: TenantId) -> List[Sample]: - ps_metrics = parse_metrics(env.pageserver.http_client().get_metrics(), "pageserver") + ps_metrics = env.pageserver.http_client().get_metrics() samples = [] for metric_name in ps_metrics.metrics: for sample in ps_metrics.query_all( @@ -307,7 +307,7 @@ def test_pageserver_with_empty_tenants( time.sleep(1) # to allow metrics propagation - ps_metrics = parse_metrics(client.get_metrics(), "pageserver") + ps_metrics = client.get_metrics() broken_tenants_metric_filter = { "tenant_id": str(tenant_without_timelines_dir), "state": "broken", diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index 3b41cc5c90..ea4b65c9a8 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -1,11 +1,11 @@ import math import queue import random -import re import threading import time from contextlib import closing from pathlib import Path +from typing import Optional import psycopg2.errors import psycopg2.extras @@ -19,9 +19,11 @@ from fixtures.neon_fixtures import ( PgBin, PortDistributor, Postgres, + RemoteStorageKind, VanillaPostgres, assert_tenant_status, wait_for_last_flush_lsn, + wait_for_upload_queue_empty, wait_until, ) from fixtures.types import TenantId, TimelineId @@ -302,8 +304,18 @@ def test_timeline_initial_logical_size_calculation_cancellation( # message emitted by the code behind failpoint "timeline-calculate-logical-size-check-dir-exists" -def test_timeline_physical_size_init(neon_simple_env: NeonEnv): - env = neon_simple_env +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_init( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + + env = neon_env_builder.init_start() + new_timeline_id = env.neon_cli.create_branch("test_timeline_physical_size_init") pg = env.postgres.create_start("test_timeline_physical_size_init") @@ -331,12 +343,22 @@ def test_timeline_physical_size_init(neon_simple_env: NeonEnv): ) assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) -def test_timeline_physical_size_post_checkpoint(neon_simple_env: NeonEnv): - env = neon_simple_env +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_post_checkpoint( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + + env = neon_env_builder.init_start() + pageserver_http = env.pageserver.http_client() new_timeline_id = env.neon_cli.create_branch("test_timeline_physical_size_post_checkpoint") pg = env.postgres.create_start("test_timeline_physical_size_post_checkpoint") @@ -354,11 +376,21 @@ def test_timeline_physical_size_post_checkpoint(neon_simple_env: NeonEnv): pageserver_http.timeline_checkpoint(env.initial_tenant, new_timeline_id) assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) -def test_timeline_physical_size_post_compaction(neon_env_builder: NeonEnvBuilder): +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_post_compaction( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + # Disable background compaction as we don't want it to happen after `get_physical_size` request # and before checking the expected size on disk, which makes the assertion failed neon_env_builder.pageserver_config_override = ( @@ -387,15 +419,33 @@ def test_timeline_physical_size_post_compaction(neon_env_builder: NeonEnvBuilder ) wait_for_last_flush_lsn(env, pg, env.initial_tenant, new_timeline_id) + + # shutdown safekeepers to prevent new data from coming in + for sk in env.safekeepers: + sk.stop() + pageserver_http.timeline_checkpoint(env.initial_tenant, new_timeline_id) pageserver_http.timeline_compact(env.initial_tenant, new_timeline_id) + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, env.initial_tenant, new_timeline_id) + assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) -def test_timeline_physical_size_post_gc(neon_env_builder: NeonEnvBuilder): +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_timeline_physical_size_post_gc( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): + + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + # Disable background compaction and GC as we don't want it to happen after `get_physical_size` request # and before checking the expected size on disk, which makes the assertion failed neon_env_builder.pageserver_config_override = "tenant_config={checkpoint_distance=100000, compaction_period='0s', gc_period='0s', pitr_interval='1s'}" @@ -431,8 +481,12 @@ def test_timeline_physical_size_post_gc(neon_env_builder: NeonEnvBuilder): pageserver_http.timeline_checkpoint(env.initial_tenant, new_timeline_id) pageserver_http.timeline_gc(env.initial_tenant, new_timeline_id, gc_horizon=None) + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, env.initial_tenant, new_timeline_id) + assert_physical_size_invariants( - get_physical_size_values(env, env.initial_tenant, new_timeline_id) + get_physical_size_values(env, env.initial_tenant, new_timeline_id, remote_storage_kind), + remote_storage_kind, ) @@ -465,26 +519,26 @@ def test_timeline_size_metrics( # get the metrics and parse the metric for the current timeline's physical size metrics = env.pageserver.http_client().get_metrics() - matches = re.search( - f'^pageserver_resident_physical_size{{tenant_id="{env.initial_tenant}",timeline_id="{new_timeline_id}"}} (\\S+)$', - metrics, - re.MULTILINE, - ) - assert matches - tl_physical_size_metric = int(matches.group(1)) + tl_physical_size_metric = metrics.query_one( + name="pageserver_resident_physical_size", + filter={ + "tenant_id": str(env.initial_tenant), + "timeline_id": str(new_timeline_id), + }, + ).value # assert that the physical size metric matches the actual physical size on disk timeline_path = env.timeline_dir(env.initial_tenant, new_timeline_id) assert tl_physical_size_metric == get_timeline_dir_size(timeline_path) # Check that the logical size metric is sane, and matches - matches = re.search( - f'^pageserver_current_logical_size{{tenant_id="{env.initial_tenant}",timeline_id="{new_timeline_id}"}} (\\S+)$', - metrics, - re.MULTILINE, - ) - assert matches - tl_logical_size_metric = int(matches.group(1)) + tl_logical_size_metric = metrics.query_one( + name="pageserver_current_logical_size", + filter={ + "tenant_id": str(env.initial_tenant), + "timeline_id": str(new_timeline_id), + }, + ).value pgdatadir = test_output_dir / "pgdata-vanilla" pg_bin = PgBin(test_output_dir, pg_distrib_dir, pg_version) @@ -516,18 +570,29 @@ def test_timeline_size_metrics( assert math.isclose(dbsize_sum, tl_logical_size_metric, abs_tol=2 * 1024 * 1024) -def test_tenant_physical_size(neon_simple_env: NeonEnv): +@pytest.mark.parametrize("remote_storage_kind", [None, RemoteStorageKind.LOCAL_FS]) +def test_tenant_physical_size( + neon_env_builder: NeonEnvBuilder, remote_storage_kind: Optional[RemoteStorageKind] +): random.seed(100) - env = neon_simple_env + if remote_storage_kind is not None: + neon_env_builder.enable_remote_storage( + remote_storage_kind, "test_timeline_physical_size_init" + ) + + env = neon_env_builder.init_start() + pageserver_http = env.pageserver.http_client() client = env.pageserver.http_client() tenant, timeline = env.neon_cli.create_tenant() + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, tenant, timeline) def get_timeline_resident_physical_size(timeline: TimelineId): - sizes = get_physical_size_values(env, tenant, timeline) - assert_physical_size_invariants(sizes) + sizes = get_physical_size_values(env, tenant, timeline, remote_storage_kind) + assert_physical_size_invariants(sizes, remote_storage_kind) return sizes.prometheus_resident_physical timeline_total_resident_physical_size = get_timeline_resident_physical_size(timeline) @@ -547,6 +612,9 @@ def test_tenant_physical_size(neon_simple_env: NeonEnv): wait_for_last_flush_lsn(env, pg, tenant, timeline) pageserver_http.timeline_checkpoint(tenant, timeline) + if remote_storage_kind is not None: + wait_for_upload_queue_empty(env.pageserver, tenant, timeline) + timeline_total_resident_physical_size += get_timeline_resident_physical_size(timeline) pg.stop() @@ -564,21 +632,39 @@ def test_tenant_physical_size(neon_simple_env: NeonEnv): class TimelinePhysicalSizeValues: api_current_physical: int - prometheus_resident_physical: int + prometheus_resident_physical: float + prometheus_remote_physical: Optional[float] = None python_timelinedir_layerfiles_physical: int + layer_map_file_size_sum: int def get_physical_size_values( - env: NeonEnv, tenant_id: TenantId, timeline_id: TimelineId + env: NeonEnv, + tenant_id: TenantId, + timeline_id: TimelineId, + remote_storage_kind: Optional[RemoteStorageKind], ) -> TimelinePhysicalSizeValues: res = TimelinePhysicalSizeValues() client = env.pageserver.http_client() - res.prometheus_resident_physical = client.get_timeline_metric( - tenant_id, timeline_id, "pageserver_resident_physical_size" + res.layer_map_file_size_sum = sum( + layer.layer_file_size or 0 + for layer in client.layer_map_info(tenant_id, timeline_id).historic_layers ) + metrics = client.get_metrics() + metrics_filter = {"tenant_id": str(tenant_id), "timeline_id": str(timeline_id)} + res.prometheus_resident_physical = metrics.query_one( + "pageserver_resident_physical_size", metrics_filter + ).value + if remote_storage_kind is not None: + res.prometheus_remote_physical = metrics.query_one( + "pageserver_remote_physical_size", metrics_filter + ).value + else: + res.prometheus_remote_physical = None + detail = client.timeline_detail( tenant_id, timeline_id, include_timeline_dir_layer_file_size_sum=True ) @@ -590,11 +676,20 @@ def get_physical_size_values( return res -def assert_physical_size_invariants(sizes: TimelinePhysicalSizeValues): +def assert_physical_size_invariants( + sizes: TimelinePhysicalSizeValues, remote_storage_kind: Optional[RemoteStorageKind] +): # resident phyiscal size is defined as assert sizes.python_timelinedir_layerfiles_physical == sizes.prometheus_resident_physical + assert sizes.python_timelinedir_layerfiles_physical == sizes.layer_map_file_size_sum + # we don't do layer eviction, so, all layers are resident assert sizes.api_current_physical == sizes.prometheus_resident_physical + if remote_storage_kind is not None: + assert sizes.prometheus_resident_physical == sizes.prometheus_remote_physical + # XXX would be nice to assert layer file physical storage utilization here as well, but we can only do that for LocalFS + else: + assert sizes.prometheus_remote_physical is None # Timeline logical size initialization is an asynchronous background task that runs once, diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 9e3b0ec02f..0ac9127c6b 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -775,6 +775,9 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): if not auth_enabled: wa_http_cli = wa.http_client() wa_http_cli.check_status() + + wa_http_cli_debug = wa.http_client() + wa_http_cli_debug.check_status() else: wa_http_cli = wa.http_client(auth_token=env.auth_keys.generate_tenant_token(tenant_id)) wa_http_cli.check_status() @@ -785,6 +788,10 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): wa_http_cli_noauth = wa.http_client() wa_http_cli_noauth.check_status() + # debug endpoint requires safekeeper scope + wa_http_cli_debug = wa.http_client(auth_token=env.auth_keys.generate_safekeeper_token()) + wa_http_cli_debug.check_status() + # fetch something sensible from status tli_status = wa_http_cli.timeline_status(tenant_id, timeline_id) epoch = tli_status.acceptor_epoch @@ -795,6 +802,12 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): with pytest.raises(cli.HTTPError, match="Forbidden|Unauthorized"): cli.timeline_status(tenant_id, timeline_id) + # fetch debug_dump endpoint + debug_dump_0 = wa_http_cli_debug.debug_dump({"dump_all": "true"}) + log.info(f"debug_dump before reboot {debug_dump_0}") + assert debug_dump_0["timelines_count"] == 1 + assert debug_dump_0["timelines"][0]["timeline_id"] == str(timeline_id) + pg.safe_psql("create table t(i int)") # ensure epoch goes up after reboot @@ -808,6 +821,25 @@ def test_timeline_status(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): # and timeline_start_lsn stays the same assert tli_status.timeline_start_lsn == timeline_start_lsn + # fetch debug_dump after reboot + debug_dump_1 = wa_http_cli_debug.debug_dump({"dump_all": "true"}) + log.info(f"debug_dump after reboot {debug_dump_1}") + assert debug_dump_1["timelines_count"] == 1 + assert debug_dump_1["timelines"][0]["timeline_id"] == str(timeline_id) + + # check that commit_lsn and flush_lsn not decreased + assert ( + debug_dump_1["timelines"][0]["memory"]["mem_state"]["commit_lsn"] + >= debug_dump_0["timelines"][0]["memory"]["mem_state"]["commit_lsn"] + ) + assert ( + debug_dump_1["timelines"][0]["memory"]["flush_lsn"] + >= debug_dump_0["timelines"][0]["memory"]["flush_lsn"] + ) + + # check .config in response + assert debug_dump_1["config"]["id"] == env.safekeepers[0].id + class SafekeeperEnv: def __init__( diff --git a/vm-cgconfig.conf b/vm-cgconfig.conf deleted file mode 100644 index a2e201708e..0000000000 --- a/vm-cgconfig.conf +++ /dev/null @@ -1,12 +0,0 @@ -# Configuration for cgroups in VM compute nodes -group neon-postgres { - perm { - admin { - uid = vm-informant; - } - task { - gid = users; - } - } - memory {} -} diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index c0cf3c5611..bd21095fff 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -21,7 +21,6 @@ crossbeam-utils = { version = "0.8" } either = { version = "1" } fail = { version = "0.5", default-features = false, features = ["failpoints"] } futures = { version = "0.3" } -futures-channel = { version = "0.3", features = ["sink"] } futures-executor = { version = "0.3" } futures-util = { version = "0.3", features = ["channel", "io", "sink"] } hashbrown = { version = "0.12", features = ["raw"] }