lints

.gitmodules

@@ -1,16 +1,16 @@
 [submodule "vendor/postgres-v14"]
 	path = vendor/postgres-v14
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_14_STABLE_neon
 [submodule "vendor/postgres-v15"]
 	path = vendor/postgres-v15
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_15_STABLE_neon
 [submodule "vendor/postgres-v16"]
 	path = vendor/postgres-v16
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_16_STABLE_neon
 [submodule "vendor/postgres-v17"]
 	path = vendor/postgres-v17
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_17_STABLE_neon

Cargo.lock

@@ -1330,7 +1330,6 @@ dependencies = [
  "chrono",
  "clap",
  "compute_api",
- "fail",
  "flate2",
  "futures",
  "hostname-validator",
@@ -1339,6 +1338,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonwebtoken",
  "metrics",
+ "neon_failpoint",
  "nix 0.30.1",
  "notify",
  "num_cpus",
@@ -2891,13 +2891,13 @@ dependencies = [
  "arc-swap",
  "bytes",
  "camino",
- "fail",
  "futures",
  "hyper 0.14.30",
  "itertools 0.10.5",
  "jemalloc_pprof",
  "jsonwebtoken",
  "metrics",
+ "neon_failpoint",
  "once_cell",
  "pprof",
  "regex",
@@ -3852,6 +3852,23 @@ dependencies = [
  "workspace_hack",
 ]
 
+[[package]]
+name = "neon_failpoint"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "either",
+ "once_cell",
+ "parking_lot 0.12.1",
+ "rand 0.8.5",
+ "regex",
+ "serde",
+ "tokio",
+ "tokio-util",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "never-say-never"
 version = "6.6.666"
@@ -4357,7 +4374,6 @@ dependencies = [
  "either",
  "enum-map",
  "enumset",
- "fail",
  "futures",
  "hashlink",
  "hex",
@@ -4372,6 +4388,7 @@ dependencies = [
  "jsonwebtoken",
  "md5",
  "metrics",
+ "neon_failpoint",
  "nix 0.30.1",
  "num-traits",
  "num_cpus",
@@ -5289,7 +5306,6 @@ dependencies = [
  "async-trait",
  "atomic-take",
  "aws-config",
- "aws-credential-types",
  "aws-sdk-iam",
  "aws-sigv4",
  "base64 0.22.1",
@@ -5329,7 +5345,6 @@ dependencies = [
  "itoa",
  "jose-jwa",
  "jose-jwk",
- "json",
  "lasso",
  "measured",
  "metrics",
@@ -6193,7 +6208,6 @@ dependencies = [
  "criterion",
  "desim",
  "env_logger",
- "fail",
  "futures",
  "hex",
  "http 1.1.0",
@@ -6203,6 +6217,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonwebtoken",
  "metrics",
+ "neon_failpoint",
  "once_cell",
  "pageserver_api",
  "parking_lot 0.12.1",
@@ -6889,7 +6904,6 @@ dependencies = [
  "diesel",
  "diesel-async",
  "diesel_migrations",
- "fail",
  "futures",
  "governor",
  "hex",
@@ -6902,6 +6916,7 @@ dependencies = [
  "lasso",
  "measured",
  "metrics",
+ "neon_failpoint",
  "once_cell",
  "pageserver_api",
  "pageserver_client",
@@ -6993,7 +7008,6 @@ dependencies = [
  "pageserver_api",
  "pageserver_client",
  "reqwest",
- "safekeeper_api",
  "serde_json",
  "storage_controller_client",
  "tokio",
@@ -8167,7 +8181,7 @@ dependencies = [
  "const_format",
  "criterion",
  "diatomic-waker",
- "fail",
+ "either",
  "futures",
  "git-version",
  "hex",
@@ -8175,6 +8189,7 @@ dependencies = [
  "humantime",
  "jsonwebtoken",
  "metrics",
+ "neon_failpoint",
  "nix 0.30.1",
  "once_cell",
  "pem",

Cargo.toml

@@ -21,6 +21,7 @@ members = [
     "workspace_hack",
     "libs/compute_api",
     "libs/http-utils",
+    "libs/neon_failpoint",
     "libs/pageserver_api",
     "libs/postgres_ffi",
     "libs/postgres_ffi_types",
@@ -97,7 +98,6 @@ diatomic-waker = { version = "0.2.3" }
 either = "1.8"
 enum-map = "2.4.2"
 enumset = "1.0.12"
-fail = "0.5.0"
 fallible-iterator = "0.2"
 framed-websockets = { version = "0.1.0", git = "https://github.com/neondatabase/framed-websockets" }
 futures = "0.3"
@@ -258,6 +258,7 @@ desim = { version = "0.1", path = "./libs/desim" }
 endpoint_storage = { version = "0.0.1", path = "./endpoint_storage/" }
 http-utils = { version = "0.1", path = "./libs/http-utils/" }
 metrics = { version = "0.1", path = "./libs/metrics/" }
+neon_failpoint = { version = "0.1", path = "./libs/neon_failpoint/" }
 neon-shmem = { version = "0.1", path = "./libs/neon-shmem/" }
 pageserver = { path = "./pageserver" }
 pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }

compute_tools/Cargo.toml

@@ -7,7 +7,7 @@ license.workspace = true
 [features]
 default = []
 # Enables test specific features.
-testing = ["fail/failpoints"]
+testing = ["neon_failpoint/testing"]
 
 [dependencies]
 async-compression.workspace = true
@@ -23,7 +23,7 @@ camino.workspace = true
 chrono.workspace = true
 cfg-if.workspace = true
 clap.workspace = true
-fail.workspace = true
+neon_failpoint.workspace = true
 flate2.workspace = true
 futures.workspace = true
 http.workspace = true

compute_tools/src/bin/compute_ctl.rs

@@ -154,7 +154,7 @@ impl Cli {
 fn main() -> Result<()> {
     let cli = Cli::parse();
 
-    let scenario = failpoint_support::init();
+    failpoint_support::init().unwrap();
 
     // For historical reasons, the main thread that processes the config and launches postgres
     // is synchronous, but we always have this tokio runtime available and we "enter" it so
@@ -201,8 +201,6 @@ fn main() -> Result<()> {
 
     let exit_code = compute_node.run()?;
 
-    scenario.teardown();
-
     deinit_and_exit(exit_code);
 }
 

compute_tools/src/compute.rs

@@ -1040,8 +1040,6 @@ impl ComputeNode {
             PageserverProtocol::Grpc => self.try_get_basebackup_grpc(spec, lsn)?,
         };
 
-        self.fix_zenith_signal_neon_signal()?;
-
         let mut state = self.state.lock().unwrap();
         state.metrics.pageserver_connect_micros =
             connected.duration_since(started).as_micros() as u64;
@@ -1051,27 +1049,6 @@ impl ComputeNode {
         Ok(())
     }
 
-    /// Move the Zenith signal file to Neon signal file location.
-    /// This makes Compute compatible with older PageServers that don't yet
-    /// know about the Zenith->Neon rename.
-    fn fix_zenith_signal_neon_signal(&self) -> Result<()> {
-        let datadir = Path::new(&self.params.pgdata);
-
-        let neonsig = datadir.join("neon.signal");
-
-        if neonsig.is_file() {
-            return Ok(());
-        }
-
-        let zenithsig = datadir.join("zenith.signal");
-
-        if zenithsig.is_file() {
-            fs::copy(zenithsig, neonsig)?;
-        }
-
-        Ok(())
-    }
-
     /// Fetches a basebackup via gRPC. The connstring must use grpc://. Returns the timestamp when
     /// the connection was established, and the (compressed) size of the basebackup.
     fn try_get_basebackup_grpc(&self, spec: &ParsedSpec, lsn: Lsn) -> Result<(Instant, usize)> {

compute_tools/src/http/routes/failpoints.rs

@@ -1,8 +1,9 @@
 use axum::response::{IntoResponse, Response};
 use http::StatusCode;
+use neon_failpoint::{configure_failpoint, configure_failpoint_with_context};
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use tracing::info;
-use utils::failpoint_support::apply_failpoint;
 
 pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;
 
@@ -11,10 +12,16 @@ pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;
 pub struct FailpointConfig {
     /// Name of the fail point
     pub name: String,
-    /// List of actions to take, using the format described in `fail::cfg`
+    /// List of actions to take, using the format described in neon_failpoint
     ///
-    /// We also support `actions = "exit"` to cause the fail point to immediately exit.
+    /// We support actions: "pause", "sleep(N)", "return", "return(value)", "exit", "off", "panic(message)"
+    /// Plus probability-based actions: "N%return(value)", "N%M*return(value)", "N%action", "N%M*action"
     pub actions: String,
+    /// Optional context matching rules for conditional failpoints
+    /// Each key-value pair specifies a context key and a regex pattern to match against
+    /// All context matchers must match for the failpoint to trigger
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub context_matchers: Option<HashMap<String, String>>,
 }
 
 use crate::http::JsonResponse;
@@ -24,7 +31,7 @@ use crate::http::extract::Json;
 pub(in crate::http) async fn configure_failpoints(
     failpoints: Json<ConfigureFailpointsRequest>,
 ) -> Response {
-    if !fail::has_failpoints() {
+    if !neon_failpoint::has_failpoints() {
         return JsonResponse::error(
             StatusCode::PRECONDITION_FAILED,
             "Cannot manage failpoints because neon was compiled without failpoints support",
@@ -32,16 +39,21 @@ pub(in crate::http) async fn configure_failpoints(
     }
 
     for fp in &*failpoints {
-        info!("cfg failpoint: {} {}", fp.name, fp.actions);
+        info!(
+            "cfg failpoint: {} {} (context: {:?})",
+            fp.name, fp.actions, fp.context_matchers
+        );
 
-        // We recognize one extra "action" that's not natively recognized
-        // by the failpoints crate: exit, to immediately kill the process
-        let cfg_result = apply_failpoint(&fp.name, &fp.actions);
+        let cfg_result = if let Some(context_matchers) = fp.context_matchers.clone() {
+            configure_failpoint_with_context(&fp.name, &fp.actions, context_matchers)
+        } else {
+            configure_failpoint(&fp.name, &fp.actions)
+        };
 
         if let Err(e) = cfg_result {
             return JsonResponse::error(
                 StatusCode::BAD_REQUEST,
-                format!("failed to configure failpoints: {e}"),
+                format!("failed to configure failpoint '{}': {e}", fp.name),
             );
         }
     }

compute_tools/src/migration.rs

@@ -1,5 +1,5 @@
 use anyhow::{Context, Result};
-use fail::fail_point;
+use neon_failpoint::fail_point;
 use tokio_postgres::{Client, Transaction};
 use tracing::{error, info};
 
@@ -40,13 +40,14 @@ impl<'m> MigrationRunner<'m> {
         // middle of applying a series of migrations fails in an expected
         // manner
         if cfg!(feature = "testing") {
-            let fail = (|| {
-                fail_point!("compute-migration", |fail_migration_id| {
+            let fail = async {
+                fail_point!("compute-migration", |fail_migration_id: Option<String>| {
                     migration_id == fail_migration_id.unwrap().parse::<i64>().unwrap()
                 });
 
                 false
-            })();
+            }
+            .await;
 
             if fail {
                 return Err(anyhow::anyhow!(format!(

control_plane/src/broker.rs

@@ -36,7 +36,7 @@ impl StorageBroker {
     pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> {
         let broker = &self.env.broker;
 
-        println!("Starting neon broker at {}", broker.client_url());
+        print!("Starting neon broker at {}", broker.client_url());
 
         let mut args = Vec::new();
 

control_plane/src/endpoint.rs

@@ -32,8 +32,7 @@
 //!     config.json                 - passed to `compute_ctl`
 //!     pgdata/
 //!         postgresql.conf       - copy of postgresql.conf created by `compute_ctl`
-//!         neon.signal
-//!         zenith.signal         - copy of neon.signal, for backward compatibility
+//!         zenith.signal
 //!         <other PostgreSQL files>
 //! ```
 //!

control_plane/src/local_env.rs

@@ -217,9 +217,6 @@ pub struct NeonStorageControllerConf {
     pub posthog_config: Option<PostHogConfig>,
 
     pub kick_secondary_downloads: Option<bool>,
-
-    #[serde(with = "humantime_serde")]
-    pub shard_split_request_timeout: Option<Duration>,
 }
 
 impl NeonStorageControllerConf {
@@ -253,7 +250,6 @@ impl Default for NeonStorageControllerConf {
             timeline_safekeeper_count: None,
             posthog_config: None,
             kick_secondary_downloads: None,
-            shard_split_request_timeout: None,
         }
     }
 }

control_plane/src/pageserver.rs

@@ -303,7 +303,7 @@ impl PageServerNode {
     async fn start_node(&self, retry_timeout: &Duration) -> anyhow::Result<()> {
         // TODO: using a thread here because start_process() is not async but we need to call check_status()
         let datadir = self.repo_path();
-        println!(
+        print!(
             "Starting pageserver node {} at '{}' in {:?}, retrying for {:?}",
             self.conf.id,
             self.pg_connection_config.raw_address(),

control_plane/src/safekeeper.rs

@@ -127,7 +127,7 @@ impl SafekeeperNode {
         extra_opts: &[String],
         retry_timeout: &Duration,
     ) -> anyhow::Result<()> {
-        println!(
+        print!(
             "Starting safekeeper at '{}' in '{}', retrying for {:?}",
             self.pg_connection_config.raw_address(),
             self.datadir_path().display(),

control_plane/src/storage_controller.rs

@@ -648,13 +648,6 @@ impl StorageController {
             args.push(format!("--timeline-safekeeper-count={sk_cnt}"));
         }
 
-        if let Some(duration) = self.config.shard_split_request_timeout {
-            args.push(format!(
-                "--shard-split-request-timeout={}",
-                humantime::Duration::from(duration)
-            ));
-        }
-
         let mut envs = vec![
             ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
             ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
@@ -667,7 +660,7 @@ impl StorageController {
             ));
         }
 
-        println!("Starting storage controller at {scheme}://{host}:{listen_port}");
+        println!("Starting storage controller");
 
         background_process::start_process(
             COMMAND,

control_plane/storcon_cli/Cargo.toml

@@ -14,7 +14,6 @@ humantime.workspace = true
 pageserver_api.workspace = true
 pageserver_client.workspace = true
 reqwest.workspace = true
-safekeeper_api.workspace=true
 serde_json = { workspace = true, features = ["raw_value"] }
 storage_controller_client.workspace = true
 tokio.workspace = true

control_plane/storcon_cli/src/main.rs

@@ -11,7 +11,7 @@ use pageserver_api::controller_api::{
     PlacementPolicy, SafekeeperDescribeResponse, SafekeeperSchedulingPolicyRequest,
     ShardSchedulingPolicy, ShardsPreferredAzsRequest, ShardsPreferredAzsResponse,
     SkSchedulingPolicy, TenantCreateRequest, TenantDescribeResponse, TenantPolicyRequest,
-    TenantShardMigrateRequest, TenantShardMigrateResponse, TimelineSafekeeperMigrateRequest,
+    TenantShardMigrateRequest, TenantShardMigrateResponse,
 };
 use pageserver_api::models::{
     EvictionPolicy, EvictionPolicyLayerAccessThreshold, ShardParameters, TenantConfig,
@@ -21,7 +21,6 @@ use pageserver_api::models::{
 use pageserver_api::shard::{ShardStripeSize, TenantShardId};
 use pageserver_client::mgmt_api::{self};
 use reqwest::{Certificate, Method, StatusCode, Url};
-use safekeeper_api::models::TimelineLocateResponse;
 use storage_controller_client::control_api::Client;
 use utils::id::{NodeId, TenantId, TimelineId};
 
@@ -280,23 +279,6 @@ enum Command {
         #[arg(long)]
         concurrency: Option<usize>,
     },
-    /// Locate safekeepers for a timeline from the storcon DB.
-    TimelineLocate {
-        #[arg(long)]
-        tenant_id: TenantId,
-        #[arg(long)]
-        timeline_id: TimelineId,
-    },
-    /// Migrate a timeline to a new set of safekeepers
-    TimelineSafekeeperMigrate {
-        #[arg(long)]
-        tenant_id: TenantId,
-        #[arg(long)]
-        timeline_id: TimelineId,
-        /// Example: --new-sk-set 1,2,3
-        #[arg(long, required = true, value_delimiter = ',')]
-        new_sk_set: Vec<NodeId>,
-    },
 }
 
 #[derive(Parser)]
@@ -1342,7 +1324,7 @@ async fn main() -> anyhow::Result<()> {
             concurrency,
         } => {
             let mut path = format!(
-                "v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/download_heatmap_layers",
+                "/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/download_heatmap_layers",
             );
 
             if let Some(c) = concurrency {
@@ -1353,41 +1335,6 @@ async fn main() -> anyhow::Result<()> {
                 .dispatch::<(), ()>(Method::POST, path, None)
                 .await?;
         }
-        Command::TimelineLocate {
-            tenant_id,
-            timeline_id,
-        } => {
-            let path = format!("debug/v1/tenant/{tenant_id}/timeline/{timeline_id}/locate");
-
-            let resp = storcon_client
-                .dispatch::<(), TimelineLocateResponse>(Method::GET, path, None)
-                .await?;
-
-            let sk_set = resp.sk_set.iter().map(|id| id.0 as i64).collect::<Vec<_>>();
-            let new_sk_set = resp
-                .new_sk_set
-                .as_ref()
-                .map(|ids| ids.iter().map(|id| id.0 as i64).collect::<Vec<_>>());
-
-            println!("generation = {}", resp.generation);
-            println!("sk_set = {sk_set:?}");
-            println!("new_sk_set = {new_sk_set:?}");
-        }
-        Command::TimelineSafekeeperMigrate {
-            tenant_id,
-            timeline_id,
-            new_sk_set,
-        } => {
-            let path = format!("v1/tenant/{tenant_id}/timeline/{timeline_id}/safekeeper_migrate");
-
-            storcon_client
-                .dispatch::<_, ()>(
-                    Method::POST,
-                    path,
-                    Some(TimelineSafekeeperMigrateRequest { new_sk_set }),
-                )
-                .await?;
-        }
     }
 
     Ok(())

docs/core_changes.md

@@ -129,10 +129,9 @@ segment to bootstrap the WAL writing, but it doesn't contain the checkpoint reco
 changes in xlog.c, to allow starting the compute node without reading the last checkpoint record
 from WAL.
 
-This includes code to read the `neon.signal` (also `zenith.signal`) file, which tells the startup 
-code the LSN to start at. When the `neon.signal` file is present, the startup uses that LSN
-instead of the last checkpoint's LSN. The system is known to be consistent at that LSN, without 
-any WAL redo.
+This includes code to read the `zenith.signal` file, which tells the startup code the LSN to start
+at. When the `zenith.signal` file is present, the startup uses that LSN instead of the last
+checkpoint's LSN. The system is known to be consistent at that LSN, without any WAL redo.
 
 
 ### How to get rid of the patch

libs/http-utils/Cargo.toml

@@ -9,7 +9,7 @@ anyhow.workspace = true
 arc-swap.workspace = true
 bytes.workspace = true
 camino.workspace = true
-fail.workspace = true
+neon_failpoint.workspace = true
 futures.workspace = true
 hyper0.workspace = true
 itertools.workspace = true

libs/http-utils/src/failpoints.rs

@@ -1,7 +1,8 @@
 use hyper::{Body, Request, Response, StatusCode};
+use neon_failpoint::{configure_failpoint, configure_failpoint_with_context, has_failpoints};
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use tokio_util::sync::CancellationToken;
-use utils::failpoint_support::apply_failpoint;
 
 use crate::error::ApiError;
 use crate::json::{json_request, json_response};
@@ -13,10 +14,16 @@ pub type ConfigureFailpointsRequest = Vec<FailpointConfig>;
 pub struct FailpointConfig {
     /// Name of the fail point
     pub name: String,
-    /// List of actions to take, using the format described in `fail::cfg`
+    /// List of actions to take, using the format described in neon_failpoint
     ///
-    /// We also support `actions = "exit"` to cause the fail point to immediately exit.
+    /// We support actions: "pause", "sleep(N)", "return", "return(value)", "exit", "off", "panic(message)"
+    /// Plus probability-based actions: "N%return(value)", "N%M*return(value)", "N%action", "N%M*action"
     pub actions: String,
+    /// Optional context matching rules for conditional failpoints
+    /// Each key-value pair specifies a context key and a regex pattern to match against
+    /// All context matchers must match for the failpoint to trigger
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub context_matchers: Option<HashMap<String, String>>,
 }
 
 /// Configure failpoints through http.
@@ -24,7 +31,7 @@ pub async fn failpoints_handler(
     mut request: Request<Body>,
     _cancel: CancellationToken,
 ) -> Result<Response<Body>, ApiError> {
-    if !fail::has_failpoints() {
+    if !has_failpoints() {
         return Err(ApiError::BadRequest(anyhow::anyhow!(
             "Cannot manage failpoints because neon was compiled without failpoints support"
         )));
@@ -32,15 +39,24 @@ pub async fn failpoints_handler(
 
     let failpoints: ConfigureFailpointsRequest = json_request(&mut request).await?;
     for fp in failpoints {
-        tracing::info!("cfg failpoint: {} {}", fp.name, fp.actions);
+        tracing::info!(
+            "cfg failpoint: {} {} (context: {:?})",
+            fp.name,
+            fp.actions,
+            fp.context_matchers
+        );
 
-        // We recognize one extra "action" that's not natively recognized
-        // by the failpoints crate: exit, to immediately kill the process
-        let cfg_result = apply_failpoint(&fp.name, &fp.actions);
+        let cfg_result = if let Some(context_matchers) = fp.context_matchers {
+            configure_failpoint_with_context(&fp.name, &fp.actions, context_matchers)
+        } else {
+            configure_failpoint(&fp.name, &fp.actions)
+        };
 
-        if let Err(err_msg) = cfg_result {
+        if let Err(err) = cfg_result {
             return Err(ApiError::BadRequest(anyhow::anyhow!(
-                "Failed to configure failpoints: {err_msg}"
+                "Failed to configure failpoint '{}': {}",
+                fp.name,
+                err
             )));
         }
     }

libs/neon_failpoint/Cargo.toml

@@ -0,0 +1,27 @@
+[package]
+name = "neon_failpoint"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+tokio = { workspace = true, features = ["time", "sync", "rt-multi-thread"] }
+tokio-util = { workspace = true }
+tracing = { workspace = true }
+serde = { workspace = true, features = ["derive"] }
+anyhow = { workspace = true }
+regex = { workspace = true }
+once_cell = { workspace = true }
+parking_lot = { workspace = true }
+rand = { workspace = true }
+either = { workspace = true }
+
+[dev-dependencies]
+tracing-subscriber = { workspace = true, features = ["fmt"] }
+
+[features]
+default = []
+testing = []
+
+[[example]]
+name = "context_demo"
+required-features = ["testing"]

libs/neon_failpoint/README.md

@@ -0,0 +1,460 @@
+# Neon Failpoint Library
+
+A modern, async-first failpoint library for Neon, replacing the `fail` crate with enhanced functionality.
+
+## Features
+
+- **Async-first**: All failpoint operations are async and don't require `spawn_blocking`
+- **Context matching**: Failpoints can be configured to trigger only when specific context conditions are met
+- **Regex support**: Context values can be matched using regular expressions
+- **Cancellation support**: All operations support cancellation tokens
+- **Dynamic reconfiguration**: Paused and sleeping tasks automatically resume when failpoint configurations change
+- **Backward compatibility**: Drop-in replacement for existing `fail` crate usage
+
+## Supported Actions
+
+- `off` - Disable the failpoint
+- `pause` - Pause indefinitely until disabled, reconfigured, or cancelled
+- `sleep(N)` - Sleep for N milliseconds (can be interrupted by reconfiguration)
+- `return` - Return early (empty value)
+- `return(value)` - Return early with a specific value
+- `exit` - Exit the process immediately
+- `panic(message)` - Panic the process with a custom message
+- `N%return(value)` - Return with a specific value N% of the time (probability-based)
+- `N%M*return(value)` - Return with a specific value N% of the time, maximum M times
+- `N%action` - Execute any action N% of the time (probability-based)
+- `N%M*action` - Execute any action N% of the time, maximum M times
+
+## Probability-Based Actions
+
+The library supports probability-based failpoints that trigger only a percentage of the time:
+
+```rust
+// 50% chance to return a value
+configure_failpoint("random_failure", "50%return(error)").unwrap();
+
+// 10% chance to sleep, maximum 3 times
+configure_failpoint("occasional_delay", "10%3*sleep(1000)").unwrap();
+
+// 25% chance to panic
+configure_failpoint("rare_panic", "25%panic(critical error)").unwrap();
+```
+
+The probability system uses a counter to track how many times a probability-based action has been triggered, allowing for precise control over test scenarios.
+
+## Dynamic Behavior
+
+When a failpoint is reconfigured while tasks are waiting on it:
+
+- **Paused tasks** will immediately resume and continue normal execution
+- **Sleeping tasks** will wake up early and continue normal execution
+- **Removed failpoints** will cause all waiting tasks to resume normally
+
+The new configuration only applies to future hits of the failpoint, not to tasks that are already waiting. This allows for flexible testing scenarios where you can pause execution, inspect state, and then resume execution dynamically.
+
+## Example: Dynamic Reconfiguration
+
+```rust
+use neon_failpoint::{configure_failpoint, failpoint, FailpointResult};
+use tokio::time::Duration;
+
+// Start a task that will hit a failpoint
+let task = tokio::spawn(async {
+    println!("About to hit failpoint");
+    match failpoint("test_pause", None).await {
+        FailpointResult::Return(value) => println!("Returned: {}", value),
+        FailpointResult::Continue => println!("Continued normally"),
+        FailpointResult::Cancelled => println!("Cancelled"),
+    }
+});
+
+// Configure the failpoint to pause
+configure_failpoint("test_pause", "pause").unwrap();
+
+// Let the task hit the failpoint and pause
+tokio::time::sleep(Duration::from_millis(10)).await;
+
+// Change the failpoint configuration - this will wake up the paused task
+// The task will resume and continue normally (not apply the new config)
+configure_failpoint("test_pause", "return(not_applied)").unwrap();
+
+// The task will complete with Continue, not Return
+let result = task.await.unwrap();
+```
+
+## Basic Usage
+
+```rust
+use neon_failpoint::{configure_failpoint, failpoint, FailpointResult};
+
+// Configure a failpoint
+configure_failpoint("my_failpoint", "return(42)").unwrap();
+
+// Use the failpoint
+match failpoint("my_failpoint", None).await {
+    FailpointResult::Return(value) => {
+        println!("Failpoint returned: {}", value);
+        return value.parse().unwrap_or_default();
+    }
+    FailpointResult::Continue => {
+        // Continue normal execution
+    }
+    FailpointResult::Cancelled => {
+        // Handle cancellation
+    }
+}
+```
+
+## Context-Based Failpoint Configuration
+
+Context allows you to create **conditional failpoints** that only trigger when specific runtime conditions are met. This is particularly useful for testing scenarios where you want to inject failures only for specific tenants, operations, or other contextual conditions.
+
+### Configuring Context-Based Failpoints
+
+Use `configure_failpoint_with_context()` to set up failpoints with context matching:
+
+```rust
+use neon_failpoint::configure_failpoint_with_context;
+use std::collections::HashMap;
+
+let mut context_matchers = HashMap::new();
+context_matchers.insert("tenant_id".to_string(), "test_.*".to_string());
+context_matchers.insert("operation".to_string(), "backup".to_string());
+
+configure_failpoint_with_context(
+    "backup_operation",           // failpoint name
+    "return(simulated_failure)",  // action to take
+    context_matchers             // context matching rules
+).unwrap();
+```
+
+### Context Matching Rules
+
+The context matching system works as follows:
+
+1. **Key-Value Matching**: Each entry in `context_matchers` specifies a key that must exist in the runtime context
+2. **Regex Support**: Values in `context_matchers` are treated as regular expressions first
+3. **Fallback to Exact Match**: If the regex compilation fails, it falls back to exact string matching
+4. **ALL Must Match**: All context matchers must match for the failpoint to trigger
+
+### Runtime Context Usage
+
+When code hits a failpoint, it provides context using a `HashMap<String, String>`:
+
+```rust
+use neon_failpoint::{failpoint, FailpointResult};
+use std::collections::HashMap;
+
+let mut context = HashMap::new();
+context.insert("tenant_id".to_string(), "test_123".to_string());
+context.insert("operation".to_string(), "backup".to_string());
+context.insert("user_id".to_string(), "user_456".to_string());
+
+match failpoint("backup_operation", Some(&context)) {
+    either::Either::Left(result) => {
+        match result {
+            FailpointResult::Return(value) => {
+                // This will only trigger if ALL context matchers match
+                println!("Backup failed: {}", value);
+            }
+            FailpointResult::Continue => {
+                // Continue with normal backup operation
+            }
+            FailpointResult::Cancelled => {}
+        }
+    }
+    either::Either::Right(future) => {
+        match future.await {
+            FailpointResult::Return(value) => {
+                // This will only trigger if ALL context matchers match
+                println!("Backup failed: {}", value);
+            }
+            FailpointResult::Continue => {
+                // Continue with normal backup operation
+            }
+            FailpointResult::Cancelled => {}
+        }
+    }
+}
+```
+
+### Context Matching Examples
+
+#### Regex Matching
+```rust
+// Configure to match test tenants only
+let mut matchers = HashMap::new();
+matchers.insert("tenant_id".to_string(), "test_.*".to_string());
+
+configure_failpoint_with_context("test_failpoint", "pause", matchers).unwrap();
+
+// This will match
+let mut context = HashMap::new();
+context.insert("tenant_id".to_string(), "test_123".to_string());
+// This will NOT match
+let mut context = HashMap::new();
+context.insert("tenant_id".to_string(), "prod_123".to_string());
+```
+
+#### Multiple Conditions
+```rust
+// Must match BOTH tenant pattern AND operation
+let mut matchers = HashMap::new();
+matchers.insert("tenant_id".to_string(), "test_.*".to_string());
+matchers.insert("operation".to_string(), "backup".to_string());
+
+configure_failpoint_with_context("backup_test", "return(failed)", matchers).unwrap();
+
+// This will match (both conditions met)
+let mut context = HashMap::new();
+context.insert("tenant_id".to_string(), "test_123".to_string());
+context.insert("operation".to_string(), "backup".to_string());
+
+// This will NOT match (missing operation)
+let mut context = HashMap::new();
+context.insert("tenant_id".to_string(), "test_123".to_string());
+context.insert("operation".to_string(), "restore".to_string());
+```
+
+#### Exact String Matching
+```rust
+// If regex compilation fails, falls back to exact match
+let mut matchers = HashMap::new();
+matchers.insert("env".to_string(), "staging".to_string());
+
+configure_failpoint_with_context("env_specific", "sleep(1000)", matchers).unwrap();
+
+// This will match
+let mut context = HashMap::new();
+context.insert("env".to_string(), "staging".to_string());
+// This will NOT match
+let mut context = HashMap::new();
+context.insert("env".to_string(), "production".to_string());
+```
+
+### Benefits of Context-Based Failpoints
+
+1. **Selective Testing**: Only inject failures for specific tenants, environments, or operations
+2. **Production Safety**: Avoid accidentally triggering failpoints in production by using context filters
+3. **Complex Scenarios**: Test interactions between different components with targeted failures
+4. **Debugging**: Isolate issues to specific contexts without affecting the entire system
+
+### Context vs. Non-Context Failpoints
+
+- **Without context**: `configure_failpoint("name", "action")` - triggers for ALL hits
+- **With context**: `configure_failpoint_with_context("name", "action", matchers)` - triggers only when context matches
+
+## Context-Specific Failpoints
+
+```rust
+use neon_failpoint::{configure_failpoint_with_context, failpoint};
+use std::collections::HashMap;
+
+// Configure a failpoint that only triggers for specific tenants
+let mut context_matchers = HashMap::new();
+context_matchers.insert("tenant_id".to_string(), "test_.*".to_string());
+context_matchers.insert("operation".to_string(), "backup".to_string());
+
+configure_failpoint_with_context(
+    "backup_operation",
+    "return(simulated_failure)",
+    context_matchers
+).unwrap();
+
+// Use with context
+let mut context = HashMap::new();
+context.insert("tenant_id".to_string(), "test_123".to_string());
+context.insert("operation".to_string(), "backup".to_string());
+
+match failpoint("backup_operation", Some(&context)) {
+    either::Either::Left(result) => {
+        match result {
+            FailpointResult::Return(value) => {
+                // This will trigger for tenant_id matching "test_.*"
+                println!("Backup failed: {}", value);
+            }
+            FailpointResult::Continue => {
+                // Continue with backup
+            }
+            FailpointResult::Cancelled => {}
+        }
+    }
+    either::Either::Right(future) => {
+        match future.await {
+            FailpointResult::Return(value) => {
+                // This will trigger for tenant_id matching "test_.*"
+                println!("Backup failed: {}", value);
+            }
+            FailpointResult::Continue => {
+                // Continue with backup
+            }
+            FailpointResult::Cancelled => {}
+        }
+    }
+}
+```
+
+## Macros
+
+The library provides convenient macros for common patterns:
+
+### `fail_point!` - Basic Failpoint Macro
+
+The `fail_point!` macro has three variants:
+
+1. **Simple failpoint** - `fail_point!(name)`
+   - Just checks the failpoint and continues or returns early (no value)
+   - Panics if the failpoint is configured with `return(value)` since no closure is provided
+
+2. **Failpoint with return handler** - `fail_point!(name, closure)`
+   - Provides a closure to handle return values from the failpoint
+   - The closure receives `Option<String>` and should return the appropriate value
+
+3. **Conditional failpoint** - `fail_point!(name, condition, closure)`
+   - Only checks the failpoint if the condition is true
+   - Provides a closure to handle return values (receives `&str`)
+
+```rust
+use neon_failpoint::fail_point;
+
+// Simple failpoint - just continue or return early
+fail_point!("my_failpoint");
+
+// Failpoint with return value handling
+fail_point!("my_failpoint", |value: Option<String>| {
+    match value {
+        Some(v) => {
+            println!("Got value: {}", v);
+            return Ok(v.parse().unwrap_or_default());
+        }
+        None => return Ok(42), // Default return value
+    }
+});
+
+// Conditional failpoint - only check if condition is met
+let should_fail = some_condition();
+fail_point!("conditional_failpoint", should_fail, |value: &str| {
+    println!("Conditional failpoint triggered with: {}", value);
+    return Err(anyhow::anyhow!("Simulated failure"));
+});
+```
+
+### `fail_point_with_context!` - Context-Aware Failpoint Macro
+
+The `fail_point_with_context!` macro has three variants that mirror `fail_point!` but include context:
+
+1. **Simple with context** - `fail_point_with_context!(name, context)`
+2. **With context and return handler** - `fail_point_with_context!(name, context, closure)`
+3. **Conditional with context** - `fail_point_with_context!(name, context, condition, closure)`
+
+```rust
+use neon_failpoint::{fail_point_with_context};
+use std::collections::HashMap;
+
+let mut context = HashMap::new();
+context.insert("tenant_id".to_string(), "test_123".to_string());
+context.insert("operation".to_string(), "backup".to_string());
+
+// Simple context failpoint
+fail_point_with_context!("backup_failpoint", &context);
+
+// Context failpoint with return handler
+fail_point_with_context!("backup_failpoint", &context, |value: Option<String>| {
+    match value {
+        Some(v) => return Err(anyhow::anyhow!("Backup failed: {}", v)),
+        None => return Err(anyhow::anyhow!("Backup failed")),
+    }
+});
+
+// Conditional context failpoint
+let is_test_tenant = tenant_id.starts_with("test_");
+fail_point_with_context!("backup_failpoint", &context, is_test_tenant, |value: Option<String>| {
+    // Only triggers for test tenants
+    return Err(anyhow::anyhow!("Test tenant backup failure"));
+});
+```
+
+### Other Utility Macros
+
+```rust
+use neon_failpoint::{pausable_failpoint, sleep_millis_async};
+
+// Pausable failpoint with cancellation
+let cancel_token = CancellationToken::new();
+if let Err(()) = pausable_failpoint!("pause_here", &cancel_token).await {
+    println!("Failpoint was cancelled");
+}
+
+// Sleep failpoint
+sleep_millis_async!("sleep_here", &cancel_token).await;
+
+// Context creation helper
+let mut context = HashMap::new();
+context.insert("key1".to_string(), "value1".to_string());
+context.insert("key2".to_string(), "value2".to_string());
+```
+
+### Argument Reference
+
+- **`name`**: String literal - the name of the failpoint
+- **`context`**: Expression that evaluates to `&HashMap<String, String>` - context for matching
+- **`condition`**: Boolean expression - only check failpoint if true
+- **`closure`**: Closure that handles return values:
+  - For `fail_point!` with closure: receives `Option<String>`
+  - For conditional variants: receives `&str`
+  - For `fail_point_with_context!` with closure: receives `Option<String>`
+- **`cancel`**: `&CancellationToken` - for cancellation support
+
+## Migration from `fail` crate
+
+The library provides a compatibility layer in `libs/utils/src/failpoint_support.rs`. Most existing code should work without changes, but you can migrate to the new async APIs for better performance:
+
+### Before (with `fail` crate):
+```rust
+use utils::failpoint_support::pausable_failpoint;
+
+// This used spawn_blocking internally
+pausable_failpoint!("my_failpoint", &cancel_token).await?;
+```
+
+### After (with `neon_failpoint`):
+```rust
+use neon_failpoint::{failpoint_with_cancellation, FailpointResult};
+
+// This is fully async
+match failpoint_with_cancellation("my_failpoint", None, &cancel_token).await {
+    FailpointResult::Continue => {},
+    FailpointResult::Cancelled => return Err(()),
+    FailpointResult::Return(_) => {},
+}
+```
+
+## Environment Variable Support
+
+Failpoints can be configured via the `FAILPOINTS` environment variable:
+
+```bash
+FAILPOINTS="failpoint1=return(42);failpoint2=sleep(1000);failpoint3=exit"
+```
+
+## Testing
+
+The library includes comprehensive tests and examples. Run them with:
+
+```bash
+cargo test --features testing
+cargo run --example context_demo --features testing
+```
+
+## HTTP Configuration
+
+The library integrates with the existing HTTP failpoint configuration API. Send POST requests to `/v1/failpoints` with:
+
+```json
+[
+  {
+    "name": "my_failpoint",
+    "actions": "return(42)"
+  }
+]
+```

libs/neon_failpoint/examples/context_demo.rs

@@ -0,0 +1,82 @@
+use neon_failpoint::{configure_failpoint_with_context, failpoint, FailpointResult};
+use std::collections::HashMap;
+
+#[tokio::main]
+async fn main() {
+    // Initialize tracing for better output
+    tracing_subscriber::fmt::init();
+
+    // Set up a context-specific failpoint
+    let mut context_matchers = HashMap::new();
+    context_matchers.insert("tenant_id".to_string(), "test_.*".to_string());
+    context_matchers.insert("operation".to_string(), "backup".to_string());
+
+    configure_failpoint_with_context(
+        "backup_operation",
+        "return(simulated_failure)",
+        context_matchers,
+    )
+    .unwrap();
+
+    // Test with matching context
+    let mut context = HashMap::new();
+    context.insert("tenant_id".to_string(), "test_123".to_string());
+    context.insert("operation".to_string(), "backup".to_string());
+
+    println!("Testing with matching context...");
+    match failpoint("backup_operation", Some(&context)) {
+        either::Either::Left(result) => match result {
+            FailpointResult::Return(value) => {
+                println!("Failpoint triggered with value: {value:?}");
+            }
+            FailpointResult::Continue => {
+                println!("Failpoint not triggered");
+            }
+            FailpointResult::Cancelled => {
+                println!("Failpoint cancelled");
+            }
+        },
+        either::Either::Right(future) => match future.await {
+            FailpointResult::Return(value) => {
+                println!("Failpoint triggered with value: {value:?}");
+            }
+            FailpointResult::Continue => {
+                println!("Failpoint not triggered");
+            }
+            FailpointResult::Cancelled => {
+                println!("Failpoint cancelled");
+            }
+        },
+    }
+
+    // Test with non-matching context
+    let mut context = HashMap::new();
+    context.insert("tenant_id".to_string(), "prod_456".to_string());
+    context.insert("operation".to_string(), "backup".to_string());
+
+    println!("Testing with non-matching context...");
+    match failpoint("backup_operation", Some(&context)) {
+        either::Either::Left(result) => match result {
+            FailpointResult::Return(value) => {
+                println!("Failpoint triggered with value: {value:?}");
+            }
+            FailpointResult::Continue => {
+                println!("Failpoint not triggered (expected)");
+            }
+            FailpointResult::Cancelled => {
+                println!("Failpoint cancelled");
+            }
+        },
+        either::Either::Right(future) => match future.await {
+            FailpointResult::Return(value) => {
+                println!("Failpoint triggered with value: {value:?}");
+            }
+            FailpointResult::Continue => {
+                println!("Failpoint not triggered (expected)");
+            }
+            FailpointResult::Cancelled => {
+                println!("Failpoint cancelled");
+            }
+        },
+    }
+}

libs/neon_failpoint/src/macros.rs

@@ -0,0 +1,356 @@
+//! Macros for convenient failpoint usage
+
+/// Simple failpoint macro - async version that doesn't require spawn_blocking
+#[macro_export]
+macro_rules! fail_point {
+    ($name:literal) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, None) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(_) => {
+                            panic!("failpoint was configured with return(X) but Rust code does not pass a closure to map X to a return value");
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(future) => {
+                    match future.await {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(_) => {
+                            panic!("failpoint was configured with return(X) but Rust code does not pass a closure to map X to a return value");
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+            }
+        }
+    }};
+    ($name:literal, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, None) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(value) => {
+                            let closure = $closure;
+                            return closure(value);
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(future) => {
+                    match future.await {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(value) => {
+                            let closure = $closure;
+                            return closure(value);
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+            }
+        }
+    }};
+    ($name:literal, $condition:expr, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            if $condition {
+                match $crate::failpoint($name, None) {
+                    $crate::either::Either::Left(result) => {
+                        match result {
+                            $crate::FailpointResult::Continue => {},
+                            $crate::FailpointResult::Return(value) => {
+                                let closure = $closure;
+                                return closure(value);
+                            },
+                            $crate::FailpointResult::Cancelled => {},
+                        }
+                    },
+                    $crate::either::Either::Right(future) => {
+                        match future.await {
+                            $crate::FailpointResult::Continue => {},
+                            $crate::FailpointResult::Return(value) => {
+                                let closure = $closure;
+                                return closure(value);
+                            },
+                            $crate::FailpointResult::Cancelled => {},
+                        }
+                    },
+                }
+            }
+        }
+    }};
+}
+
+/// Simple failpoint macro - sync version that panics if async action is triggered
+#[macro_export]
+macro_rules! fail_point_sync {
+    ($name:literal) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, None) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(_) => {
+                            panic!("failpoint was configured with return(X) but Rust code does not pass a closure to map X to a return value");
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(_) => {
+                    panic!("failpoint '{}' triggered an async action (sleep/pause) but fail_point_sync! was used. Use fail_point! instead.", $name);
+                },
+            }
+        }
+    }};
+    ($name:literal, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, None) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(value) => {
+                            let closure = $closure;
+                            return closure(value);
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(_) => {
+                    panic!("failpoint '{}' triggered an async action (sleep/pause) but fail_point_sync! was used. Use fail_point! instead.", $name);
+                },
+            }
+        }
+    }};
+    ($name:literal, $condition:expr, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            if $condition {
+                match $crate::failpoint($name, None) {
+                    $crate::either::Either::Left(result) => {
+                        match result {
+                            $crate::FailpointResult::Continue => {},
+                            $crate::FailpointResult::Return(value) => {
+                                let closure = $closure;
+                                return closure(value);
+                            },
+                            $crate::FailpointResult::Cancelled => {},
+                        }
+                    },
+                    $crate::either::Either::Right(_) => {
+                        panic!("failpoint '{}' triggered an async action (sleep/pause) but fail_point_sync! was used. Use fail_point! instead.", $name);
+                    },
+                }
+            }
+        }
+    }};
+}
+
+/// Failpoint macro with context support
+#[macro_export]
+macro_rules! fail_point_with_context {
+    ($name:literal, $context:expr) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, Some($context)) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(_) => {
+                            panic!("failpoint was configured with return(X) but Rust code does not pass a closure to map X to a return value");
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(future) => {
+                    match future.await {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(_) => {
+                            panic!("failpoint was configured with return(X) but Rust code does not pass a closure to map X to a return value");
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+            }
+        }
+    }};
+    ($name:literal, $context:expr, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, Some($context)) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(value) => {
+                            let closure = $closure;
+                            return closure(value);
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(future) => {
+                    match future.await {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(value) => {
+                            let closure = $closure;
+                            return closure(value);
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+            }
+        }
+    }};
+    ($name:literal, $context:expr, $condition:expr, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            if $condition {
+                match $crate::failpoint($name, Some($context)) {
+                    $crate::either::Either::Left(result) => {
+                        match result {
+                            $crate::FailpointResult::Continue => {},
+                            $crate::FailpointResult::Return(value) => {
+                                let closure = $closure;
+                                return closure(value);
+                            },
+                            $crate::FailpointResult::Cancelled => {},
+                        }
+                    },
+                    $crate::either::Either::Right(future) => {
+                        match future.await {
+                            $crate::FailpointResult::Continue => {},
+                            $crate::FailpointResult::Return(value) => {
+                                let closure = $closure;
+                                return closure(value);
+                            },
+                            $crate::FailpointResult::Cancelled => {},
+                        }
+                    },
+                }
+            }
+        }
+    }};
+}
+
+/// Failpoint macro with context support - sync version
+#[macro_export]
+macro_rules! fail_point_with_context_sync {
+    ($name:literal, $context:expr) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, Some($context)) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(_) => {
+                            panic!("failpoint was configured with return(X) but Rust code does not pass a closure to map X to a return value");
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(_) => {
+                    panic!("failpoint '{}' triggered an async action (sleep/pause) but fail_point_with_context_sync! was used. Use fail_point_with_context! instead.", $name);
+                },
+            }
+        }
+    }};
+    ($name:literal, $context:expr, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, Some($context)) {
+                $crate::either::Either::Left(result) => {
+                    match result {
+                        $crate::FailpointResult::Continue => {},
+                        $crate::FailpointResult::Return(value) => {
+                            let closure = $closure;
+                            return closure(value);
+                        },
+                        $crate::FailpointResult::Cancelled => {},
+                    }
+                },
+                $crate::either::Either::Right(_) => {
+                    panic!("failpoint '{}' triggered an async action (sleep/pause) but fail_point_with_context_sync! was used. Use fail_point_with_context! instead.", $name);
+                },
+            }
+        }
+    }};
+    ($name:literal, $context:expr, $condition:expr, $closure:expr) => {{
+        if cfg!(feature = "testing") {
+            if $condition {
+                match $crate::failpoint($name, Some($context)) {
+                    $crate::either::Either::Left(result) => {
+                        match result {
+                            $crate::FailpointResult::Continue => {},
+                            $crate::FailpointResult::Return(value) => {
+                                let closure = $closure;
+                                return closure(value);
+                            },
+                            $crate::FailpointResult::Cancelled => {},
+                        }
+                    },
+                    $crate::either::Either::Right(_) => {
+                        panic!("failpoint '{}' triggered an async action (sleep/pause) but fail_point_with_context_sync! was used. Use fail_point_with_context! instead.", $name);
+                    },
+                }
+            }
+        }
+    }};
+}
+
+/// Pausable failpoint macro - equivalent to the old pausable_failpoint
+#[macro_export]
+macro_rules! pausable_failpoint {
+    ($name:literal) => {{
+        if cfg!(feature = "testing") {
+            let cancel = ::tokio_util::sync::CancellationToken::new();
+            let _ = $crate::pausable_failpoint!($name, &cancel);
+        }
+    }};
+    ($name:literal, $cancel:expr) => {{
+        if cfg!(feature = "testing") {
+            ::tracing::info!("at failpoint {}", $name); // tests rely on this
+            match $crate::failpoint_with_cancellation($name, None, $cancel) {
+                $crate::either::Either::Left(result) => match result {
+                    $crate::FailpointResult::Continue => Ok(()),
+                    $crate::FailpointResult::Return(_) => Ok(()),
+                    $crate::FailpointResult::Cancelled => Err(()),
+                },
+                $crate::either::Either::Right(future) => match future.await {
+                    $crate::FailpointResult::Continue => Ok(()),
+                    $crate::FailpointResult::Return(_) => Ok(()),
+                    $crate::FailpointResult::Cancelled => Err(()),
+                },
+            }
+        } else {
+            Ok(())
+        }
+    }};
+}
+
+/// Sleep failpoint macro - for async sleep operations
+#[macro_export]
+macro_rules! sleep_millis_async {
+    ($name:literal) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint($name, None) {
+                $crate::either::Either::Left(_) => {}
+                $crate::either::Either::Right(future) => {
+                    future.await;
+                }
+            }
+        }
+    }};
+    ($name:literal, $cancel:expr) => {{
+        if cfg!(feature = "testing") {
+            match $crate::failpoint_with_cancellation($name, None, $cancel) {
+                $crate::either::Either::Left(_) => {}
+                $crate::either::Either::Right(future) => {
+                    future.await;
+                }
+            }
+        }
+    }};
+}
+
+// Re-export for convenience
+pub use fail_point;
+pub use fail_point_sync;
+pub use fail_point_with_context;
+pub use fail_point_with_context_sync;
+pub use pausable_failpoint;
+pub use sleep_millis_async;

libs/pageserver_api/src/controller_api.rs

@@ -10,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use utils::id::{NodeId, TenantId, TimelineId};
 use utils::lsn::Lsn;
 
-use crate::models::{PageserverUtilization, ShardParameters, TenantConfig, TimelineInfo};
+use crate::models::{PageserverUtilization, ShardParameters, TenantConfig};
 use crate::shard::{ShardStripeSize, TenantShardId};
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -126,13 +126,6 @@ pub struct TenantDescribeResponse {
     pub config: TenantConfig,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct TenantTimelineDescribeResponse {
-    pub shards: Vec<TimelineInfo>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image_consistent_lsn: Option<Lsn>,
-}
-
 #[derive(Serialize, Deserialize, Debug)]
 pub struct NodeShardResponse {
     pub node_id: NodeId,

libs/pageserver_api/src/models.rs

@@ -1622,9 +1622,6 @@ pub struct TimelineInfo {
 
     /// Whether the timeline is invisible in synthetic size calculations.
     pub is_invisible: Option<bool>,
-    // HADRON: the largest LSN below which all page updates have been included in the image layers.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image_consistent_lsn: Option<Lsn>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

libs/remote_storage/src/simulate_failures.rs

@@ -31,7 +31,6 @@ pub struct UnreliableWrapper {
     /* BEGIN_HADRON */
     // This the probability of failure for each operation, ranged from [0, 100].
     // The probability is default to 100, which means that all operations will fail.
-    // Storage will fail by probability up to attempts_to_fail times.
     attempt_failure_probability: u64,
     /* END_HADRON */
 }

libs/safekeeper_api/src/models.rs

@@ -11,7 +11,7 @@ use utils::id::{NodeId, TenantId, TenantTimelineId, TimelineId};
 use utils::lsn::Lsn;
 use utils::pageserver_feedback::PageserverFeedback;
 
-use crate::membership::{Configuration, SafekeeperGeneration};
+use crate::membership::Configuration;
 use crate::{ServerInfo, Term};
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -311,12 +311,3 @@ pub struct PullTimelineResponse {
     pub safekeeper_host: Option<String>,
     // TODO: add more fields?
 }
-
-/// Response to a timeline locate request.
-/// Storcon-only API.
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct TimelineLocateResponse {
-    pub generation: SafekeeperGeneration,
-    pub sk_set: Vec<NodeId>,
-    pub new_sk_set: Option<Vec<NodeId>>,
-}

libs/utils/Cargo.toml

@@ -9,7 +9,7 @@ default = ["rename_noreplace"]
 rename_noreplace = []
 # Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
 # which adds some runtime cost to run tests on outage conditions
-testing = ["fail/failpoints"]
+testing = ["neon_failpoint/testing"]
 
 [dependencies]
 arc-swap.workspace = true
@@ -21,10 +21,11 @@ bytes.workspace = true
 camino.workspace = true
 chrono.workspace = true
 diatomic-waker.workspace = true
+either.workspace = true
 git-version.workspace = true
 hex = { workspace = true, features = ["serde"] }
 humantime.workspace = true
-fail.workspace = true
+neon_failpoint.workspace = true
 futures = { workspace = true }
 jsonwebtoken.workspace = true
 nix = { workspace = true, features = ["ioctl"] }

libs/utils/src/env.rs

@@ -47,7 +47,6 @@ where
 
 /* BEGIN_HADRON */
 pub enum DeploymentMode {
-    Local,
     Dev,
     Staging,
     Prod,
@@ -65,7 +64,7 @@ pub fn get_deployment_mode() -> Option<DeploymentMode> {
             }
         },
         Err(_) => {
-            // tracing::error!("DEPLOYMENT_MODE not set");
+            tracing::error!("DEPLOYMENT_MODE not set");
             None
         }
     }

libs/utils/src/failpoint_support.rs

@@ -1,59 +1,22 @@
 //! Failpoint support code shared between pageserver and safekeepers.
+//!
+//! This module provides a compatibility layer over the new neon_failpoint crate.
 
+pub use neon_failpoint::{configure_failpoint as apply_failpoint, has_failpoints, init};
 use tokio_util::sync::CancellationToken;
 
-/// Declare a failpoint that can use to `pause` failpoint action.
-/// We don't want to block the executor thread, hence, spawn_blocking + await.
-///
-/// Optionally pass a cancellation token, and this failpoint will drop out of
-/// its pause when the cancellation token fires. This is useful for testing
-/// cases where we would like to block something, but test its clean shutdown behavior.
-/// The macro evaluates to a Result in that case, where Ok(()) is the case
-/// where the failpoint was not paused, and Err() is the case where cancellation
-/// token fired while evaluating the failpoint.
-///
-/// Remember to unpause the failpoint in the test; until that happens, one of the
-/// limited number of spawn_blocking thread pool threads is leaked.
+/// Mere forward to neon_failpoint::pausable_failpoint
 #[macro_export]
 macro_rules! pausable_failpoint {
-    ($name:literal) => {{
-        if cfg!(feature = "testing") {
-            let cancel = ::tokio_util::sync::CancellationToken::new();
-            let _ = $crate::pausable_failpoint!($name, &cancel);
-        }
-    }};
-    ($name:literal, $cancel:expr) => {{
-        if cfg!(feature = "testing") {
-            let failpoint_fut = ::tokio::task::spawn_blocking({
-                let current = ::tracing::Span::current();
-                move || {
-                    let _entered = current.entered();
-                    ::tracing::info!("at failpoint {}", $name);
-                    ::fail::fail_point!($name);
-                }
-            });
-            let cancel_fut = async move {
-                $cancel.cancelled().await;
-            };
-            ::tokio::select! {
-                res = failpoint_fut => {
-                    res.expect("spawn_blocking");
-                    // continue with execution
-                    Ok(())
-                },
-                _ = cancel_fut => {
-                    Err(())
-                }
-            }
-        } else {
-            Ok(())
-        }
-    }};
+    ($name:literal) => {
+        ::neon_failpoint::pausable_failpoint!($name)
+    };
+    ($name:literal, $cancel:expr) => {
+        ::neon_failpoint::pausable_failpoint!($name, $cancel)
+    };
 }
 
-pub use pausable_failpoint;
-
-/// use with fail::cfg("$name", "return(2000)")
+/// DEPRECATED! - use with fail::cfg("$name", "return(2000)")
 ///
 /// The effect is similar to a "sleep(2000)" action, i.e. we sleep for the
 /// specified time (in milliseconds). The main difference is that we use async
@@ -69,7 +32,7 @@ macro_rules! __failpoint_sleep_millis_async {
         // If the failpoint is used with a "return" action, set should_sleep to the
         // returned value (as string). Otherwise it's set to None.
         let should_sleep = (|| {
-            ::fail::fail_point!($name, |x| x);
+            ::neon_failpoint::fail_point_sync!($name, |x| x);
             ::std::option::Option::None
         })();
 
@@ -82,7 +45,7 @@ macro_rules! __failpoint_sleep_millis_async {
         // If the failpoint is used with a "return" action, set should_sleep to the
         // returned value (as string). Otherwise it's set to None.
         let should_sleep = (|| {
-            ::fail::fail_point!($name, |x| x);
+            ::neon_failpoint::fail_point_sync!($name, |x| x);
             ::std::option::Option::None
         })();
 
@@ -126,60 +89,3 @@ pub async fn failpoint_sleep_cancellable_helper(
     tokio::time::timeout(d, cancel.cancelled()).await.ok();
     tracing::info!("failpoint {:?}: sleep done", name);
 }
-
-/// Initialize the configured failpoints
-///
-/// You must call this function before any concurrent threads do operations.
-pub fn init() -> fail::FailScenario<'static> {
-    // The failpoints lib provides support for parsing the `FAILPOINTS` env var.
-    // We want non-default behavior for `exit`, though, so, we handle it separately.
-    //
-    // Format for FAILPOINTS is "name=actions" separated by ";".
-    let actions = std::env::var("FAILPOINTS");
-    if actions.is_ok() {
-        // SAFETY: this function should before any threads start and access env vars concurrently
-        unsafe {
-            std::env::remove_var("FAILPOINTS");
-        }
-    } else {
-        // let the library handle non-utf8, or nothing for not present
-    }
-
-    let scenario = fail::FailScenario::setup();
-
-    if let Ok(val) = actions {
-        val.split(';')
-            .enumerate()
-            .map(|(i, s)| s.split_once('=').ok_or((i, s)))
-            .for_each(|res| {
-                let (name, actions) = match res {
-                    Ok(t) => t,
-                    Err((i, s)) => {
-                        panic!(
-                            "startup failpoints: missing action on the {}th failpoint; try `{s}=return`",
-                            i + 1,
-                        );
-                    }
-                };
-                if let Err(e) = apply_failpoint(name, actions) {
-                    panic!("startup failpoints: failed to apply failpoint {name}={actions}: {e}");
-                }
-            });
-    }
-
-    scenario
-}
-
-pub fn apply_failpoint(name: &str, actions: &str) -> Result<(), String> {
-    if actions == "exit" {
-        fail::cfg_callback(name, exit_failpoint)
-    } else {
-        fail::cfg(name, actions)
-    }
-}
-
-#[inline(never)]
-fn exit_failpoint() {
-    tracing::info!("Exit requested by failpoint");
-    std::process::exit(1);
-}

pageserver/Cargo.toml

@@ -8,7 +8,7 @@ license.workspace = true
 default = []
 # Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
 # which adds some runtime cost to run tests on outage conditions
-testing = ["fail/failpoints", "pageserver_api/testing", "wal_decoder/testing", "pageserver_client/testing"]
+testing = ["neon_failpoint/testing", "pageserver_api/testing", "wal_decoder/testing", "pageserver_client/testing"]
 
 fuzz-read-path = ["testing"]
 
@@ -33,7 +33,7 @@ crc32c.workspace = true
 either.workspace = true
 enum-map.workspace = true
 enumset = { workspace = true, features = ["serde"]}
-fail.workspace = true
+neon_failpoint.workspace = true
 futures.workspace = true
 hashlink.workspace = true
 hex.workspace = true

pageserver/src/basebackup.rs

@@ -17,6 +17,7 @@ use anyhow::{Context, anyhow};
 use async_compression::tokio::write::GzipEncoder;
 use bytes::{BufMut, Bytes, BytesMut};
 use fail::fail_point;
+use neon_failpoint as fail;
 use pageserver_api::key::{Key, rel_block_to_key};
 use pageserver_api::reltag::{RelTag, SlruKind};
 use postgres_ffi::pg_constants::{PG_HBA, PGDATA_SPECIAL_FILES};
@@ -114,7 +115,7 @@ where
     // Compute postgres doesn't have any previous WAL files, but the first
     // record that it's going to write needs to include the LSN of the
     // previous record (xl_prev). We include prev_record_lsn in the
-    // "neon.signal" file, so that postgres can read it during startup.
+    // "zenith.signal" file, so that postgres can read it during startup.
     //
     // We don't keep full history of record boundaries in the page server,
     // however, only the predecessor of the latest record on each
@@ -751,39 +752,34 @@ where
 
     //
     // Add generated pg_control file and bootstrap WAL segment.
-    // Also send neon.signal and zenith.signal file with extra bootstrap data.
+    // Also send zenith.signal file with extra bootstrap data.
     //
     async fn add_pgcontrol_file(
         &mut self,
         pg_control_bytes: Bytes,
         system_identifier: u64,
     ) -> Result<(), BasebackupError> {
-        // add neon.signal file
-        let mut neon_signal = String::new();
+        // add zenith.signal file
+        let mut zenith_signal = String::new();
         if self.prev_record_lsn == Lsn(0) {
             if self.timeline.is_ancestor_lsn(self.lsn) {
-                write!(neon_signal, "PREV LSN: none")
+                write!(zenith_signal, "PREV LSN: none")
                     .map_err(|e| BasebackupError::Server(e.into()))?;
             } else {
-                write!(neon_signal, "PREV LSN: invalid")
+                write!(zenith_signal, "PREV LSN: invalid")
                     .map_err(|e| BasebackupError::Server(e.into()))?;
             }
         } else {
-            write!(neon_signal, "PREV LSN: {}", self.prev_record_lsn)
+            write!(zenith_signal, "PREV LSN: {}", self.prev_record_lsn)
                 .map_err(|e| BasebackupError::Server(e.into()))?;
         }
-
-        // TODO: Remove zenith.signal once all historical computes have been replaced
-        // ... and thus support the neon.signal file.
-        for signalfilename in ["neon.signal", "zenith.signal"] {
-            self.ar
-                .append(
-                    &new_tar_header(signalfilename, neon_signal.len() as u64)?,
-                    neon_signal.as_bytes(),
-                )
-                .await
-                .map_err(|e| BasebackupError::Client(e, "add_pgcontrol_file,neon.signal"))?;
-        }
+        self.ar
+            .append(
+                &new_tar_header("zenith.signal", zenith_signal.len() as u64)?,
+                zenith_signal.as_bytes(),
+            )
+            .await
+            .map_err(|e| BasebackupError::Client(e, "add_pgcontrol_file,zenith.signal"))?;
 
         //send pg_control
         let header = new_tar_header("global/pg_control", pg_control_bytes.len() as u64)?;

pageserver/src/bin/pageserver.rs

@@ -68,7 +68,7 @@ const FEATURES: &[&str] = &[
 fn version() -> String {
     format!(
         "{GIT_VERSION} failpoints: {}, features: {:?}",
-        fail::has_failpoints(),
+        neon_failpoint::has_failpoints(),
         FEATURES,
     )
 }
@@ -84,7 +84,7 @@ fn main() -> anyhow::Result<()> {
     }
 
     // Initialize up failpoints support
-    let scenario = failpoint_support::init();
+    failpoint_support::init().unwrap();
 
     let workdir = arg_matches
         .get_one::<String>("workdir")
@@ -221,7 +221,6 @@ fn main() -> anyhow::Result<()> {
 
     start_pageserver(launch_ts, conf, ignored, otel_guard).context("Failed to start pageserver")?;
 
-    scenario.teardown();
     Ok(())
 }
 
@@ -366,16 +365,9 @@ fn start_pageserver(
 
     // If any failpoints were set from FAILPOINTS environment variable,
     // print them to the log for debugging purposes
-    let failpoints = fail::list();
-    if !failpoints.is_empty() {
-        info!(
-            "started with failpoints: {}",
-            failpoints
-                .iter()
-                .map(|(name, actions)| format!("{name}={actions}"))
-                .collect::<Vec<String>>()
-                .join(";")
-        )
+    let failpoints = neon_failpoint::list();
+    for (name, actions) in failpoints {
+        info!("starting with failpoint: {name} {actions}");
     }
 
     // Create and lock PID file. This ensures that there cannot be more than one
@@ -917,6 +909,11 @@ async fn create_remote_storage_client(
     // If `test_remote_failures` is non-zero, wrap the client with a
     // wrapper that simulates failures.
     if conf.test_remote_failures > 0 {
+        if !cfg!(feature = "testing") {
+            anyhow::bail!(
+                "test_remote_failures option is not available because pageserver was compiled without the 'testing' feature"
+            );
+        }
         info!(
             "Simulating remote failures for first {} attempts of each op",
             conf.test_remote_failures

pageserver/src/consumption_metrics/disk_cache.rs

@@ -6,6 +6,8 @@ use camino::{Utf8Path, Utf8PathBuf};
 use super::{NewMetricsRoot, NewRawMetric, RawMetric};
 use crate::consumption_metrics::NewMetricsRefRoot;
 
+use neon_failpoint as fail;
+
 pub(super) fn read_metrics_from_serde_value(
     json_value: serde_json::Value,
 ) -> anyhow::Result<Vec<NewRawMetric>> {
@@ -129,7 +131,7 @@ pub(super) async fn flush_metrics_to_disk(
             tempfile.flush()?;
             tempfile.as_file().sync_all()?;
 
-            fail::fail_point!("before-persist-last-metrics-collected");
+            fail::fail_point_sync!("before-persist-last-metrics-collected");
 
             drop(tempfile.persist(&*path).map_err(|e| e.error)?);
 

pageserver/src/deletion_queue/deleter.rs

@@ -8,6 +8,7 @@
 
 use std::time::Duration;
 
+use neon_failpoint as fail;
 use remote_storage::{GenericRemoteStorage, RemotePath, TimeoutOrCancel};
 use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};

pageserver/src/http/routes.rs

@@ -28,6 +28,7 @@ use http_utils::{RequestExt, RouterBuilder};
 use humantime::format_rfc3339;
 use hyper::{Body, Request, Response, StatusCode, Uri, header};
 use metrics::launch_timestamp::LaunchTimestamp;
+use neon_failpoint as fail;
 use pageserver_api::models::virtual_file::IoMode;
 use pageserver_api::models::{
     DetachBehavior, DownloadRemoteLayersTaskSpawnRequest, IngestAuxFilesRequest,
@@ -397,7 +398,6 @@ async fn build_timeline_info(
     timeline: &Arc<Timeline>,
     include_non_incremental_logical_size: bool,
     force_await_initial_logical_size: bool,
-    include_image_consistent_lsn: bool,
     ctx: &RequestContext,
 ) -> anyhow::Result<TimelineInfo> {
     crate::tenant::debug_assert_current_span_has_tenant_and_timeline_id();
@@ -422,10 +422,6 @@ async fn build_timeline_info(
                 .await?,
         );
     }
-    // HADRON
-    if include_image_consistent_lsn {
-        info.image_consistent_lsn = Some(timeline.compute_image_consistent_lsn().await?);
-    }
     Ok(info)
 }
 
@@ -515,8 +511,6 @@ async fn build_timeline_info_common(
         is_invisible: Some(is_invisible),
 
         walreceiver_status,
-        // HADRON
-        image_consistent_lsn: None,
     };
     Ok(info)
 }
@@ -719,8 +713,6 @@ async fn timeline_list_handler(
         parse_query_param(&request, "include-non-incremental-logical-size")?;
     let force_await_initial_logical_size: Option<bool> =
         parse_query_param(&request, "force-await-initial-logical-size")?;
-    let include_image_consistent_lsn: Option<bool> =
-        parse_query_param(&request, "include-image-consistent-lsn")?;
     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
 
     let state = get_state(&request);
@@ -741,7 +733,6 @@ async fn timeline_list_handler(
                 &timeline,
                 include_non_incremental_logical_size.unwrap_or(false),
                 force_await_initial_logical_size.unwrap_or(false),
-                include_image_consistent_lsn.unwrap_or(false),
                 &ctx,
             )
             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
@@ -770,9 +761,6 @@ async fn timeline_and_offloaded_list_handler(
         parse_query_param(&request, "include-non-incremental-logical-size")?;
     let force_await_initial_logical_size: Option<bool> =
         parse_query_param(&request, "force-await-initial-logical-size")?;
-    let include_image_consistent_lsn: Option<bool> =
-        parse_query_param(&request, "include-image-consistent-lsn")?;
-
     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
 
     let state = get_state(&request);
@@ -793,7 +781,6 @@ async fn timeline_and_offloaded_list_handler(
                 &timeline,
                 include_non_incremental_logical_size.unwrap_or(false),
                 force_await_initial_logical_size.unwrap_or(false),
-                include_image_consistent_lsn.unwrap_or(false),
                 &ctx,
             )
             .instrument(info_span!("build_timeline_info", timeline_id = %timeline.timeline_id))
@@ -978,9 +965,6 @@ async fn timeline_detail_handler(
         parse_query_param(&request, "include-non-incremental-logical-size")?;
     let force_await_initial_logical_size: Option<bool> =
         parse_query_param(&request, "force-await-initial-logical-size")?;
-    // HADRON
-    let include_image_consistent_lsn: Option<bool> =
-        parse_query_param(&request, "include-image-consistent-lsn")?;
     check_permission(&request, Some(tenant_shard_id.tenant_id))?;
 
     // Logical size calculation needs downloading.
@@ -1001,7 +985,6 @@ async fn timeline_detail_handler(
             &timeline,
             include_non_incremental_logical_size.unwrap_or(false),
             force_await_initial_logical_size.unwrap_or(false),
-            include_image_consistent_lsn.unwrap_or(false),
             ctx,
         )
         .await
@@ -3661,7 +3644,6 @@ async fn activate_post_import_handler(
         let timeline_info = build_timeline_info(
             &timeline, false, // include_non_incremental_logical_size,
             false, // force_await_initial_logical_size
-            false, // include_image_consistent_lsn
             &ctx,
         )
         .await
@@ -3994,7 +3976,7 @@ pub fn make_router(
         .get("/profile/cpu", |r| request_span(r, profile_cpu_handler))
         .get("/profile/heap", |r| request_span(r, profile_heap_handler))
         .get("/v1/status", |r| api_handler(r, status_handler))
-        .put("/v1/failpoints", |r| {
+        .post("/v1/failpoints", |r| {
             testing_api_handler("manage failpoints", r, failpoints_handler)
         })
         .post("/v1/reload_auth_validation_keys", |r| {
@@ -4183,7 +4165,7 @@ pub fn make_router(
         })
         .get(
             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/getpage",
-            |r|  testing_api_handler("getpage@lsn", r, getpage_at_lsn_handler),
+            |r| testing_api_handler("getpage@lsn", r, getpage_at_lsn_handler),
         )
         .get(
             "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/touchpage",

pageserver/src/import_datadir.rs

@@ -610,13 +610,13 @@ async fn import_file(
         debug!("imported twophase file");
     } else if file_path.starts_with("pg_wal") {
         debug!("found wal file in base section. ignore it");
-    } else if file_path.starts_with("zenith.signal") || file_path.starts_with("neon.signal") {
+    } else if file_path.starts_with("zenith.signal") {
         // Parse zenith signal file to set correct previous LSN
         let bytes = read_all_bytes(reader).await?;
-        // neon.signal format is "PREV LSN: prev_lsn"
+        // zenith.signal format is "PREV LSN: prev_lsn"
         // TODO write serialization and deserialization in the same place.
-        let neon_signal = std::str::from_utf8(&bytes)?.trim();
-        let prev_lsn = match neon_signal {
+        let zenith_signal = std::str::from_utf8(&bytes)?.trim();
+        let prev_lsn = match zenith_signal {
             "PREV LSN: none" => Lsn(0),
             "PREV LSN: invalid" => Lsn(0),
             other => {
@@ -624,17 +624,17 @@ async fn import_file(
                 split[1]
                     .trim()
                     .parse::<Lsn>()
-                    .context("can't parse neon.signal")?
+                    .context("can't parse zenith.signal")?
             }
         };
 
-        // neon.signal is not necessarily the last file, that we handle
+        // zenith.signal is not necessarily the last file, that we handle
         // but it is ok to call `finish_write()`, because final `modification.commit()`
         // will update lsn once more to the final one.
         let writer = modification.tline.writer().await;
         writer.finish_write(prev_lsn);
 
-        debug!("imported neon signal {}", prev_lsn);
+        debug!("imported zenith signal {}", prev_lsn);
     } else if file_path.starts_with("pg_tblspc") {
         // TODO Backups exported from neon won't have pg_tblspc, but we will need
         // this to import arbitrary postgres databases.

pageserver/src/page_service.rs

@@ -19,6 +19,7 @@ use futures::future::BoxFuture;
 use futures::{FutureExt, Stream};
 use itertools::Itertools;
 use jsonwebtoken::TokenData;
+use neon_failpoint as fail;
 use once_cell::sync::OnceCell;
 use pageserver_api::config::{
     GetVectoredConcurrentIo, PageServicePipeliningConfig, PageServicePipeliningConfigPipelined,
@@ -335,18 +336,21 @@ async fn page_service_conn_main(
 
     let default_timeout_ms = 10 * 60 * 1000; // 10 minutes by default
     let socket_timeout_ms = (|| {
-        fail::fail_point!("simulated-bad-compute-connection", |avg_timeout_ms| {
-            // Exponential distribution for simulating
-            // poor network conditions, expect about avg_timeout_ms to be around 15
-            // in tests
-            if let Some(avg_timeout_ms) = avg_timeout_ms {
-                let avg = avg_timeout_ms.parse::<i64>().unwrap() as f32;
-                let u = rand::random::<f32>();
-                ((1.0 - u).ln() / (-avg)) as u64
-            } else {
-                default_timeout_ms
+        fail::fail_point_sync!(
+            "simulated-bad-compute-connection",
+            |avg_timeout_ms: Option<String>| {
+                // Exponential distribution for simulating
+                // poor network conditions, expect about avg_timeout_ms to be around 15
+                // in tests
+                if let Some(avg_timeout_ms) = avg_timeout_ms {
+                    let avg = avg_timeout_ms.parse::<i64>().unwrap() as f32;
+                    let u = rand::random::<f32>();
+                    ((1.0 - u).ln() / (-avg)) as u64
+                } else {
+                    default_timeout_ms
+                }
             }
-        });
+        );
         default_timeout_ms
     })();
 
@@ -2167,7 +2171,7 @@ impl PageServerHandler {
     fn effective_request_lsn(
         timeline: &Timeline,
         last_record_lsn: Lsn,
-        mut request_lsn: Lsn,
+        request_lsn: Lsn,
         not_modified_since: Lsn,
         latest_gc_cutoff_lsn: &RcuReadGuard<Lsn>,
     ) -> Result<Lsn, PageStreamError> {
@@ -2195,16 +2199,12 @@ impl PageServerHandler {
         if request_lsn < **latest_gc_cutoff_lsn && !timeline.is_gc_blocked_by_lsn_lease_deadline() {
             let gc_info = &timeline.gc_info.read().unwrap();
             if !gc_info.lsn_covered_by_lease(request_lsn) {
-                // While request was in flight, replica apply_lsn may be advanced.
-                // latest_gc_cutoff_lsn is conservative estimation for min(redo_lsn) for all replicas,
-                // so it is safe to move request_lsn forward to latest_gc_cutoff_lsn.
-                // If replica lease is expired and latest_gc_cutoff_lsn>redo_lsn for this replica,
-                // then check of page LSN at replia protects it from getting too new version of the page.
-                warn!(
-                    "Tried to request a page version that was garbage collected. requested at {} gc cutoff {}",
-                    request_lsn, **latest_gc_cutoff_lsn
+                return Err(
+                    PageStreamError::BadRequest(format!(
+                        "tried to request a page version that was garbage collected. requested at {} gc cutoff {}",
+                        request_lsn, **latest_gc_cutoff_lsn
+                    ).into())
                 );
-                request_lsn = **latest_gc_cutoff_lsn;
             }
         }
 
@@ -3047,7 +3047,7 @@ where
         _pgb: &mut PostgresBackend<IO>,
         sm: &FeStartupPacket,
     ) -> Result<(), QueryError> {
-        fail::fail_point!("ps::connection-start::startup-packet");
+        fail::fail_point_sync!("ps::connection-start::startup-packet");
 
         if let FeStartupPacket::StartupMessage { params, .. } = sm {
             if let Some(app_name) = params.get("application_name") {

pageserver/src/pgdatadir_mapping.rs

@@ -14,6 +14,7 @@ use crate::{PERF_TRACE_TARGET, ensure_walingest};
 use anyhow::Context;
 use bytes::{Buf, Bytes, BytesMut};
 use enum_map::Enum;
+use neon_failpoint as fail;
 use pageserver_api::key::{
     AUX_FILES_KEY, CHECKPOINT_KEY, CONTROLFILE_KEY, CompactKey, DBDIR_KEY, Key, RelDirExists,
     TWOPHASEDIR_KEY, dbdir_key_range, rel_block_to_key, rel_dir_to_key, rel_key_range,

pageserver/src/tenant.rs

@@ -30,6 +30,7 @@ use enumset::EnumSet;
 use futures::StreamExt;
 use futures::stream::FuturesUnordered;
 use itertools::Itertools as _;
+use neon_failpoint as fail;
 use once_cell::sync::Lazy;
 pub use pageserver_api::models::TenantState;
 use pageserver_api::models::{self, RelSizeMigration};
@@ -9571,7 +9572,7 @@ mod tests {
         writer.finish_write(Lsn(0x30));
         drop(writer);
 
-        fail::cfg(
+        neon_failpoint::configure_failpoint(
             "flush-layer-before-update-remote-consistent-lsn",
             "return()",
         )
@@ -12816,40 +12817,6 @@ mod tests {
                 },
             ]
         );
-
-        Ok(())
-    }
-
-    #[tokio::test]
-    async fn test_get_force_image_creation_lsn() -> anyhow::Result<()> {
-        let tenant_conf = pageserver_api::models::TenantConfig {
-            pitr_interval: Some(Duration::from_secs(7 * 3600)),
-            image_layer_force_creation_period: Some(Duration::from_secs(3600)),
-            ..Default::default()
-        };
-
-        let tenant_id = TenantId::generate();
-
-        let harness = TenantHarness::create_custom(
-            "test_get_force_image_creation_lsn",
-            tenant_conf,
-            tenant_id,
-            ShardIdentity::unsharded(),
-            Generation::new(1),
-        )
-        .await?;
-        let (tenant, ctx) = harness.load().await;
-        let timeline = tenant
-            .create_test_timeline(TIMELINE_ID, Lsn(0x10), DEFAULT_PG_VERSION, &ctx)
-            .await?;
-        timeline.gc_info.write().unwrap().cutoffs.time = Some(Lsn(100));
-        {
-            let writer = timeline.writer().await;
-            writer.finish_write(Lsn(5000));
-        }
-
-        let image_creation_lsn = timeline.get_force_image_creation_lsn().unwrap();
-        assert_eq!(image_creation_lsn, Lsn(4300));
         Ok(())
     }
 }

pageserver/src/tenant/layer_map.rs

@@ -46,11 +46,10 @@
 mod historic_layer_coverage;
 mod layer_coverage;
 
-use std::collections::{BTreeMap, HashMap, VecDeque};
+use std::collections::{HashMap, VecDeque};
 use std::iter::Peekable;
 use std::ops::Range;
 use std::sync::Arc;
-use std::time::Instant;
 
 use anyhow::Result;
 use historic_layer_coverage::BufferedHistoricLayerCoverage;
@@ -905,103 +904,6 @@ impl LayerMap {
         max_stacked_deltas
     }
 
-    /* BEGIN_HADRON */
-    /**
-     * Compute the image consistent LSN, the largest LSN below which all pages have been redone successfully.
-     * It works by first finding the latest image layers and store them into a map. Then for each delta layer,
-     * find all overlapping image layers in order to potentially increase the image LSN in case there are gaps
-     * (e.g., if an image is created at LSN 100 but the delta layer spans LSN [150, 200], then we can increase
-     * image LSN to 150 because there is no WAL record in between).
-     * Finally, the image consistent LSN is computed by taking the minimum of all image layers.
-     */
-    pub fn compute_image_consistent_lsn(&self, disk_consistent_lsn: Lsn) -> Lsn {
-        struct ImageLayerInfo {
-            // creation LSN of the image layer
-            image_lsn: Lsn,
-            // the current minimum LSN of newer delta layers with overlapping key ranges
-            min_delta_lsn: Lsn,
-        }
-        let started_at = Instant::now();
-
-        let min_l0_deltas_lsn = {
-            let l0_deltas = self.level0_deltas();
-            l0_deltas
-                .iter()
-                .map(|layer| layer.get_lsn_range().start)
-                .min()
-                .unwrap_or(disk_consistent_lsn)
-        };
-        let global_key_range = Key::MIN..Key::MAX;
-
-        // step 1: collect all most recent image layers into a map
-        // map: end key to image_layer_info
-        let mut image_map: BTreeMap<Key, ImageLayerInfo> = BTreeMap::new();
-        for (img_range, img) in self.image_coverage(&global_key_range, disk_consistent_lsn) {
-            let img_lsn = img.map(|layer| layer.get_lsn_range().end).unwrap_or(Lsn(0));
-            image_map.insert(
-                img_range.end,
-                ImageLayerInfo {
-                    image_lsn: img_lsn,
-                    min_delta_lsn: min_l0_deltas_lsn,
-                },
-            );
-        }
-
-        // step 2: go through all delta layers, and update the image layer info with overlapping
-        // key ranges
-        for layer in self.historic.iter() {
-            if !layer.is_delta {
-                continue;
-            }
-            let delta_key_range = layer.get_key_range();
-            let delta_lsn_range = layer.get_lsn_range();
-            for (img_end_key, img_info) in image_map.range_mut(delta_key_range.start..Key::MAX) {
-                debug_assert!(img_end_key >= &delta_key_range.start);
-                if delta_lsn_range.end > img_info.image_lsn {
-                    // the delta layer includes WAL records after the image
-                    // it's possibel that the delta layer's start LSN < image LSN, which will be simply ignored by step 3
-                    img_info.min_delta_lsn =
-                        std::cmp::min(img_info.min_delta_lsn, delta_lsn_range.start);
-                }
-                if img_end_key >= &delta_key_range.end {
-                    // we have fully processed all overlapping image layers
-                    break;
-                }
-            }
-        }
-
-        // step 3, go through all image layers and find the image consistent LSN
-        let mut img_consistent_lsn = min_l0_deltas_lsn.checked_sub(Lsn(1)).unwrap();
-        let mut prev_key = Key::MIN;
-        for (img_key, img_info) in image_map {
-            tracing::debug!(
-                "Image layer {:?}:{} has min delta lsn {}",
-                Range {
-                    start: prev_key,
-                    end: img_key,
-                },
-                img_info.image_lsn,
-                img_info.min_delta_lsn,
-            );
-            let image_lsn = std::cmp::max(
-                img_info.image_lsn,
-                img_info.min_delta_lsn.checked_sub(Lsn(1)).unwrap_or(Lsn(0)),
-            );
-            img_consistent_lsn = std::cmp::min(img_consistent_lsn, image_lsn);
-            prev_key = img_key;
-        }
-        tracing::info!(
-            "computed image_consistent_lsn {} for disk_consistent_lsn {} in {}ms. Processed {} layrs in total.",
-            img_consistent_lsn,
-            disk_consistent_lsn,
-            started_at.elapsed().as_millis(),
-            self.historic.len()
-        );
-        img_consistent_lsn
-    }
-
-    /* END_HADRON */
-
     /// Return all L0 delta layers
     pub fn level0_deltas(&self) -> &Vec<Arc<PersistentLayerDesc>> {
         &self.l0_delta_layers
@@ -1677,138 +1579,6 @@ mod tests {
             LayerVisibilityHint::Visible
         ));
     }
-
-    /* BEGIN_HADRON */
-    #[test]
-    fn test_compute_image_consistent_lsn() {
-        let mut layer_map = LayerMap::default();
-
-        let disk_consistent_lsn = Lsn(1000);
-        // case 1: empty layer map
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(
-            disk_consistent_lsn.checked_sub(Lsn(1)).unwrap(),
-            image_consistent_lsn
-        );
-
-        // case 2: only L0 delta layer
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(100),
-                Lsn(900)..Lsn(990),
-                true,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(100),
-                Lsn(850)..Lsn(899),
-                true,
-            ));
-        }
-
-        // should use min L0 delta LSN - 1 as image consistent LSN
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(849), image_consistent_lsn);
-
-        // case 3: 3 images, no L1 delta
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(40),
-                Lsn(100)..Lsn(100),
-                false,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(40)..Key::from_i128(70),
-                Lsn(200)..Lsn(200),
-                false,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(70)..Key::from_i128(100),
-                Lsn(150)..Lsn(150),
-                false,
-            ));
-        }
-        // should use min L0 delta LSN - 1 as image consistent LSN
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(849), image_consistent_lsn);
-
-        // case 4: 3 images with 1 L1 delta
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(50),
-                Lsn(300)..Lsn(350),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(299), image_consistent_lsn);
-
-        // case 5: 3 images with 1 more L1 delta with smaller LSN
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(50)..Key::from_i128(72),
-                Lsn(200)..Lsn(300),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(199), image_consistent_lsn);
-
-        // case 6: 3 images with more newer L1 deltas (no impact on final results)
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(30),
-                Lsn(400)..Lsn(500),
-                true,
-            ));
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(35)..Key::from_i128(100),
-                Lsn(450)..Lsn(600),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(199), image_consistent_lsn);
-
-        // case 7: 3 images with more older L1 deltas (no impact on final results)
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(40),
-                Lsn(0)..Lsn(50),
-                true,
-            ));
-
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(50)..Key::from_i128(100),
-                Lsn(10)..Lsn(60),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(199), image_consistent_lsn);
-
-        // case 8: 3 images with one more L1 delta with overlapping LSN range
-        {
-            let mut updates = layer_map.batch_update();
-            updates.insert_historic(PersistentLayerDesc::new_test(
-                Key::from_i128(0)..Key::from_i128(50),
-                Lsn(50)..Lsn(250),
-                true,
-            ));
-        }
-        let image_consistent_lsn = layer_map.compute_image_consistent_lsn(disk_consistent_lsn);
-        assert_eq!(Lsn(100), image_consistent_lsn);
-    }
-
-    /* END_HADRON */
 }
 
 #[cfg(test)]

pageserver/src/tenant/mgr.rs

@@ -12,6 +12,7 @@ use anyhow::Context;
 use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf};
 use futures::StreamExt;
 use itertools::Itertools;
+use neon_failpoint as fail;
 use pageserver_api::key::Key;
 use pageserver_api::models::{DetachBehavior, LocationConfigMode};
 use pageserver_api::shard::{
@@ -1678,8 +1679,6 @@ impl TenantManager {
         // Phase 6: Release the InProgress on the parent shard
         drop(parent_slot_guard);
 
-        utils::pausable_failpoint!("shard-split-post-finish-pause");
-
         Ok(child_shards)
     }
 

pageserver/src/tenant/remote_timeline_client.rs

@@ -194,6 +194,7 @@ pub(crate) use download::{
 };
 use index::GcCompactionState;
 pub(crate) use index::LayerFileMetadata;
+use neon_failpoint as fail;
 use pageserver_api::models::{RelSizeMigration, TimelineArchivalState, TimelineVisibilityState};
 use pageserver_api::shard::{ShardIndex, TenantShardId};
 use regex::Regex;

pageserver/src/tenant/remote_timeline_client/download.rs

@@ -11,6 +11,7 @@ use std::time::SystemTime;
 
 use anyhow::{Context, anyhow};
 use camino::{Utf8Path, Utf8PathBuf};
+use neon_failpoint as fail;
 use pageserver_api::shard::TenantShardId;
 use remote_storage::{
     DownloadError, DownloadKind, DownloadOpts, GenericRemoteStorage, ListingMode, RemotePath,

pageserver/src/tenant/remote_timeline_client/upload.rs

@@ -8,6 +8,7 @@ use anyhow::{Context, bail};
 use bytes::Bytes;
 use camino::Utf8Path;
 use fail::fail_point;
+use neon_failpoint as fail;
 use pageserver_api::shard::TenantShardId;
 use remote_storage::{GenericRemoteStorage, RemotePath, TimeTravelError};
 use tokio::fs::{self, File};

pageserver/src/tenant/timeline.rs

@@ -39,6 +39,7 @@ use layer_manager::{
     LayerManagerLockHolder, LayerManagerReadGuard, LayerManagerWriteGuard, LockedLayerManager,
     Shutdown,
 };
+use neon_failpoint as fail;
 
 use once_cell::sync::Lazy;
 use pageserver_api::config::tenant_conf_defaults::DEFAULT_PITR_INTERVAL;
@@ -351,6 +352,13 @@ pub struct Timeline {
     last_image_layer_creation_check_at: AtomicLsn,
     last_image_layer_creation_check_instant: std::sync::Mutex<Option<Instant>>,
 
+    // HADRON
+    /// If a key range has writes with LSN > force_image_creation_lsn, then we should force image layer creation
+    /// on this key range.
+    force_image_creation_lsn: AtomicLsn,
+    /// The last time instant when force_image_creation_lsn is computed.
+    force_image_creation_lsn_computed_at: std::sync::Mutex<Option<Instant>>,
+
     /// Current logical size of the "datadir", at the last LSN.
     current_logical_size: LogicalSize,
 
@@ -2847,7 +2855,7 @@ impl Timeline {
     }
 
     // HADRON
-    fn get_image_layer_force_creation_period(&self) -> Option<Duration> {
+    fn get_image_creation_timeout(&self) -> Option<Duration> {
         let tenant_conf = self.tenant_conf.load();
         tenant_conf
             .tenant_conf
@@ -3127,6 +3135,9 @@ impl Timeline {
                 repartition_threshold: 0,
                 last_image_layer_creation_check_at: AtomicLsn::new(0),
                 last_image_layer_creation_check_instant: Mutex::new(None),
+                // HADRON
+                force_image_creation_lsn: AtomicLsn::new(0),
+                force_image_creation_lsn_computed_at: std::sync::Mutex::new(None),
                 last_received_wal: Mutex::new(None),
                 rel_size_latest_cache: RwLock::new(HashMap::new()),
                 rel_size_snapshot_cache: Mutex::new(LruCache::new(relsize_snapshot_cache_capacity)),
@@ -5173,7 +5184,9 @@ impl Timeline {
             *self.applied_gc_cutoff_lsn.read(),
         );
 
-        fail_point!("checkpoint-before-saving-metadata", |x| bail!(
+        neon_failpoint::fail_point_sync!("checkpoint-before-saving-metadata", |x: Option<
+            String,
+        >| bail!(
             "{}",
             x.unwrap()
         ));
@@ -5371,16 +5384,13 @@ impl Timeline {
         }
 
         // HADRON
-        // for child timelines, we consider all pages up to ancestor_LSN are redone successfully by the parent timeline
-        min_image_lsn = min_image_lsn.max(self.get_ancestor_lsn());
         if min_image_lsn < force_image_creation_lsn.unwrap_or(Lsn(0)) && max_deltas > 0 {
             info!(
-                "forcing image creation for partitioned range {}-{}. Min image LSN: {}, force image creation LSN: {}, num deltas: {}",
+                "forcing image creation for partitioned range {}-{}. Min image LSN: {}, force image creation LSN: {}",
                 partition.ranges[0].start,
                 partition.ranges[0].end,
                 min_image_lsn,
-                force_image_creation_lsn.unwrap(),
-                max_deltas
+                force_image_creation_lsn.unwrap()
             );
             return true;
         }
@@ -5604,11 +5614,10 @@ impl Timeline {
     /// Predicate function which indicates whether we should check if new image layers
     /// are required. Since checking if new image layers are required is expensive in
     /// terms of CPU, we only do it in the following cases:
-    /// 1. If the timeline has ingested sufficient WAL to justify the cost or ...
+    /// 1. If the timeline has ingested sufficient WAL to justify the cost
     /// 2. If enough time has passed since the last check:
     ///     1. For large tenants, we wish to perform the check more often since they
-    ///        suffer from the lack of image layers. Note that we assume sharded tenants
-    ///        to be large since non-zero shards do not track the logical size.
+    ///        suffer from the lack of image layers
     ///     2. For small tenants (that can mostly fit in RAM), we use a much longer interval
     fn should_check_if_image_layers_required(self: &Arc<Timeline>, lsn: Lsn) -> bool {
         let large_timeline_threshold = self.conf.image_layer_generation_large_timeline_threshold;
@@ -5622,39 +5631,30 @@ impl Timeline {
 
         let distance_based_decision = distance.0 >= min_distance;
 
+        let mut time_based_decision = false;
         let mut last_check_instant = self.last_image_layer_creation_check_instant.lock().unwrap();
-        let check_required_after = (|| {
-            if self.shard_identity.is_unsharded() {
-                if let CurrentLogicalSize::Exact(logical_size) =
-                    self.current_logical_size.current_size()
-                {
-                    if Some(Into::<u64>::into(&logical_size)) < large_timeline_threshold {
-                        return Duration::from_secs(3600 * 48);
-                    }
+        if let CurrentLogicalSize::Exact(logical_size) = self.current_logical_size.current_size() {
+            let check_required_after =
+                if Some(Into::<u64>::into(&logical_size)) >= large_timeline_threshold {
+                    self.get_checkpoint_timeout()
+                } else {
+                    Duration::from_secs(3600 * 48)
+                };
+
+            time_based_decision = match *last_check_instant {
+                Some(last_check) => {
+                    let elapsed = last_check.elapsed();
+                    elapsed >= check_required_after
                 }
-            }
-
-            self.get_checkpoint_timeout()
-        })();
-
-        let time_based_decision = match *last_check_instant {
-            Some(last_check) => {
-                let elapsed = last_check.elapsed();
-                elapsed >= check_required_after
-            }
-            None => true,
-        };
+                None => true,
+            };
+        }
 
         // Do the expensive delta layer counting only if this timeline has ingested sufficient
         // WAL since the last check or a checkpoint timeout interval has elapsed since the last
         // check.
         let decision = distance_based_decision || time_based_decision;
-        tracing::info!(
-            "Decided to check image layers: {}. Distance-based decision: {}, time-based decision: {}",
-            decision,
-            distance_based_decision,
-            time_based_decision
-        );
+
         if decision {
             self.last_image_layer_creation_check_at.store(lsn);
             *last_check_instant = Some(Instant::now());
@@ -7156,19 +7156,6 @@ impl Timeline {
             .unwrap()
             .clone()
     }
-
-    /* BEGIN_HADRON */
-    pub(crate) async fn compute_image_consistent_lsn(&self) -> anyhow::Result<Lsn> {
-        let guard = self
-            .layers
-            .read(LayerManagerLockHolder::ComputeImageConsistentLsn)
-            .await;
-        let layer_map = guard.layer_map()?;
-        let disk_consistent_lsn = self.get_disk_consistent_lsn();
-
-        Ok(layer_map.compute_image_consistent_lsn(disk_consistent_lsn))
-    }
-    /* END_HADRON */
 }
 
 impl Timeline {

pageserver/src/tenant/timeline/compaction.rs

@@ -8,7 +8,7 @@ use std::cmp::min;
 use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
 use std::ops::{Deref, Range};
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::{Duration, Instant, SystemTime};
 
 use super::layer_manager::LayerManagerLockHolder;
 use super::{
@@ -26,6 +26,7 @@ use enumset::EnumSet;
 use fail::fail_point;
 use futures::FutureExt;
 use itertools::Itertools;
+use neon_failpoint as fail;
 use once_cell::sync::Lazy;
 use pageserver_api::config::tenant_conf_defaults::DEFAULT_CHECKPOINT_DISTANCE;
 use pageserver_api::key::{KEY_SIZE, Key};
@@ -34,6 +35,7 @@ use pageserver_api::models::{CompactInfoResponse, CompactKeyRange};
 use pageserver_api::shard::{ShardCount, ShardIdentity, TenantShardId};
 use pageserver_compaction::helpers::{fully_contains, overlaps_with};
 use pageserver_compaction::interface::*;
+use postgres_ffi::to_pg_timestamp;
 use serde::Serialize;
 use tokio::sync::{OwnedSemaphorePermit, Semaphore};
 use tokio_util::sync::CancellationToken;
@@ -46,6 +48,7 @@ use wal_decoder::models::value::Value;
 
 use crate::context::{AccessStatsBehavior, RequestContext, RequestContextBuilder};
 use crate::page_cache;
+use crate::pgdatadir_mapping::LsnForTimestamp;
 use crate::statvfs::Statvfs;
 use crate::tenant::checks::check_valid_layermap;
 use crate::tenant::gc_block::GcBlock;
@@ -1269,7 +1272,10 @@ impl Timeline {
         // Define partitioning schema if needed
 
         // HADRON
-        let force_image_creation_lsn = self.get_force_image_creation_lsn();
+        let force_image_creation_lsn = self
+            .get_or_compute_force_image_creation_lsn(cancel, ctx)
+            .await
+            .map_err(CompactionError::Other)?;
 
         // 1. L0 Compact
         let l0_outcome = {
@@ -1479,37 +1485,59 @@ impl Timeline {
     }
 
     /* BEGIN_HADRON */
-    // Get the force image creation LSN based on gc_cutoff_lsn.
-    // Note that this is an estimation and the workload rate may suddenly change. When that happens,
-    // the force image creation may be too early or too late, but eventually it should be able to catch up.
-    pub(crate) fn get_force_image_creation_lsn(self: &Arc<Self>) -> Option<Lsn> {
-        let image_creation_period = self.get_image_layer_force_creation_period()?;
-        let current_lsn = self.get_last_record_lsn();
-        let pitr_lsn = self.gc_info.read().unwrap().cutoffs.time?;
-        let pitr_interval = self.get_pitr_interval();
-        if pitr_lsn == Lsn::INVALID || pitr_interval.is_zero() {
-            tracing::warn!(
-                "pitr LSN/interval not found, skipping force image creation LSN calculation"
-            );
-            return None;
+    // Get the force image creation LSN. Compute it if the last computed LSN is too old.
+    async fn get_or_compute_force_image_creation_lsn(
+        self: &Arc<Self>,
+        cancel: &CancellationToken,
+        ctx: &RequestContext,
+    ) -> anyhow::Result<Option<Lsn>> {
+        const FORCE_IMAGE_CREATION_LSN_COMPUTE_INTERVAL: Duration = Duration::from_secs(10 * 60); // 10 minutes
+        let image_layer_force_creation_period = self.get_image_creation_timeout();
+        if image_layer_force_creation_period.is_none() {
+            return Ok(None);
         }
 
-        let delta_lsn = current_lsn.checked_sub(pitr_lsn).unwrap().0
-            * image_creation_period.as_secs()
-            / pitr_interval.as_secs();
-        let force_image_creation_lsn = current_lsn.checked_sub(delta_lsn).unwrap_or(Lsn(0));
-
-        tracing::info!(
-            "Tenant shard {} computed force_image_creation_lsn: {}. Current lsn: {}, image_layer_force_creation_period: {:?}, GC cutoff: {}, PITR interval: {:?}",
-            self.tenant_shard_id,
-            force_image_creation_lsn,
-            current_lsn,
-            image_creation_period,
-            pitr_lsn,
-            pitr_interval
-        );
-
-        Some(force_image_creation_lsn)
+        let image_layer_force_creation_period = image_layer_force_creation_period.unwrap();
+        let force_image_creation_lsn_computed_at =
+            *self.force_image_creation_lsn_computed_at.lock().unwrap();
+        if force_image_creation_lsn_computed_at.is_none()
+            || force_image_creation_lsn_computed_at.unwrap().elapsed()
+                > FORCE_IMAGE_CREATION_LSN_COMPUTE_INTERVAL
+        {
+            let now: SystemTime = SystemTime::now();
+            let timestamp = now
+                .checked_sub(image_layer_force_creation_period)
+                .ok_or_else(|| {
+                    anyhow::anyhow!(
+                        "image creation timeout is too large: {image_layer_force_creation_period:?}"
+                    )
+                })?;
+            let timestamp = to_pg_timestamp(timestamp);
+            let force_image_creation_lsn = match self
+                .find_lsn_for_timestamp(timestamp, cancel, ctx)
+                .await?
+            {
+                LsnForTimestamp::Present(lsn) | LsnForTimestamp::Future(lsn) => lsn,
+                _ => {
+                    let gc_lsn = *self.get_applied_gc_cutoff_lsn();
+                    tracing::info!(
+                        "no LSN found for timestamp {timestamp:?}, using latest GC cutoff LSN {}",
+                        gc_lsn
+                    );
+                    gc_lsn
+                }
+            };
+            self.force_image_creation_lsn
+                .store(force_image_creation_lsn);
+            *self.force_image_creation_lsn_computed_at.lock().unwrap() = Some(Instant::now());
+            tracing::info!(
+                "computed force image creation LSN: {}",
+                force_image_creation_lsn
+            );
+            Ok(Some(force_image_creation_lsn))
+        } else {
+            Ok(Some(self.force_image_creation_lsn.load()))
+        }
     }
     /* END_HADRON */
 

pageserver/src/tenant/timeline/delete.rs

@@ -2,6 +2,7 @@ use std::ops::{Deref, DerefMut};
 use std::sync::Arc;
 
 use anyhow::Context;
+use neon_failpoint as fail;
 use pageserver_api::models::TimelineState;
 use pageserver_api::shard::TenantShardId;
 use remote_storage::DownloadError;

pageserver/src/tenant/timeline/detach_ancestor.rs

@@ -4,6 +4,7 @@ use std::sync::Arc;
 use anyhow::Context;
 use bytes::Bytes;
 use http_utils::error::ApiError;
+use neon_failpoint as fail;
 use pageserver_api::key::Key;
 use pageserver_api::keyspace::KeySpace;
 use pageserver_api::models::DetachBehavior;
@@ -1113,7 +1114,7 @@ pub(super) async fn detach_and_reparent(
     // others will fail as if those timelines had been stopped for whatever reason.
     #[cfg(feature = "testing")]
     let failpoint_sem = || -> Option<Arc<Semaphore>> {
-        fail::fail_point!("timeline-detach-ancestor::allow_one_reparented", |_| Some(
+        fail::fail_point_sync!("timeline-detach-ancestor::allow_one_reparented", |_| Some(
             Arc::new(Semaphore::new(1))
         ));
         None

pageserver/src/tenant/timeline/handle.rs

@@ -359,14 +359,14 @@ impl<T: Types> Cache<T> {
                 Err(e) => {
                     // Retry on tenant manager error to handle tenant split more gracefully
                     if attempt < GET_MAX_RETRIES {
+                        tracing::warn!(
+                            "Fail to resolve tenant shard in attempt {}: {:?}. Retrying...",
+                            attempt,
+                            e
+                        );
                         tokio::time::sleep(RETRY_BACKOFF).await;
                         continue;
                     } else {
-                        tracing::warn!(
-                            "Failed to resolve tenant shard after {} attempts: {:?}",
-                            GET_MAX_RETRIES,
-                            e
-                        );
                         return Err(e);
                     }
                 }

pageserver/src/tenant/timeline/layer_manager.rs

@@ -47,7 +47,6 @@ pub(crate) enum LayerManagerLockHolder {
     ImportPgData,
     DetachAncestor,
     Eviction,
-    ComputeImageConsistentLsn,
     #[cfg(test)]
     Testing,
 }

pageserver/src/tenant/timeline/uninit.rs

@@ -5,6 +5,7 @@ use std::sync::Arc;
 
 use anyhow::Context;
 use camino::Utf8PathBuf;
+use neon_failpoint as fail;
 use tracing::{error, info, info_span};
 use utils::fs_ext;
 use utils::id::TimelineId;

pageserver/src/tenant/timeline/walreceiver/walreceiver_connection.rs

@@ -11,6 +11,7 @@ use bytes::BytesMut;
 use chrono::{NaiveDateTime, Utc};
 use fail::fail_point;
 use futures::StreamExt;
+use neon_failpoint as fail;
 use postgres_backend::is_expected_io_error;
 use postgres_connection::PgConnectionConfig;
 use postgres_ffi::WAL_SEGMENT_SIZE;

pageserver/src/walredo.rs

@@ -147,16 +147,6 @@ pub enum RedoAttemptType {
     GcCompaction,
 }
 
-impl std::fmt::Display for RedoAttemptType {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            RedoAttemptType::ReadPage => write!(f, "read page"),
-            RedoAttemptType::LegacyCompaction => write!(f, "legacy compaction"),
-            RedoAttemptType::GcCompaction => write!(f, "gc compaction"),
-        }
-    }
-}
-
 ///
 /// Public interface of WAL redo manager
 ///
@@ -209,7 +199,6 @@ impl PostgresRedoManager {
                         self.conf.wal_redo_timeout,
                         pg_version,
                         max_retry_attempts,
-                        redo_attempt_type,
                     )
                     .await
                 };
@@ -232,7 +221,6 @@ impl PostgresRedoManager {
                 self.conf.wal_redo_timeout,
                 pg_version,
                 max_retry_attempts,
-                redo_attempt_type,
             )
             .await
         }
@@ -457,7 +445,6 @@ impl PostgresRedoManager {
         wal_redo_timeout: Duration,
         pg_version: PgMajorVersion,
         max_retry_attempts: u32,
-        redo_attempt_type: RedoAttemptType,
     ) -> Result<Bytes, Error> {
         *(self.last_redo_at.lock().unwrap()) = Some(Instant::now());
 
@@ -498,28 +485,17 @@ impl PostgresRedoManager {
                 );
 
                 if let Err(e) = result.as_ref() {
-                    macro_rules! message {
-                        ($level:tt) => {
-                            $level!(
-                                "error applying {} WAL records {}..{} ({} bytes) to key {} during {}, from base image with LSN {} to reconstruct page image at LSN {} n_attempts={}: {:?}",
-                                records.len(),
-                                records.first().map(|p| p.0).unwrap_or(Lsn(0)),
-                                records.last().map(|p| p.0).unwrap_or(Lsn(0)),
-                                nbytes,
-                                key,
-                                redo_attempt_type,
-                                base_img_lsn,
-                                lsn,
-                                n_attempts,
-                                e,
-                            )
-                        }
-                    }
-                    match redo_attempt_type {
-                        RedoAttemptType::ReadPage => message!(error),
-                        RedoAttemptType::LegacyCompaction => message!(error),
-                        RedoAttemptType::GcCompaction => message!(warn),
-                    }
+                    error!(
+                        "error applying {} WAL records {}..{} ({} bytes) to key {key}, from base image with LSN {} to reconstruct page image at LSN {} n_attempts={}: {:?}",
+                        records.len(),
+                        records.first().map(|p| p.0).unwrap_or(Lsn(0)),
+                        records.last().map(|p| p.0).unwrap_or(Lsn(0)),
+                        nbytes,
+                        base_img_lsn,
+                        lsn,
+                        n_attempts,
+                        e,
+                    );
                 }
 
                 result.map_err(Error::Other)

pgxn/neon/communicator.c

@@ -1045,34 +1045,6 @@ prefetch_do_request(PrefetchRequest *slot, neon_request_lsns *force_request_lsns
 	Assert(!found);
 }
 
-/*
- * Check that pahge LSN returned by PS to replica is not beyand replay LSN.
- * It can happen only in case of deteriorated lease.
- */
-static bool
-check_page_lsn(NeonGetPageResponse* resp, XLogRecPtr* replay_lsn_ptr)
-{
-	if (RecoveryInProgress())
-	{
-		XLogRecPtr page_lsn = PageGetLSN((Page)resp->page);
-#if PG_VERSION_NUM >= 150000
-		XLogRecPtr replay_lsn = GetCurrentReplayRecPtr(NULL);
-#else
-		/*
-		 * PG14 doesn't have GetCurrentReplayRecPtr() function which returns end of currently applied record.
-		 * And GetXLogReplayRecPtr returns end of WAL records which was already applied.
-		 * So we have to use this hack with resp->req.lsn which is expected to contain end record ptr in this case.
-		 * But it works onlyfor v3 protocol version.
-		 */
-		XLogRecPtr replay_lsn = Max(GetXLogReplayRecPtr(NULL), resp->req.hdr.lsn);
-#endif
-		if (replay_lsn_ptr)
-			*replay_lsn_ptr = replay_lsn;
-		return replay_lsn == 0 || page_lsn <= replay_lsn;
-	}
-	return true;
-}
-
 /*
  * Lookup of already received prefetch requests. Only already received responses matching required LSNs are accepted.
  * Present pages are marked in "mask" bitmap and total number of such pages is returned.
@@ -1096,7 +1068,7 @@ communicator_prefetch_lookupv(NRelFileInfo rinfo, ForkNumber forknum, BlockNumbe
 	for (int i = 0; i < nblocks; i++)
 	{
 		PrfHashEntry *entry;
-		NeonGetPageResponse* resp;
+
 		hashkey.buftag.blockNum = blocknum + i;
 		entry = prfh_lookup(MyPState->prf_hash, &hashkey);
 
@@ -1129,16 +1101,8 @@ communicator_prefetch_lookupv(NRelFileInfo rinfo, ForkNumber forknum, BlockNumbe
 				continue;
 			}
 			Assert(slot->response->tag == T_NeonGetPageResponse); /* checked by check_getpage_response when response was assigned to the slot */
-			resp = (NeonGetPageResponse*)slot->response;
+			memcpy(buffers[i], ((NeonGetPageResponse*)slot->response)->page, BLCKSZ);
 
-			/*
-			 * Ignore "in-future" responses caused by deteriorated lease
-			 */
-			if (!check_page_lsn(resp, NULL))
-			{
-				continue;
-			}
-			memcpy(buffers[i], resp->page, BLCKSZ);
 
 			/*
 			 * With lfc_store_prefetch_result=true prefetch result is stored in LFC in prefetch_pump_state when response is received
@@ -2263,15 +2227,6 @@ Retry:
 			case T_NeonGetPageResponse:
 			{
 				NeonGetPageResponse* getpage_resp = (NeonGetPageResponse *) resp;
-				XLogRecPtr replay_lsn;
-				if (!check_page_lsn(getpage_resp, &replay_lsn))
-				{
-					/* Alternative to throw error is to repeat the query with request_lsn=replay_lsn */
-					ereport(ERROR,
-							(errcode(ERRCODE_IO_ERROR),
-							 errmsg("There is no more version of page %u of relation %u/%u/%u.%u at LSN %X/%X at page server, request LSN %X/%X, latest version is at LSN %X/%X",
-									blockno, RelFileInfoFmt(rinfo), forkNum, LSN_FORMAT_ARGS(replay_lsn), LSN_FORMAT_ARGS(resp->lsn), LSN_FORMAT_ARGS(PageGetLSN((Page)getpage_resp->page)))));
-				}
 				memcpy(buffer, getpage_resp->page, BLCKSZ);
 
 				/*

pgxn/neon/file_cache.c

@@ -162,34 +162,8 @@ typedef struct FileCacheControl
 	dlist_head	lru;			/* double linked list for LRU replacement
 								 * algorithm */
 	dlist_head  holes;          /* double linked list of punched holes */
-
+	HyperLogLogState wss_estimation; /* estimation of working set size */
 	ConditionVariable cv[N_COND_VARS]; /* turnstile of condition variables */
-
-	/*
-	 * Estimation of working set size.
-	 *
-	 * This is not guarded by the lock. No locking is needed because all the
-	 * writes to the "registers" are simple 64-bit stores, to update a
-	 * timestamp. We assume that:
-	 *
-	 * - 64-bit stores are atomic. We could enforce that by using
-	 *   pg_atomic_uint64 instead of TimestampTz as the datatype in hll.h, but
-	 *   for now we just rely on it implicitly.
-	 *
-	 * - Even if they're not, and there is a race between two stores, it
-	 *   doesn't matter much which one wins because they're both updating the
-	 *   register with the current timestamp. Or you have a race between
-	 *   resetting the register and updating it, in which case it also doesn't
-	 *   matter much which one wins.
-	 *
-	 * - If they're not atomic, you might get an occasional "torn write" if
-	 *   you're really unlucky, but we tolerate that too. It just means that
-	 *   the estimate will be a little off, until the register is updated
-	 *   again.
-	 */
-	HyperLogLogState wss_estimation;
-
-	/* Prewarmer state */
 	PrewarmWorkerState prewarm_workers[MAX_PREWARM_WORKERS];
 	size_t n_prewarm_workers;
 	size_t n_prewarm_entries;
@@ -231,8 +205,6 @@ bool AmPrewarmWorker;
 
 #define LFC_ENABLED() (lfc_ctl->limit != 0)
 
-PGDLLEXPORT void lfc_prewarm_main(Datum main_arg);
-
 /*
  * Close LFC file if opened.
  * All backends should close their LFC files once LFC is disabled.
@@ -1170,13 +1142,6 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 
 	CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber);
 
-	/* Update working set size estimate for the blocks */
-	for (int i = 0; i < nblocks; i++)
-	{
-		tag.blockNum = blkno + i;
-		addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag)));
-	}
-
 	/*
 	 * For every chunk that has blocks we're interested in, we
 	 * 1. get the chunk header
@@ -1255,6 +1220,14 @@ lfc_readv_select(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 		}
 
 		entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL);
+
+		/* Approximate working set for the blocks assumed in this entry */
+		for (int i = 0; i < blocks_in_chunk; i++)
+		{
+			tag.blockNum = blkno + i;
+			addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag)));
+		}
+
 		if (entry == NULL)
 		{
 			/* Pages are not cached */
@@ -1531,15 +1504,9 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno,
 		return false;
 
 	CopyNRelFileInfoToBufTag(tag, rinfo);
-	CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber);
 	tag.forkNum = forknum;
 
-	/* Update working set size estimate for the blocks */
-	if (lfc_prewarm_update_ws_estimation)
-	{
-		tag.blockNum = blkno;
-		addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag)));
-	}
+	CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber);
 
 	tag.blockNum = blkno - chunk_offs;
 	hash = get_hash_value(lfc_hash, &tag);
@@ -1557,13 +1524,19 @@ lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blkno,
 
 	if (lwlsn > lsn)
 	{
-		elog(DEBUG1, "Skip LFC write for %u because LwLSN=%X/%X is greater than not_nodified_since LSN %X/%X",
+		elog(DEBUG1, "Skip LFC write for %d because LwLSN=%X/%X is greater than not_nodified_since LSN %X/%X",
 			 blkno, LSN_FORMAT_ARGS(lwlsn), LSN_FORMAT_ARGS(lsn));
 		LWLockRelease(lfc_lock);
 		return false;
 	}
 
 	entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_ENTER, &found);
+
+	if (lfc_prewarm_update_ws_estimation)
+	{
+		tag.blockNum = blkno;
+		addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag)));
+	}
 	if (found)
 	{
 		state = GET_STATE(entry, chunk_offs);
@@ -1676,15 +1649,9 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 		return;
 
 	CopyNRelFileInfoToBufTag(tag, rinfo);
-	CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber);
 	tag.forkNum = forkNum;
 
-	/* Update working set size estimate for the blocks */
-	for (int i = 0; i < nblocks; i++)
-	{
-		tag.blockNum = blkno + i;
-		addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag)));
-	}
+	CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber);
 
 	LWLockAcquire(lfc_lock, LW_EXCLUSIVE);
 
@@ -1725,6 +1692,14 @@ lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 		cv = &lfc_ctl->cv[hash % N_COND_VARS];
 
 		entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_ENTER, &found);
+
+		/* Approximate working set for the blocks assumed in this entry */
+		for (int i = 0; i < blocks_in_chunk; i++)
+		{
+			tag.blockNum = blkno + i;
+			addSHLL(&lfc_ctl->wss_estimation, hash_bytes((uint8_t const*)&tag, sizeof(tag)));
+		}
+
 		if (found)
 		{
 			/*
@@ -2160,23 +2135,40 @@ local_cache_pages(PG_FUNCTION_ARGS)
 		SRF_RETURN_DONE(funcctx);
 }
 
+PG_FUNCTION_INFO_V1(approximate_working_set_size_seconds);
 
-/*
- * Internal implementation of the approximate_working_set_size_seconds()
- * function.
- */
-int32
-lfc_approximate_working_set_size_seconds(time_t duration, bool reset)
+Datum
+approximate_working_set_size_seconds(PG_FUNCTION_ARGS)
 {
-	int32		dc;
+	if (lfc_size_limit != 0)
+	{
+		int32 dc;
+		time_t duration = PG_ARGISNULL(0) ? (time_t)-1 : PG_GETARG_INT32(0);
+		LWLockAcquire(lfc_lock, LW_SHARED);
+		dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, duration);
+		LWLockRelease(lfc_lock);
+		PG_RETURN_INT32(dc);
+	}
+	PG_RETURN_NULL();
+}
 
-	if (lfc_size_limit == 0)
-		return -1;
+PG_FUNCTION_INFO_V1(approximate_working_set_size);
 
-	dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, duration);
-	if (reset)
-		memset(lfc_ctl->wss_estimation.regs, 0, sizeof lfc_ctl->wss_estimation.regs);
-	return dc;
+Datum
+approximate_working_set_size(PG_FUNCTION_ARGS)
+{
+	if (lfc_size_limit != 0)
+	{
+		int32 dc;
+		bool reset = PG_GETARG_BOOL(0);
+		LWLockAcquire(lfc_lock, reset ? LW_EXCLUSIVE : LW_SHARED);
+		dc = (int32) estimateSHLL(&lfc_ctl->wss_estimation, (time_t)-1);
+		if (reset)
+			memset(lfc_ctl->wss_estimation.regs, 0, sizeof lfc_ctl->wss_estimation.regs);
+		LWLockRelease(lfc_lock);
+		PG_RETURN_INT32(dc);
+	}
+	PG_RETURN_NULL();
 }
 
 PG_FUNCTION_INFO_V1(get_local_cache_state);

pgxn/neon/file_cache.h

@@ -47,8 +47,7 @@ extern bool lfc_prefetch(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blk
 extern FileCacheState* lfc_get_state(size_t max_entries);
 extern void lfc_prewarm(FileCacheState* fcs, uint32 n_workers);
 
-extern int32 lfc_approximate_working_set_size_seconds(time_t duration, bool reset);
-
+PGDLLEXPORT void lfc_prewarm_main(Datum main_arg);
 
 static inline bool
 lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,

pgxn/neon/neon.c

@@ -561,8 +561,6 @@ _PG_init(void)
 PG_FUNCTION_INFO_V1(pg_cluster_size);
 PG_FUNCTION_INFO_V1(backpressure_lsns);
 PG_FUNCTION_INFO_V1(backpressure_throttling_time);
-PG_FUNCTION_INFO_V1(approximate_working_set_size_seconds);
-PG_FUNCTION_INFO_V1(approximate_working_set_size);
 
 Datum
 pg_cluster_size(PG_FUNCTION_ARGS)
@@ -609,34 +607,6 @@ backpressure_throttling_time(PG_FUNCTION_ARGS)
 	PG_RETURN_UINT64(BackpressureThrottlingTime());
 }
 
-Datum
-approximate_working_set_size_seconds(PG_FUNCTION_ARGS)
-{
-	time_t		duration;
-	int32		dc;
-
-	duration = PG_ARGISNULL(0) ? (time_t) -1 : PG_GETARG_INT32(0);
-
-	dc = lfc_approximate_working_set_size_seconds(duration, false);
-	if (dc < 0)
-		PG_RETURN_NULL();
-	else
-		PG_RETURN_INT32(dc);
-}
-
-Datum
-approximate_working_set_size(PG_FUNCTION_ARGS)
-{
-	bool		reset = PG_GETARG_BOOL(0);
-	int32		dc;
-
-	dc = lfc_approximate_working_set_size_seconds(-1, reset);
-	if (dc < 0)
-		PG_RETURN_NULL();
-	else
-		PG_RETURN_INT32(dc);
-}
-
 #if PG_MAJORVERSION_NUM >= 16
 static void
 neon_shmem_startup_hook(void)

pgxn/neon_test_utils/neontest.c

@@ -236,13 +236,13 @@ clear_buffer_cache(PG_FUNCTION_ARGS)
 	bool		save_neon_test_evict;
 
 	/*
-	 * Temporarily set the neon_test_evict GUC, so that when we pin and
+	 * Temporarily set the zenith_test_evict GUC, so that when we pin and
 	 * unpin a buffer, the buffer is evicted. We use that hack to evict all
 	 * buffers, as there is no explicit "evict this buffer" function in the
 	 * buffer manager.
 	 */
-	save_neon_test_evict = neon_test_evict;
-	neon_test_evict = true;
+	save_neon_test_evict = zenith_test_evict;
+	zenith_test_evict = true;
 	PG_TRY();
 	{
 		/* Scan through all the buffers */
@@ -273,7 +273,7 @@ clear_buffer_cache(PG_FUNCTION_ARGS)
 
 			/*
 			 * Pin the buffer, and release it again. Because we have
-			 * neon_test_evict==true, this will evict the page from the
+			 * zenith_test_evict==true, this will evict the page from the
 			 * buffer cache if no one else is holding a pin on it.
 			 */
 			if (isvalid)
@@ -286,7 +286,7 @@ clear_buffer_cache(PG_FUNCTION_ARGS)
 	PG_FINALLY();
 	{
 		/* restore the GUC */
-		neon_test_evict = save_neon_test_evict;
+		zenith_test_evict = save_neon_test_evict;
 	}
 	PG_END_TRY();
 

pgxn/typedefs.list

@@ -2953,17 +2953,17 @@ XmlTableBuilderData
 YYLTYPE
 YYSTYPE
 YY_BUFFER_STATE
-NeonErrorResponse
-NeonExistsRequest
-NeonExistsResponse
-NeonGetPageRequest
-NeonGetPageResponse
-NeonMessage
-NeonMessageTag
-NeonNblocksRequest
-NeonNblocksResponse
-NeonRequest
-NeonResponse
+ZenithErrorResponse
+ZenithExistsRequest
+ZenithExistsResponse
+ZenithGetPageRequest
+ZenithGetPageResponse
+ZenithMessage
+ZenithMessageTag
+ZenithNblocksRequest
+ZenithNblocksResponse
+ZenithRequest
+ZenithResponse
 _SPI_connection
 _SPI_plan
 __AssignProcessToJobObject

proxy/Cargo.toml

@@ -16,7 +16,6 @@ async-compression.workspace = true
 async-trait.workspace = true
 atomic-take.workspace = true
 aws-config.workspace = true
-aws-credential-types.workspace = true
 aws-sdk-iam.workspace = true
 aws-sigv4.workspace = true
 base64.workspace = true
@@ -49,7 +48,6 @@ indexmap = { workspace = true, features = ["serde"] }
 ipnet.workspace = true
 itertools.workspace = true
 itoa.workspace = true
-json = { path = "../libs/proxy/json" }
 lasso = { workspace = true, features = ["multi-threaded"] }
 measured = { workspace = true, features = ["lasso"] }
 metrics.workspace = true
@@ -129,4 +127,4 @@ rstest.workspace = true
 walkdir.workspace = true
 rand_distr = "0.4"
 tokio-postgres.workspace = true
-tracing-test = "0.2"
+tracing-test = "0.2"

proxy/README.md

@@ -123,11 +123,6 @@ docker exec -it proxy-postgres psql -U postgres -c "CREATE TABLE neon_control_pl
 docker exec -it proxy-postgres psql -U postgres -c "CREATE ROLE proxy WITH SUPERUSER LOGIN PASSWORD 'password';"
 ```
 
-If you want to test query cancellation, redis is also required:
-```sh
-docker run --detach --name proxy-redis --publish 6379:6379 redis:7.0
-```
-
 Let's create self-signed certificate by running:
 ```sh
 openssl req -new -x509 -days 365 -nodes -text -out server.crt -keyout server.key -subj "/CN=*.local.neon.build"
@@ -135,10 +130,7 @@ openssl req -new -x509 -days 365 -nodes -text -out server.crt -keyout server.key
 
 Then we need to build proxy with 'testing' feature and run, e.g.:
 ```sh
-RUST_LOG=proxy LOGFMT=text cargo run -p proxy --bin proxy --features testing -- \
-  --auth-backend postgres --auth-endpoint 'postgresql://postgres:proxy-postgres@127.0.0.1:5432/postgres' \
-  --redis-auth-type="plain" --redis-plain="redis://127.0.0.1:6379" \
-  -c server.crt -k server.key
+RUST_LOG=proxy LOGFMT=text cargo run -p proxy --bin proxy --features testing -- --auth-backend postgres --auth-endpoint 'postgresql://postgres:proxy-postgres@127.0.0.1:5432/postgres' -c server.crt -k server.key
 ```
 
 Now from client you can start a new session:

proxy/src/batch.rs

@@ -7,17 +7,13 @@ use std::pin::pin;
 use std::sync::Mutex;
 
 use scopeguard::ScopeGuard;
-use tokio::sync::oneshot;
 use tokio::sync::oneshot::error::TryRecvError;
 
 use crate::ext::LockExt;
 
-type ProcResult<P> = Result<<P as QueueProcessing>::Res, <P as QueueProcessing>::Err>;
-
 pub trait QueueProcessing: Send + 'static {
     type Req: Send + 'static;
     type Res: Send;
-    type Err: Send + Clone;
 
     /// Get the desired batch size.
     fn batch_size(&self, queue_size: usize) -> usize;
@@ -28,18 +24,7 @@ pub trait QueueProcessing: Send + 'static {
     /// If this apply can error, it's expected that errors be forwarded to each Self::Res.
     ///
     /// Batching does not need to happen atomically.
-    fn apply(
-        &mut self,
-        req: Vec<Self::Req>,
-    ) -> impl Future<Output = Result<Vec<Self::Res>, Self::Err>> + Send;
-}
-
-#[derive(thiserror::Error)]
-pub enum BatchQueueError<E: Clone, C> {
-    #[error(transparent)]
-    Result(E),
-    #[error(transparent)]
-    Cancelled(C),
+    fn apply(&mut self, req: Vec<Self::Req>) -> impl Future<Output = Vec<Self::Res>> + Send;
 }
 
 pub struct BatchQueue<P: QueueProcessing> {
@@ -49,7 +34,7 @@ pub struct BatchQueue<P: QueueProcessing> {
 
 struct BatchJob<P: QueueProcessing> {
     req: P::Req,
-    res: tokio::sync::oneshot::Sender<Result<P::Res, P::Err>>,
+    res: tokio::sync::oneshot::Sender<P::Res>,
 }
 
 impl<P: QueueProcessing> BatchQueue<P> {
@@ -70,11 +55,11 @@ impl<P: QueueProcessing> BatchQueue<P> {
         &self,
         req: P::Req,
         cancelled: impl Future<Output = R>,
-    ) -> Result<P::Res, BatchQueueError<P::Err, R>> {
+    ) -> Result<P::Res, R> {
         let (id, mut rx) = self.inner.lock_propagate_poison().register_job(req);
 
         let mut cancelled = pin!(cancelled);
-        let resp: Option<Result<P::Res, P::Err>> = loop {
+        let resp = loop {
             // try become the leader, or try wait for success.
             let mut processor = tokio::select! {
                 // try become leader.
@@ -87,7 +72,7 @@ impl<P: QueueProcessing> BatchQueue<P> {
                     if inner.queue.remove(&id).is_some() {
                         tracing::warn!("batched task cancelled before completion");
                     }
-                    return Err(BatchQueueError::Cancelled(cancel));
+                    return Err(cancel);
                 },
             };
 
@@ -111,30 +96,18 @@ impl<P: QueueProcessing> BatchQueue<P> {
             // good: we didn't get cancelled.
             ScopeGuard::into_inner(cancel_safety);
 
-            match values {
-                Ok(values) => {
-                    if values.len() != resps.len() {
-                        tracing::error!(
-                            "batch: invalid response size, expected={}, got={}",
-                            resps.len(),
-                            values.len()
-                        );
-                    }
+            if values.len() != resps.len() {
+                tracing::error!(
+                    "batch: invalid response size, expected={}, got={}",
+                    resps.len(),
+                    values.len()
+                );
+            }
 
-                    // send response values.
-                    for (tx, value) in std::iter::zip(resps, values) {
-                        if tx.send(Ok(value)).is_err() {
-                            // receiver hung up but that's fine.
-                        }
-                    }
-                }
-
-                Err(err) => {
-                    for tx in resps {
-                        if tx.send(Err(err.clone())).is_err() {
-                            // receiver hung up but that's fine.
-                        }
-                    }
+            // send response values.
+            for (tx, value) in std::iter::zip(resps, values) {
+                if tx.send(value).is_err() {
+                    // receiver hung up but that's fine.
                 }
             }
 
@@ -156,8 +129,7 @@ impl<P: QueueProcessing> BatchQueue<P> {
 
         tracing::debug!(id, "batch: job completed");
 
-        resp.expect("no response found. batch processer should not panic")
-            .map_err(BatchQueueError::Result)
+        Ok(resp.expect("no response found. batch processer should not panic"))
     }
 }
 
@@ -167,8 +139,8 @@ struct BatchQueueInner<P: QueueProcessing> {
 }
 
 impl<P: QueueProcessing> BatchQueueInner<P> {
-    fn register_job(&mut self, req: P::Req) -> (u64, oneshot::Receiver<ProcResult<P>>) {
-        let (tx, rx) = oneshot::channel();
+    fn register_job(&mut self, req: P::Req) -> (u64, tokio::sync::oneshot::Receiver<P::Res>) {
+        let (tx, rx) = tokio::sync::oneshot::channel();
 
         let id = self.version;
 
@@ -186,7 +158,7 @@ impl<P: QueueProcessing> BatchQueueInner<P> {
         (id, rx)
     }
 
-    fn get_batch(&mut self, p: &P) -> (Vec<P::Req>, Vec<oneshot::Sender<ProcResult<P>>>) {
+    fn get_batch(&mut self, p: &P) -> (Vec<P::Req>, Vec<tokio::sync::oneshot::Sender<P::Res>>) {
         let batch_size = p.batch_size(self.queue.len());
         let mut reqs = Vec::with_capacity(batch_size);
         let mut resps = Vec::with_capacity(batch_size);

proxy/src/binary/proxy.rs

@@ -522,7 +522,15 @@ pub async fn run() -> anyhow::Result<()> {
         maintenance_tasks.spawn(usage_metrics::task_main(metrics_config));
     }
 
-    if let Some(client) = redis_client {
+    if let Either::Left(auth::Backend::ControlPlane(api, ())) = &auth_backend
+        && let crate::control_plane::client::ControlPlaneClient::ProxyV1(api) = &**api
+        && let Some(client) = redis_client
+    {
+        // project info cache and invalidation of that cache.
+        let cache = api.caches.project_info.clone();
+        maintenance_tasks.spawn(notifications::task_main(client.clone(), cache.clone()));
+        maintenance_tasks.spawn(async move { cache.clone().gc_worker().await });
+
         // Try to connect to Redis 3 times with 1 + (0..0.1) second interval.
         // This prevents immediate exit and pod restart,
         // which can cause hammering of the redis in case of connection issues.
@@ -552,16 +560,6 @@ pub async fn run() -> anyhow::Result<()> {
                 }
             }
         }
-
-        #[allow(irrefutable_let_patterns)]
-        if let Either::Left(auth::Backend::ControlPlane(api, ())) = &auth_backend
-            && let crate::control_plane::client::ControlPlaneClient::ProxyV1(api) = &**api
-        {
-            // project info cache and invalidation of that cache.
-            let cache = api.caches.project_info.clone();
-            maintenance_tasks.spawn(notifications::task_main(client, cache.clone()));
-            maintenance_tasks.spawn(async move { cache.gc_worker().await });
-        }
     }
 
     let maintenance = loop {

proxy/src/cancellation.rs

@@ -4,11 +4,12 @@ use std::pin::pin;
 use std::sync::{Arc, OnceLock};
 use std::time::Duration;
 
+use anyhow::anyhow;
 use futures::FutureExt;
 use ipnet::{IpNet, Ipv4Net, Ipv6Net};
 use postgres_client::RawCancelToken;
 use postgres_client::tls::MakeTlsConnect;
-use redis::{Cmd, FromRedisValue, SetExpiry, SetOptions, Value};
+use redis::{Cmd, FromRedisValue, Value};
 use serde::{Deserialize, Serialize};
 use thiserror::Error;
 use tokio::net::TcpStream;
@@ -17,7 +18,7 @@ use tracing::{debug, error, info};
 
 use crate::auth::AuthError;
 use crate::auth::backend::ComputeUserInfo;
-use crate::batch::{BatchQueue, BatchQueueError, QueueProcessing};
+use crate::batch::{BatchQueue, QueueProcessing};
 use crate::config::ComputeConfig;
 use crate::context::RequestContext;
 use crate::control_plane::ControlPlaneApi;
@@ -27,7 +28,7 @@ use crate::metrics::{CancelChannelSizeGuard, CancellationRequest, Metrics, Redis
 use crate::pqproto::CancelKeyData;
 use crate::rate_limiter::LeakyBucketRateLimiter;
 use crate::redis::keys::KeyPrefix;
-use crate::redis::kv_ops::{RedisKVClient, RedisKVClientError};
+use crate::redis::kv_ops::RedisKVClient;
 
 type IpSubnetKey = IpNet;
 
@@ -44,17 +45,6 @@ pub enum CancelKeyOp {
     GetCancelData {
         key: CancelKeyData,
     },
-    GetCancelDataOld {
-        key: CancelKeyData,
-    },
-}
-
-#[derive(thiserror::Error, Debug, Clone)]
-pub enum PipelineError {
-    #[error("could not send cmd to redis: {0}")]
-    RedisKVClient(Arc<RedisKVClientError>),
-    #[error("incorrect number of responses from redis")]
-    IncorrectNumberOfResponses,
 }
 
 pub struct Pipeline {
@@ -70,7 +60,7 @@ impl Pipeline {
         }
     }
 
-    async fn execute(self, client: &mut RedisKVClient) -> Result<Vec<Value>, PipelineError> {
+    async fn execute(self, client: &mut RedisKVClient) -> Vec<anyhow::Result<Value>> {
         let responses = self.replies;
         let batch_size = self.inner.len();
 
@@ -88,20 +78,30 @@ impl Pipeline {
                     batch_size,
                     responses, "successfully completed cancellation jobs",
                 );
-                Ok(values.into_iter().collect())
+                values.into_iter().map(Ok).collect()
             }
             Ok(value) => {
                 error!(batch_size, ?value, "unexpected redis return value");
-                Err(PipelineError::IncorrectNumberOfResponses)
+                std::iter::repeat_with(|| Err(anyhow!("incorrect response type from redis")))
+                    .take(responses)
+                    .collect()
+            }
+            Err(err) => {
+                std::iter::repeat_with(|| Err(anyhow!("could not send cmd to redis: {err}")))
+                    .take(responses)
+                    .collect()
             }
-            Err(err) => Err(PipelineError::RedisKVClient(Arc::new(err))),
         }
     }
 
-    fn add_command(&mut self, cmd: Cmd) {
+    fn add_command_with_reply(&mut self, cmd: Cmd) {
         self.inner.add_command(cmd);
         self.replies += 1;
     }
+
+    fn add_command_no_reply(&mut self, cmd: Cmd) {
+        self.inner.add_command(cmd).ignore();
+    }
 }
 
 impl CancelKeyOp {
@@ -109,19 +109,12 @@ impl CancelKeyOp {
         match self {
             CancelKeyOp::StoreCancelKey { key, value, expire } => {
                 let key = KeyPrefix::Cancel(*key).build_redis_key();
-                pipe.add_command(Cmd::set_options(
-                    &key,
-                    &**value,
-                    SetOptions::default().with_expiration(SetExpiry::EX(expire.as_secs())),
-                ));
-            }
-            CancelKeyOp::GetCancelDataOld { key } => {
-                let key = KeyPrefix::Cancel(*key).build_redis_key();
-                pipe.add_command(Cmd::hget(key, "data"));
+                pipe.add_command_with_reply(Cmd::hset(&key, "data", &**value));
+                pipe.add_command_no_reply(Cmd::expire(&key, expire.as_secs() as i64));
             }
             CancelKeyOp::GetCancelData { key } => {
                 let key = KeyPrefix::Cancel(*key).build_redis_key();
-                pipe.add_command(Cmd::get(key));
+                pipe.add_command_with_reply(Cmd::hget(key, "data"));
             }
         }
     }
@@ -134,14 +127,13 @@ pub struct CancellationProcessor {
 
 impl QueueProcessing for CancellationProcessor {
     type Req = (CancelChannelSizeGuard<'static>, CancelKeyOp);
-    type Res = redis::Value;
-    type Err = PipelineError;
+    type Res = anyhow::Result<redis::Value>;
 
     fn batch_size(&self, _queue_size: usize) -> usize {
         self.batch_size
     }
 
-    async fn apply(&mut self, batch: Vec<Self::Req>) -> Result<Vec<Self::Res>, Self::Err> {
+    async fn apply(&mut self, batch: Vec<Self::Req>) -> Vec<Self::Res> {
         if !self.client.credentials_refreshed() {
             // this will cause a timeout for cancellation operations
             tracing::debug!(
@@ -252,18 +244,18 @@ impl CancellationHandler {
         &self,
         key: CancelKeyData,
     ) -> Result<Option<CancelClosure>, CancelError> {
-        const TIMEOUT: Duration = Duration::from_secs(5);
+        let guard = Metrics::get()
+            .proxy
+            .cancel_channel_size
+            .guard(RedisMsgKind::HGet);
+        let op = CancelKeyOp::GetCancelData { key };
 
         let Some(tx) = self.tx.get() else {
             tracing::warn!("cancellation handler is not available");
             return Err(CancelError::InternalError);
         };
 
-        let guard = Metrics::get()
-            .proxy
-            .cancel_channel_size
-            .guard(RedisMsgKind::Get);
-        let op = CancelKeyOp::GetCancelData { key };
+        const TIMEOUT: Duration = Duration::from_secs(5);
         let result = timeout(
             TIMEOUT,
             tx.call((guard, op), std::future::pending::<Infallible>()),
@@ -272,37 +264,10 @@ impl CancellationHandler {
         .map_err(|_| {
             tracing::warn!("timed out waiting to receive GetCancelData response");
             CancelError::RateLimit
-        })?;
-
-        // We may still have cancel keys set with HSET <key> "data".
-        // Check error type and retry with HGET.
-        // TODO: remove code after HSET is not used anymore.
-        let result = if let Err(err) = result.as_ref()
-            && let BatchQueueError::Result(err) = err
-            && let PipelineError::RedisKVClient(err) = err
-            && let RedisKVClientError::Redis(err) = &**err
-            && let Some(errcode) = err.code()
-            && errcode == "WRONGTYPE"
-        {
-            let guard = Metrics::get()
-                .proxy
-                .cancel_channel_size
-                .guard(RedisMsgKind::HGet);
-            let op = CancelKeyOp::GetCancelDataOld { key };
-            timeout(
-                TIMEOUT,
-                tx.call((guard, op), std::future::pending::<Infallible>()),
-            )
-            .await
-            .map_err(|_| {
-                tracing::warn!("timed out waiting to receive GetCancelData response");
-                CancelError::RateLimit
-            })?
-        } else {
-            result
-        };
-
-        let result = result.map_err(|e| {
+        })?
+        // cannot be cancelled
+        .unwrap_or_else(|x| match x {})
+        .map_err(|e| {
             tracing::warn!("failed to receive GetCancelData response: {e}");
             CancelError::InternalError
         })?;
@@ -477,7 +442,7 @@ impl Session {
             let guard = Metrics::get()
                 .proxy
                 .cancel_channel_size
-                .guard(RedisMsgKind::Set);
+                .guard(RedisMsgKind::HSet);
             let op = CancelKeyOp::StoreCancelKey {
                 key: self.key,
                 value: closure_json.clone(),
@@ -491,7 +456,7 @@ impl Session {
             );
 
             match tx.call((guard, op), cancel.as_mut()).await {
-                Ok(_) => {
+                Ok(Ok(_)) => {
                     tracing::debug!(
                         src=%self.key,
                         dest=?cancel_closure.cancel_token,
@@ -502,10 +467,10 @@ impl Session {
                     tokio::time::sleep(CANCEL_KEY_REFRESH).await;
                 }
                 // retry immediately.
-                Err(BatchQueueError::Result(error)) => {
+                Ok(Err(error)) => {
                     tracing::warn!(?error, "error registering cancellation key");
                 }
-                Err(BatchQueueError::Cancelled(Err(_cancelled))) => break,
+                Err(Err(_cancelled)) => break,
             }
         }
 

proxy/src/metrics.rs

@@ -374,9 +374,11 @@ pub enum Waiting {
 #[label(singleton = "kind")]
 #[allow(clippy::enum_variant_names)]
 pub enum RedisMsgKind {
-    Set,
-    Get,
+    HSet,
+    HSetMultiple,
     HGet,
+    HGetAll,
+    HDel,
 }
 
 #[derive(Default, Clone)]

proxy/src/redis/connection_with_credentials_provider.rs

@@ -4,12 +4,11 @@ use std::time::Duration;
 
 use futures::FutureExt;
 use redis::aio::{ConnectionLike, MultiplexedConnection};
-use redis::{ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisError, RedisResult};
+use redis::{ConnectionInfo, IntoConnectionInfo, RedisConnectionInfo, RedisResult};
 use tokio::task::AbortHandle;
 use tracing::{error, info, warn};
 
 use super::elasticache::CredentialsProvider;
-use crate::redis::elasticache::CredentialsProviderError;
 
 enum Credentials {
     Static(ConnectionInfo),
@@ -27,14 +26,6 @@ impl Clone for Credentials {
     }
 }
 
-#[derive(thiserror::Error, Debug)]
-pub enum ConnectionProviderError {
-    #[error(transparent)]
-    Redis(#[from] RedisError),
-    #[error(transparent)]
-    CredentialsProvider(#[from] CredentialsProviderError),
-}
-
 /// A wrapper around `redis::MultiplexedConnection` that automatically refreshes the token.
 /// Provides PubSub connection without credentials refresh.
 pub struct ConnectionWithCredentialsProvider {
@@ -95,18 +86,15 @@ impl ConnectionWithCredentialsProvider {
         }
     }
 
-    async fn ping(con: &mut MultiplexedConnection) -> Result<(), ConnectionProviderError> {
-        redis::cmd("PING")
-            .query_async(con)
-            .await
-            .map_err(Into::into)
+    async fn ping(con: &mut MultiplexedConnection) -> RedisResult<()> {
+        redis::cmd("PING").query_async(con).await
     }
 
     pub(crate) fn credentials_refreshed(&self) -> bool {
         self.credentials_refreshed.load(Ordering::Relaxed)
     }
 
-    pub(crate) async fn connect(&mut self) -> Result<(), ConnectionProviderError> {
+    pub(crate) async fn connect(&mut self) -> anyhow::Result<()> {
         let _guard = self.mutex.lock().await;
         if let Some(con) = self.con.as_mut() {
             match Self::ping(con).await {
@@ -153,7 +141,7 @@ impl ConnectionWithCredentialsProvider {
         Ok(())
     }
 
-    async fn get_connection_info(&self) -> Result<ConnectionInfo, ConnectionProviderError> {
+    async fn get_connection_info(&self) -> anyhow::Result<ConnectionInfo> {
         match &self.credentials {
             Credentials::Static(info) => Ok(info.clone()),
             Credentials::Dynamic(provider, addr) => {
@@ -172,7 +160,7 @@ impl ConnectionWithCredentialsProvider {
         }
     }
 
-    async fn get_client(&self) -> Result<redis::Client, ConnectionProviderError> {
+    async fn get_client(&self) -> anyhow::Result<redis::Client> {
         let client = redis::Client::open(self.get_connection_info().await?)?;
         self.credentials_refreshed.store(true, Ordering::Relaxed);
         Ok(client)

proxy/src/redis/elasticache.rs

@@ -9,12 +9,10 @@ use aws_config::meta::region::RegionProviderChain;
 use aws_config::profile::ProfileFileCredentialsProvider;
 use aws_config::provider_config::ProviderConfig;
 use aws_config::web_identity_token::WebIdentityTokenCredentialsProvider;
-use aws_credential_types::provider::error::CredentialsError;
 use aws_sdk_iam::config::ProvideCredentials;
 use aws_sigv4::http_request::{
-    self, SignableBody, SignableRequest, SignatureLocation, SigningError, SigningSettings,
+    self, SignableBody, SignableRequest, SignatureLocation, SigningSettings,
 };
-use aws_sigv4::sign::v4::signing_params::BuildError;
 use tracing::info;
 
 #[derive(Debug)]
@@ -42,18 +40,6 @@ impl AWSIRSAConfig {
     }
 }
 
-#[derive(thiserror::Error, Debug)]
-pub enum CredentialsProviderError {
-    #[error(transparent)]
-    AwsCredentials(#[from] CredentialsError),
-    #[error(transparent)]
-    AwsSigv4Build(#[from] BuildError),
-    #[error(transparent)]
-    AwsSigv4Singing(#[from] SigningError),
-    #[error(transparent)]
-    Http(#[from] http::Error),
-}
-
 /// Credentials provider for AWS elasticache authentication.
 ///
 /// Official documentation:
@@ -106,9 +92,7 @@ impl CredentialsProvider {
         })
     }
 
-    pub(crate) async fn provide_credentials(
-        &self,
-    ) -> Result<(String, String), CredentialsProviderError> {
+    pub(crate) async fn provide_credentials(&self) -> anyhow::Result<(String, String)> {
         let aws_credentials = self
             .credentials_provider
             .provide_credentials()

proxy/src/redis/kv_ops.rs

@@ -2,18 +2,9 @@ use std::time::Duration;
 
 use futures::FutureExt;
 use redis::aio::ConnectionLike;
-use redis::{Cmd, FromRedisValue, Pipeline, RedisError, RedisResult};
+use redis::{Cmd, FromRedisValue, Pipeline, RedisResult};
 
 use super::connection_with_credentials_provider::ConnectionWithCredentialsProvider;
-use crate::redis::connection_with_credentials_provider::ConnectionProviderError;
-
-#[derive(thiserror::Error, Debug)]
-pub enum RedisKVClientError {
-    #[error(transparent)]
-    Redis(#[from] RedisError),
-    #[error(transparent)]
-    ConnectionProvider(#[from] ConnectionProviderError),
-}
 
 pub struct RedisKVClient {
     client: ConnectionWithCredentialsProvider,
@@ -41,13 +32,12 @@ impl RedisKVClient {
         Self { client }
     }
 
-    pub async fn try_connect(&mut self) -> Result<(), RedisKVClientError> {
+    pub async fn try_connect(&mut self) -> anyhow::Result<()> {
         self.client
             .connect()
             .boxed()
             .await
             .inspect_err(|e| tracing::error!("failed to connect to redis: {e}"))
-            .map_err(Into::into)
     }
 
     pub(crate) fn credentials_refreshed(&self) -> bool {
@@ -57,7 +47,7 @@ impl RedisKVClient {
     pub(crate) async fn query<T: FromRedisValue>(
         &mut self,
         q: &impl Queryable,
-    ) -> Result<T, RedisKVClientError> {
+    ) -> anyhow::Result<T> {
         let e = match q.query(&mut self.client).await {
             Ok(t) => return Ok(t),
             Err(e) => e,

proxy/src/serverless/json.rs

@@ -1,7 +1,6 @@
-use json::{ListSer, ObjectSer, ValueSer};
 use postgres_client::Row;
 use postgres_client::types::{Kind, Type};
-use serde_json::Value;
+use serde_json::{Map, Value};
 
 //
 // Convert json non-string types to strings, so that they can be passed to Postgres
@@ -75,40 +74,44 @@ pub(crate) enum JsonConversionError {
     UnbalancedString,
 }
 
-enum OutputMode<'a> {
-    Array(ListSer<'a>),
-    Object(ObjectSer<'a>),
+enum OutputMode {
+    Array(Vec<Value>),
+    Object(Map<String, Value>),
 }
 
-impl OutputMode<'_> {
-    fn key(&mut self, key: &str) -> ValueSer<'_> {
+impl OutputMode {
+    fn key(&mut self, key: &str) -> &mut Value {
         match self {
-            OutputMode::Array(values) => values.entry(),
-            OutputMode::Object(map) => map.key(key),
+            OutputMode::Array(values) => push_entry(values, Value::Null),
+            OutputMode::Object(map) => map.entry(key.to_string()).or_insert(Value::Null),
         }
     }
 
-    fn finish(self) {
+    fn finish(self) -> Value {
         match self {
-            OutputMode::Array(values) => values.finish(),
-            OutputMode::Object(map) => map.finish(),
+            OutputMode::Array(values) => Value::Array(values),
+            OutputMode::Object(map) => Value::Object(map),
         }
     }
 }
 
+fn push_entry<T>(arr: &mut Vec<T>, t: T) -> &mut T {
+    arr.push(t);
+    arr.last_mut().expect("a value was just inserted")
+}
+
 //
 // Convert postgres row with text-encoded values to JSON object
 //
 pub(crate) fn pg_text_row_to_json(
-    output: ValueSer,
     row: &Row,
     raw_output: bool,
     array_mode: bool,
-) -> Result<(), JsonConversionError> {
+) -> Result<Value, JsonConversionError> {
     let mut entries = if array_mode {
-        OutputMode::Array(output.list())
+        OutputMode::Array(Vec::with_capacity(row.columns().len()))
     } else {
-        OutputMode::Object(output.object())
+        OutputMode::Object(Map::with_capacity(row.columns().len()))
     };
 
     for (i, column) in row.columns().iter().enumerate() {
@@ -117,48 +120,53 @@ pub(crate) fn pg_text_row_to_json(
         let value = entries.key(column.name());
 
         match pg_value {
-            Some(v) if raw_output => value.value(v),
+            Some(v) if raw_output => *value = Value::String(v.to_string()),
             Some(v) => pg_text_to_json(value, v, column.type_())?,
-            None => value.value(json::Null),
+            None => *value = Value::Null,
         }
     }
 
-    entries.finish();
-    Ok(())
+    Ok(entries.finish())
 }
 
 //
 // Convert postgres text-encoded value to JSON value
 //
-fn pg_text_to_json(output: ValueSer, val: &str, pg_type: &Type) -> Result<(), JsonConversionError> {
+fn pg_text_to_json(
+    output: &mut Value,
+    val: &str,
+    pg_type: &Type,
+) -> Result<(), JsonConversionError> {
     if let Kind::Array(elem_type) = pg_type.kind() {
         // todo: we should fetch this from postgres.
         let delimiter = ',';
 
-        json::value_as_list!(|output| pg_array_parse(output, val, elem_type, delimiter)?);
+        let mut array = vec![];
+        pg_array_parse(&mut array, val, elem_type, delimiter)?;
+        *output = Value::Array(array);
         return Ok(());
     }
 
     match *pg_type {
-        Type::BOOL => output.value(val == "t"),
+        Type::BOOL => *output = Value::Bool(val == "t"),
         Type::INT2 | Type::INT4 => {
             let val = val.parse::<i32>()?;
-            output.value(val);
+            *output = Value::Number(serde_json::Number::from(val));
         }
         Type::FLOAT4 | Type::FLOAT8 => {
             let fval = val.parse::<f64>()?;
-            if fval.is_finite() {
-                output.value(fval);
+            let num = serde_json::Number::from_f64(fval);
+            if let Some(num) = num {
+                *output = Value::Number(num);
             } else {
                 // Pass Nan, Inf, -Inf as strings
                 // JS JSON.stringify() does converts them to null, but we
                 // want to preserve them, so we pass them as strings
-                output.value(val);
+                *output = Value::String(val.to_string());
             }
         }
-        // we assume that the string value is valid json.
-        Type::JSON | Type::JSONB => output.write_raw_json(val.as_bytes()),
-        _ => output.value(val),
+        Type::JSON | Type::JSONB => *output = serde_json::from_str(val)?,
+        _ => *output = Value::String(val.to_string()),
     }
 
     Ok(())
@@ -184,7 +192,7 @@ fn pg_text_to_json(output: ValueSer, val: &str, pg_type: &Type) -> Result<(), Js
 /// gets its own level of curly braces, and delimiters must be written between adjacent
 /// curly-braced entities of the same level.
 fn pg_array_parse(
-    elements: &mut ListSer,
+    elements: &mut Vec<Value>,
     mut pg_array: &str,
     elem: &Type,
     delim: char,
@@ -213,7 +221,7 @@ fn pg_array_parse(
 /// reads a single array from the `pg_array` string and pushes each values to `elements`.
 /// returns the rest of the `pg_array` string that was not read.
 fn pg_array_parse_inner<'a>(
-    elements: &mut ListSer,
+    elements: &mut Vec<Value>,
     mut pg_array: &'a str,
     elem: &Type,
     delim: char,
@@ -226,7 +234,7 @@ fn pg_array_parse_inner<'a>(
     let mut q = String::new();
 
     loop {
-        let value = elements.entry();
+        let value = push_entry(elements, Value::Null);
         pg_array = pg_array_parse_item(value, &mut q, pg_array, elem, delim)?;
 
         // check for separator.
@@ -252,7 +260,7 @@ fn pg_array_parse_inner<'a>(
 ///
 /// `quoted` is a scratch allocation that has no defined output.
 fn pg_array_parse_item<'a>(
-    output: ValueSer,
+    output: &mut Value,
     quoted: &mut String,
     mut pg_array: &'a str,
     elem: &Type,
@@ -268,8 +276,9 @@ fn pg_array_parse_item<'a>(
 
     if pg_array.starts_with('{') {
         // nested array.
-        pg_array =
-            json::value_as_list!(|output| pg_array_parse_inner(output, pg_array, elem, delim))?;
+        let mut nested = vec![];
+        pg_array = pg_array_parse_inner(&mut nested, pg_array, elem, delim)?;
+        *output = Value::Array(nested);
         return Ok(pg_array);
     }
 
@@ -297,7 +306,7 @@ fn pg_array_parse_item<'a>(
     // we might have an item string:
     // check for null
     if item == "NULL" {
-        output.value(json::Null);
+        *output = Value::Null;
     } else {
         pg_text_to_json(output, item, elem)?;
     }
@@ -431,15 +440,15 @@ mod tests {
     }
 
     fn pg_text_to_json(val: &str, pg_type: &Type) -> Value {
-        let output = json::value_to_string!(|v| super::pg_text_to_json(v, val, pg_type).unwrap());
-        serde_json::from_str(&output).unwrap()
+        let mut v = Value::Null;
+        super::pg_text_to_json(&mut v, val, pg_type).unwrap();
+        v
     }
 
     fn pg_array_parse(pg_array: &str, pg_type: &Type) -> Value {
-        let output = json::value_to_string!(|v| json::value_as_list!(|v| {
-            super::pg_array_parse(v, pg_array, pg_type, ',').unwrap();
-        }));
-        serde_json::from_str(&output).unwrap()
+        let mut array = vec![];
+        super::pg_array_parse(&mut array, pg_array, pg_type, ',').unwrap();
+        Value::Array(array)
     }
 
     #[test]

proxy/src/serverless/sql_over_http.rs

@@ -14,7 +14,10 @@ use hyper::http::{HeaderName, HeaderValue};
 use hyper::{Request, Response, StatusCode, header};
 use indexmap::IndexMap;
 use postgres_client::error::{DbError, ErrorPosition, SqlState};
-use postgres_client::{GenericClient, IsolationLevel, NoTls, ReadyForQueryStatus, Transaction};
+use postgres_client::{
+    GenericClient, IsolationLevel, NoTls, ReadyForQueryStatus, RowStream, Transaction,
+};
+use serde::Serialize;
 use serde_json::Value;
 use serde_json::value::RawValue;
 use tokio::time::{self, Instant};
@@ -684,21 +687,32 @@ impl QueryData {
         let (inner, mut discard) = client.inner();
         let cancel_token = inner.cancel_token();
 
-        let mut json_buf = vec![];
-
-        let batch_result = match select(
+        match select(
             pin!(query_to_json(
                 config,
                 &mut *inner,
                 self,
-                json::ValueSer::new(&mut json_buf),
+                &mut 0,
                 parsed_headers
             )),
             pin!(cancel.cancelled()),
         )
         .await
         {
-            Either::Left((res, __not_yet_cancelled)) => res,
+            // The query successfully completed.
+            Either::Left((Ok((status, results)), __not_yet_cancelled)) => {
+                discard.check_idle(status);
+
+                let json_output =
+                    serde_json::to_string(&results).expect("json serialization should not fail");
+                Ok(json_output)
+            }
+            // The query failed with an error
+            Either::Left((Err(e), __not_yet_cancelled)) => {
+                discard.discard();
+                Err(e)
+            }
+            // The query was cancelled.
             Either::Right((_cancelled, query)) => {
                 tracing::info!("cancelling query");
                 if let Err(err) = cancel_token.cancel_query(NoTls).await {
@@ -707,7 +721,13 @@ impl QueryData {
                 // wait for the query cancellation
                 match time::timeout(time::Duration::from_millis(100), query).await {
                     // query successed before it was cancelled.
-                    Ok(Ok(status)) => Ok(status),
+                    Ok(Ok((status, results))) => {
+                        discard.check_idle(status);
+
+                        let json_output = serde_json::to_string(&results)
+                            .expect("json serialization should not fail");
+                        Ok(json_output)
+                    }
                     // query failed or was cancelled.
                     Ok(Err(error)) => {
                         let db_error = match &error {
@@ -723,29 +743,14 @@ impl QueryData {
                             discard.discard();
                         }
 
-                        return Err(SqlOverHttpError::Cancelled(SqlOverHttpCancel::Postgres));
+                        Err(SqlOverHttpError::Cancelled(SqlOverHttpCancel::Postgres))
                     }
                     Err(_timeout) => {
                         discard.discard();
-                        return Err(SqlOverHttpError::Cancelled(SqlOverHttpCancel::Postgres));
+                        Err(SqlOverHttpError::Cancelled(SqlOverHttpCancel::Postgres))
                     }
                 }
             }
-        };
-
-        match batch_result {
-            // The query successfully completed.
-            Ok(status) => {
-                discard.check_idle(status);
-
-                let json_output = String::from_utf8(json_buf).expect("json should be valid utf8");
-                Ok(json_output)
-            }
-            // The query failed with an error
-            Err(e) => {
-                discard.discard();
-                Err(e)
-            }
         }
     }
 }
@@ -782,7 +787,7 @@ impl BatchQueryData {
             })
             .map_err(SqlOverHttpError::Postgres)?;
 
-        let json_output = match query_batch_to_json(
+        let json_output = match query_batch(
             config,
             cancel.child_token(),
             &mut transaction,
@@ -840,21 +845,24 @@ async fn query_batch(
     transaction: &mut Transaction<'_>,
     queries: BatchQueryData,
     parsed_headers: HttpHeaders,
-    results: &mut json::ListSer<'_>,
-) -> Result<(), SqlOverHttpError> {
+) -> Result<String, SqlOverHttpError> {
+    let mut results = Vec::with_capacity(queries.queries.len());
+    let mut current_size = 0;
     for stmt in queries.queries {
         let query = pin!(query_to_json(
             config,
             transaction,
             stmt,
-            results.entry(),
+            &mut current_size,
             parsed_headers,
         ));
         let cancelled = pin!(cancel.cancelled());
         let res = select(query, cancelled).await;
         match res {
             // TODO: maybe we should check that the transaction bit is set here
-            Either::Left((Ok(_), _cancelled)) => {}
+            Either::Left((Ok((_, values)), _cancelled)) => {
+                results.push(values);
+            }
             Either::Left((Err(e), _cancelled)) => {
                 return Err(e);
             }
@@ -864,22 +872,8 @@ async fn query_batch(
         }
     }
 
-    Ok(())
-}
-
-async fn query_batch_to_json(
-    config: &'static HttpConfig,
-    cancel: CancellationToken,
-    tx: &mut Transaction<'_>,
-    queries: BatchQueryData,
-    headers: HttpHeaders,
-) -> Result<String, SqlOverHttpError> {
-    let json_output = json::value_to_string!(|obj| json::value_as_object!(|obj| {
-        let results = obj.key("results");
-        json::value_as_list!(|results| {
-            query_batch(config, cancel, tx, queries, headers, results).await?;
-        });
-    }));
+    let results = json!({ "results": results });
+    let json_output = serde_json::to_string(&results).expect("json serialization should not fail");
 
     Ok(json_output)
 }
@@ -888,54 +882,54 @@ async fn query_to_json<T: GenericClient>(
     config: &'static HttpConfig,
     client: &mut T,
     data: QueryData,
-    output: json::ValueSer<'_>,
+    current_size: &mut usize,
     parsed_headers: HttpHeaders,
-) -> Result<ReadyForQueryStatus, SqlOverHttpError> {
+) -> Result<(ReadyForQueryStatus, impl Serialize + use<T>), SqlOverHttpError> {
     let query_start = Instant::now();
 
-    let mut output = json::ObjectSer::new(output);
+    let query_params = data.params;
     let mut row_stream = client
-        .query_raw_txt(&data.query, data.params)
+        .query_raw_txt(&data.query, query_params)
         .await
         .map_err(SqlOverHttpError::Postgres)?;
     let query_acknowledged = Instant::now();
 
-    let mut json_fields = output.key("fields").list();
-    for c in row_stream.statement.columns() {
-        let json_field = json_fields.entry();
-        json::value_as_object!(|json_field| {
-            json_field.entry("name", c.name());
-            json_field.entry("dataTypeID", c.type_().oid());
-            json_field.entry("tableID", c.table_oid());
-            json_field.entry("columnID", c.column_id());
-            json_field.entry("dataTypeSize", c.type_size());
-            json_field.entry("dataTypeModifier", c.type_modifier());
-            json_field.entry("format", "text");
-        });
-    }
-    json_fields.finish();
+    let columns_len = row_stream.statement.columns().len();
+    let mut fields = Vec::with_capacity(columns_len);
+
+    for c in row_stream.statement.columns() {
+        fields.push(json!({
+            "name": c.name().to_owned(),
+            "dataTypeID": c.type_().oid(),
+            "tableID": c.table_oid(),
+            "columnID": c.column_id(),
+            "dataTypeSize": c.type_size(),
+            "dataTypeModifier": c.type_modifier(),
+            "format": "text",
+        }));
+    }
 
-    let array_mode = data.array_mode.unwrap_or(parsed_headers.default_array_mode);
     let raw_output = parsed_headers.raw_output;
+    let array_mode = data.array_mode.unwrap_or(parsed_headers.default_array_mode);
 
     // Manually drain the stream into a vector to leave row_stream hanging
     // around to get a command tag. Also check that the response is not too
     // big.
-    let mut rows = 0;
-    let mut json_rows = output.key("rows").list();
+    let mut rows = Vec::new();
     while let Some(row) = row_stream.next().await {
         let row = row.map_err(SqlOverHttpError::Postgres)?;
+        *current_size += row.body_len();
 
         // we don't have a streaming response support yet so this is to prevent OOM
         // from a malicious query (eg a cross join)
-        if json_rows.as_buffer().len() > config.max_response_size_bytes {
+        if *current_size > config.max_response_size_bytes {
             return Err(SqlOverHttpError::ResponseTooLarge(
                 config.max_response_size_bytes,
             ));
         }
 
-        pg_text_row_to_json(json_rows.entry(), &row, raw_output, array_mode)?;
-        rows += 1;
+        let row = pg_text_row_to_json(&row, raw_output, array_mode)?;
+        rows.push(row);
 
         // assumption: parsing pg text and converting to json takes CPU time.
         // let's assume it is slightly expensive, so we should consume some cooperative budget.
@@ -943,14 +937,16 @@ async fn query_to_json<T: GenericClient>(
         // of rows and never hit the tokio mpsc for a long time (although unlikely).
         tokio::task::consume_budget().await;
     }
-    json_rows.finish();
 
     let query_resp_end = Instant::now();
-
-    let ready = row_stream.status;
+    let RowStream {
+        command_tag,
+        status: ready,
+        ..
+    } = row_stream;
 
     // grab the command tag and number of rows affected
-    let command_tag = row_stream.command_tag.unwrap_or_default();
+    let command_tag = command_tag.unwrap_or_default();
     let mut command_tag_split = command_tag.split(' ');
     let command_tag_name = command_tag_split.next().unwrap_or_default();
     let command_tag_count = if command_tag_name == "INSERT" {
@@ -963,7 +959,7 @@ async fn query_to_json<T: GenericClient>(
     .and_then(|s| s.parse::<i64>().ok());
 
     info!(
-        rows,
+        rows = rows.len(),
         ?ready,
         command_tag,
         acknowledgement = ?(query_acknowledged - query_start),
@@ -971,12 +967,16 @@ async fn query_to_json<T: GenericClient>(
         "finished executing query"
     );
 
-    output.entry("command", command_tag_name);
-    output.entry("rowCount", command_tag_count);
-    output.entry("rowAsArray", array_mode);
+    // Resulting JSON format is based on the format of node-postgres result.
+    let results = json!({
+        "command": command_tag_name.to_string(),
+        "rowCount": command_tag_count,
+        "rows": rows,
+        "fields": fields,
+        "rowAsArray": array_mode,
+    });
 
-    output.finish();
-    Ok(ready)
+    Ok((ready, results))
 }
 
 enum Client {

safekeeper/Cargo.toml

@@ -6,9 +6,7 @@ license.workspace = true
 
 [features]
 default = []
-# Enables test-only APIs, incuding failpoints. In particular, enables the `fail_point!` macro,
-# which adds some runtime cost to run tests on outage conditions
-testing = ["fail/failpoints"]
+testing = ["neon_failpoint/testing"]
 benchmarking = []
 
 [dependencies]
@@ -21,7 +19,7 @@ camino-tempfile.workspace = true
 chrono.workspace = true
 clap = { workspace = true, features = ["derive"] }
 crc32c.workspace = true
-fail.workspace = true
+neon_failpoint.workspace = true
 hex.workspace = true
 humantime.workspace = true
 http.workspace = true

safekeeper/src/bin/safekeeper.rs

@@ -65,7 +65,7 @@ const FEATURES: &[&str] = &[
 fn version() -> String {
     format!(
         "{GIT_VERSION} failpoints: {}, features: {:?}",
-        fail::has_failpoints(),
+        neon_failpoint::has_failpoints(),
         FEATURES,
     )
 }

safekeeper/src/http/routes.rs

@@ -717,7 +717,7 @@ pub fn make_router(
         .get("/profile/cpu", |r| request_span(r, profile_cpu_handler))
         .get("/profile/heap", |r| request_span(r, profile_heap_handler))
         .get("/v1/status", |r| request_span(r, status_handler))
-        .put("/v1/failpoints", |r| {
+        .post("/v1/failpoints", |r| {
             request_span(r, move |r| async {
                 check_permission(&r, None)?;
                 let cancel = CancellationToken::new();

safekeeper/src/send_wal.rs

@@ -872,14 +872,15 @@ impl<IO: AsyncRead + AsyncWrite + Unpin> WalSender<'_, IO> {
     async fn wait_wal(&mut self) -> Result<(), CopyStreamHandlerEnd> {
         loop {
             self.end_pos = self.end_watch.get();
-            let have_something_to_send = (|| {
-                fail::fail_point!(
+            let have_something_to_send = async {
+                neon_failpoint::fail_point!(
                     "sk-pause-send",
                     self.appname.as_deref() != Some("pageserver"),
                     |_| { false }
                 );
                 self.end_pos > self.start_pos
-            })();
+            }
+            .await;
 
             if have_something_to_send {
                 trace!("got end_pos {:?}, streaming", self.end_pos);
@@ -931,14 +932,15 @@ impl<IO: AsyncRead + AsyncWrite + Unpin> WalSender<'_, IO> {
     /// - Err in case of error -- only if 1) term changed while fetching in recovery
     ///   mode 2) watch channel closed, which must never happen.
     async fn wait_for_lsn(&mut self) -> anyhow::Result<Option<Lsn>> {
-        let fp = (|| {
-            fail::fail_point!(
+        let fp = async {
+            neon_failpoint::fail_point!(
                 "sk-pause-send",
                 self.appname.as_deref() != Some("pageserver"),
                 |_| { true }
             );
             false
-        })();
+        }
+        .await;
         if fp {
             tokio::time::sleep(POLL_STATE_TIMEOUT).await;
             return Ok(None);

safekeeper/src/wal_backup.rs

@@ -657,7 +657,7 @@ pub async fn delete_timeline(
 
     pausable_failpoint!("sk-delete-timeline-remote-pause");
 
-    fail::fail_point!("sk-delete-timeline-remote", |_| {
+    neon_failpoint::fail_point!("sk-delete-timeline-remote", |_| {
         Err(anyhow::anyhow!("failpoint: sk-delete-timeline-remote"))
     });
 

safekeeper/src/wal_storage.rs

@@ -301,7 +301,7 @@ impl PhysicalStorage {
                 format!("Failed to open tmp wal file {:?}", &tmp_path)
             })?;
 
-            fail::fail_point!("sk-zero-segment", |_| {
+            neon_failpoint::fail_point!("sk-zero-segment", |_| {
                 info!("sk-zero-segment failpoint hit");
                 Err(anyhow::anyhow!("failpoint: sk-zero-segment"))
             });

storage_controller/Cargo.toml

@@ -22,7 +22,7 @@ clap.workspace = true
 clashmap.workspace = true
 compute_api.workspace = true
 cron.workspace = true
-fail.workspace = true
+neon_failpoint.workspace = true
 futures.workspace = true
 governor.workspace = true
 hex.workspace = true

storage_controller/src/http.rs

@@ -850,31 +850,6 @@ async fn handle_tenant_describe(
     json_response(StatusCode::OK, service.tenant_describe(tenant_id)?)
 }
 
-/* BEGIN_HADRON */
-async fn handle_tenant_timeline_describe(
-    service: Arc<Service>,
-    req: Request<Body>,
-) -> Result<Response<Body>, ApiError> {
-    check_permissions(&req, Scope::Scrubber)?;
-
-    let tenant_id: TenantId = parse_request_param(&req, "tenant_id")?;
-    let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?;
-    match maybe_forward(req).await {
-        ForwardOutcome::Forwarded(res) => {
-            return res;
-        }
-        ForwardOutcome::NotForwarded(_req) => {}
-    };
-
-    json_response(
-        StatusCode::OK,
-        service
-            .tenant_timeline_describe(tenant_id, timeline_id)
-            .await?,
-    )
-}
-/* END_HADRON */
-
 async fn handle_tenant_list(
     service: Arc<Service>,
     req: Request<Body>,
@@ -2505,13 +2480,6 @@ pub fn make_router(
             )
         })
         // Timeline operations
-        .get("/control/v1/tenant/:tenant_id/timeline/:timeline_id", |r| {
-            tenant_service_handler(
-                r,
-                handle_tenant_timeline_describe,
-                RequestName("v1_tenant_timeline_describe"),
-            )
-        })
         .delete("/v1/tenant/:tenant_id/timeline/:timeline_id", |r| {
             tenant_service_handler(
                 r,

storage_controller/src/main.rs

@@ -222,9 +222,6 @@ struct Cli {
     /// Primarily useful for testing to reduce test execution time.
     #[arg(long, default_value = "false", action=ArgAction::Set)]
     kick_secondary_downloads: bool,
-
-    #[arg(long)]
-    shard_split_request_timeout: Option<humantime::Duration>,
 }
 
 enum StrictMode {
@@ -473,10 +470,6 @@ async fn async_main() -> anyhow::Result<()> {
         timeline_safekeeper_count: args.timeline_safekeeper_count,
         posthog_config: posthog_config.clone(),
         kick_secondary_downloads: args.kick_secondary_downloads,
-        shard_split_request_timeout: args
-            .shard_split_request_timeout
-            .map(humantime::Duration::into)
-            .unwrap_or(Duration::MAX),
     };
 
     // Validate that we can connect to the database

storage_controller/src/pageserver_client.rs

@@ -86,23 +86,6 @@ impl PageserverClient {
         )
     }
 
-    /* BEGIN_HADRON */
-    pub(crate) async fn tenant_timeline_describe(
-        &self,
-        tenant_shard_id: &TenantShardId,
-        timeline_id: &TimelineId,
-    ) -> Result<TimelineInfo> {
-        measured_request!(
-            "tenant_timeline_describe",
-            crate::metrics::Method::Get,
-            &self.node_id_label,
-            self.inner
-                .tenant_timeline_describe(tenant_shard_id, timeline_id,)
-                .await
-        )
-    }
-    /* END_HADRON */
-
     pub(crate) async fn tenant_scan_remote_storage(
         &self,
         tenant_id: TenantId,

storage_controller/src/service.rs

@@ -25,6 +25,7 @@ use futures::stream::FuturesUnordered;
 use http_utils::error::ApiError;
 use hyper::Uri;
 use itertools::Itertools;
+use neon_failpoint as fail;
 use pageserver_api::config::PostHogConfig;
 use pageserver_api::controller_api::{
     AvailabilityZone, MetadataHealthRecord, MetadataHealthUpdateRequest, NodeAvailability,
@@ -32,7 +33,7 @@ use pageserver_api::controller_api::{
     ShardSchedulingPolicy, ShardsPreferredAzsRequest, ShardsPreferredAzsResponse,
     SkSchedulingPolicy, TenantCreateRequest, TenantCreateResponse, TenantCreateResponseShard,
     TenantDescribeResponse, TenantDescribeResponseShard, TenantLocateResponse, TenantPolicyRequest,
-    TenantShardMigrateRequest, TenantShardMigrateResponse, TenantTimelineDescribeResponse,
+    TenantShardMigrateRequest, TenantShardMigrateResponse,
 };
 use pageserver_api::models::{
     self, DetachBehavior, LocationConfig, LocationConfigListResponse, LocationConfigMode, LsnLease,
@@ -60,7 +61,6 @@ use tokio::sync::mpsc::error::TrySendError;
 use tokio_util::sync::CancellationToken;
 use tracing::{Instrument, debug, error, info, info_span, instrument, warn};
 use utils::completion::Barrier;
-use utils::env;
 use utils::generation::Generation;
 use utils::id::{NodeId, TenantId, TimelineId};
 use utils::lsn::Lsn;
@@ -484,9 +484,6 @@ pub struct Config {
 
     /// When set, actively checks and initiates heatmap downloads/uploads.
     pub kick_secondary_downloads: bool,
-
-    /// Timeout used for HTTP client of split requests. [`Duration::MAX`] if None.
-    pub shard_split_request_timeout: Duration,
 }
 
 impl From<DatabaseError> for ApiError {
@@ -5210,9 +5207,6 @@ impl Service {
                 match res {
                     Ok(ok) => Ok(ok),
                     Err(mgmt_api::Error::ApiError(StatusCode::CONFLICT, _)) => Ok(StatusCode::CONFLICT),
-                    Err(mgmt_api::Error::ApiError(StatusCode::PRECONDITION_FAILED, msg)) if msg.contains("Requested tenant is missing") => {
-                        Err(ApiError::ResourceUnavailable("Tenant migration in progress".into()))
-                    },
                     Err(mgmt_api::Error::ApiError(StatusCode::SERVICE_UNAVAILABLE, msg)) => Err(ApiError::ResourceUnavailable(msg.into())),
                     Err(e) => {
                         Err(
@@ -5493,92 +5487,6 @@ impl Service {
         .ok_or_else(|| ApiError::NotFound(anyhow::anyhow!("Tenant {tenant_id} not found").into()))
     }
 
-    /* BEGIN_HADRON */
-    pub(crate) async fn tenant_timeline_describe(
-        &self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-    ) -> Result<TenantTimelineDescribeResponse, ApiError> {
-        self.tenant_remote_mutation(tenant_id, |locations| async move {
-            if locations.0.is_empty() {
-                return Err(ApiError::NotFound(
-                    anyhow::anyhow!("Tenant not found").into(),
-                ));
-            };
-
-            let locations: Vec<(TenantShardId, Node)> = locations
-                .0
-                .iter()
-                .map(|t| (*t.0, t.1.latest.node.clone()))
-                .collect();
-            let mut futs = FuturesUnordered::new();
-
-            for (shard_id, node) in locations {
-                futs.push({
-                    async move {
-                        let result = node
-                            .with_client_retries(
-                                |client| async move {
-                                    client
-                                        .tenant_timeline_describe(&shard_id, &timeline_id)
-                                        .await
-                                },
-                                &self.http_client,
-                                &self.config.pageserver_jwt_token,
-                                3,
-                                3,
-                                Duration::from_secs(30),
-                                &self.cancel,
-                            )
-                            .await;
-                        (result, shard_id, node.get_id())
-                    }
-                });
-            }
-
-            let mut results: Vec<TimelineInfo> = Vec::new();
-            while let Some((result, tenant_shard_id, node_id)) = futs.next().await {
-                match result {
-                    Some(Ok(timeline_info)) => results.push(timeline_info),
-                    Some(Err(e)) => {
-                        tracing::warn!(
-                            "Failed to describe tenant {} timeline {} for pageserver {}: {e}",
-                            tenant_shard_id,
-                            timeline_id,
-                            node_id,
-                        );
-                        return Err(ApiError::ResourceUnavailable(format!("{e}").into()));
-                    }
-                    None => return Err(ApiError::Cancelled),
-                }
-            }
-            let mut image_consistent_lsn: Option<Lsn> = Some(Lsn::MAX);
-            for timeline_info in &results {
-                if let Some(tline_image_consistent_lsn) = timeline_info.image_consistent_lsn {
-                    image_consistent_lsn = Some(std::cmp::min(
-                        image_consistent_lsn.unwrap(),
-                        tline_image_consistent_lsn,
-                    ));
-                } else {
-                    tracing::warn!(
-                        "Timeline {} on shard {} does not have image consistent lsn",
-                        timeline_info.timeline_id,
-                        timeline_info.tenant_id
-                    );
-                    image_consistent_lsn = None;
-                    break;
-                }
-            }
-
-            Ok(TenantTimelineDescribeResponse {
-                shards: results,
-                image_consistent_lsn,
-            })
-        })
-        .await?
-    }
-    /* END_HADRON */
-
     /// limit & offset are pagination parameters. Since we are walking an in-memory HashMap, `offset` does not
     /// avoid traversing data, it just avoid returning it. This is suitable for our purposes, since our in memory
     /// maps are small enough to traverse fast, our pagination is just to avoid serializing huge JSON responses
@@ -6119,7 +6027,7 @@ impl Service {
         tenant_id: TenantId,
         split_req: TenantShardSplitRequest,
     ) -> Result<ShardSplitAction, ApiError> {
-        fail::fail_point!("shard-split-validation", |_| Err(ApiError::BadRequest(
+        fail::fail_point_sync!("shard-split-validation", |_| Err(ApiError::BadRequest(
             anyhow::anyhow!("failpoint")
         )));
 
@@ -6410,39 +6318,18 @@ impl Service {
         // TODO: issue split calls concurrently (this only matters once we're splitting
         // N>1 shards into M shards -- initially we're usually splitting 1 shard into N).
 
-        // HADRON: set a timeout for splitting individual shards on page servers.
-        // Currently we do not perform any retry because it's not clear if page server can handle
-        // partially split shards correctly.
-        let shard_split_timeout =
-            if let Some(env::DeploymentMode::Local) = env::get_deployment_mode() {
-                Duration::from_secs(30)
-            } else {
-                self.config.shard_split_request_timeout
-            };
-        let mut http_client_builder = reqwest::ClientBuilder::new()
-            .pool_max_idle_per_host(0)
-            .timeout(shard_split_timeout);
-
-        for ssl_ca_cert in &self.config.ssl_ca_certs {
-            http_client_builder = http_client_builder.add_root_certificate(ssl_ca_cert.clone());
-        }
-        let http_client = http_client_builder
-            .build()
-            .expect("Failed to construct HTTP client");
         for target in &targets {
             let ShardSplitTarget {
                 parent_id,
                 node,
                 child_ids,
             } = target;
-
             let client = PageserverClient::new(
                 node.get_id(),
-                http_client.clone(),
+                self.http_client.clone(),
                 node.base_url(),
                 self.config.pageserver_jwt_token.as_deref(),
             );
-
             let response = client
                 .tenant_shard_split(
                     *parent_id,

storage_controller/src/service/safekeeper_service.rs

@@ -25,8 +25,7 @@ use pageserver_api::models::{SafekeeperInfo, SafekeepersInfo, TimelineInfo};
 use safekeeper_api::PgVersionId;
 use safekeeper_api::membership::{self, MemberSet, SafekeeperGeneration};
 use safekeeper_api::models::{
-    PullTimelineRequest, TimelineLocateResponse, TimelineMembershipSwitchRequest,
-    TimelineMembershipSwitchResponse,
+    PullTimelineRequest, TimelineMembershipSwitchRequest, TimelineMembershipSwitchResponse,
 };
 use safekeeper_api::{INITIAL_TERM, Term};
 use safekeeper_client::mgmt_api;
@@ -38,14 +37,21 @@ use utils::lsn::Lsn;
 
 use super::Service;
 
+#[derive(serde::Serialize, serde::Deserialize, Clone)]
+pub struct TimelineLocateResponse {
+    pub generation: SafekeeperGeneration,
+    pub sk_set: Vec<NodeId>,
+    pub new_sk_set: Option<Vec<NodeId>>,
+}
+
 impl Service {
-    fn make_member_set(safekeepers: &[Safekeeper]) -> Result<MemberSet, anyhow::Error> {
+    fn make_member_set(safekeepers: &[Safekeeper]) -> Result<MemberSet, ApiError> {
         let members = safekeepers
             .iter()
             .map(|sk| sk.get_safekeeper_id())
             .collect::<Vec<_>>();
 
-        MemberSet::new(members)
+        MemberSet::new(members).map_err(ApiError::InternalServerError)
     }
 
     fn get_safekeepers(&self, ids: &[i64]) -> Result<Vec<Safekeeper>, ApiError> {
@@ -80,7 +86,7 @@ impl Service {
     ) -> Result<Vec<NodeId>, ApiError> {
         let safekeepers = self.get_safekeepers(&timeline_persistence.sk_set)?;
 
-        let mset = Self::make_member_set(&safekeepers).map_err(ApiError::InternalServerError)?;
+        let mset = Self::make_member_set(&safekeepers)?;
         let mconf = safekeeper_api::membership::Configuration::new(mset);
 
         let req = safekeeper_api::models::TimelineCreateRequest {
@@ -1105,26 +1111,6 @@ impl Service {
             }
         }
 
-        if new_sk_set.is_empty() {
-            return Err(ApiError::BadRequest(anyhow::anyhow!(
-                "new safekeeper set is empty"
-            )));
-        }
-
-        if new_sk_set.len() < self.config.timeline_safekeeper_count {
-            return Err(ApiError::BadRequest(anyhow::anyhow!(
-                "new safekeeper set must have at least {} safekeepers",
-                self.config.timeline_safekeeper_count
-            )));
-        }
-
-        let new_sk_set_i64 = new_sk_set.iter().map(|id| id.0 as i64).collect::<Vec<_>>();
-        let new_safekeepers = self.get_safekeepers(&new_sk_set_i64)?;
-        // Construct new member set in advance to validate it.
-        // E.g. validates that there is no duplicate safekeepers.
-        let new_sk_member_set =
-            Self::make_member_set(&new_safekeepers).map_err(ApiError::BadRequest)?;
-
         // TODO(diko): per-tenant lock is too wide. Consider introducing per-timeline locks.
         let _tenant_lock = trace_shared_lock(
             &self.tenant_op_locks,
@@ -1155,18 +1141,6 @@ impl Service {
             .map(|&id| NodeId(id as u64))
             .collect::<Vec<_>>();
 
-        // Validate that we are not migrating to a decomissioned safekeeper.
-        for sk in new_safekeepers.iter() {
-            if !cur_sk_set.contains(&sk.get_id())
-                && sk.scheduling_policy() == SkSchedulingPolicy::Decomissioned
-            {
-                return Err(ApiError::BadRequest(anyhow::anyhow!(
-                    "safekeeper {} is decomissioned",
-                    sk.get_id()
-                )));
-            }
-        }
-
         tracing::info!(
             ?cur_sk_set,
             ?new_sk_set,
@@ -1209,8 +1183,11 @@ impl Service {
         }
 
         let cur_safekeepers = self.get_safekeepers(&timeline.sk_set)?;
-        let cur_sk_member_set =
-            Self::make_member_set(&cur_safekeepers).map_err(ApiError::InternalServerError)?;
+        let cur_sk_member_set = Self::make_member_set(&cur_safekeepers)?;
+
+        let new_sk_set_i64 = new_sk_set.iter().map(|id| id.0 as i64).collect::<Vec<_>>();
+        let new_safekeepers = self.get_safekeepers(&new_sk_set_i64)?;
+        let new_sk_member_set = Self::make_member_set(&new_safekeepers)?;
 
         let joint_config = membership::Configuration {
             generation,

test_runner/fixtures/endpoint/http.py

@@ -159,16 +159,59 @@ class EndpointHttpClient(requests.Session):
         res.raise_for_status()
         return res.json()
 
-    def configure_failpoints(self, *args: tuple[str, str]) -> None:
-        body: list[dict[str, str]] = []
+    def configure_failpoints(
+        self, *args: tuple[str, str] | list[dict[str, str | dict[str, str]]]
+    ) -> None:
+        """Configure failpoints for testing purposes.
 
-        for fp in args:
-            body.append(
-                {
-                    "name": fp[0],
-                    "action": fp[1],
+        Args:
+            *args: Can be one of:
+                - Variable number of (name, actions) tuples
+                - Single list of dicts with keys: name, actions, and optionally context_matchers
+
+        Examples:
+            # Basic failpoints
+            client.configure_failpoints(("test_fp", "return(error)"))
+            client.configure_failpoints(("fp1", "return"), ("fp2", "sleep(1000)"))
+
+            # Probability-based failpoint
+            client.configure_failpoints(("test_fp", "50%return(error)"))
+
+            # Context-based failpoint
+            client.configure_failpoints([{
+                "name": "test_fp",
+                "actions": "return(error)",
+                "context_matchers": {"tenant_id": ".*test.*"}
+            }])
+        """
+
+        request_body: list[dict[str, Any]] = []
+
+        if (
+            len(args) == 1
+            and isinstance(args[0], list)
+            and args[0]
+            and isinstance(args[0][0], dict)
+        ):
+            # Handle list of dicts (context-based failpoints)
+            failpoint_configs = args[0]
+            for config in failpoint_configs:
+                server_config: dict[str, Any] = {
+                    "name": config["name"],
+                    "actions": config["actions"],
                 }
-            )
+                if "context_matchers" in config:
+                    server_config["context_matchers"] = config["context_matchers"]
+                request_body.append(server_config)
+        else:
+            # Handle tuples (basic failpoints)
+            for fp in args:
+                request_body.append(
+                    {
+                        "name": fp[0],
+                        "actions": fp[1],
+                    }
+                )
 
-        res = self.post(f"http://localhost:{self.internal_port}/failpoints", json=body)
+        res = self.post(f"http://localhost:{self.internal_port}/failpoints", json=request_body)
         res.raise_for_status()

test_runner/fixtures/neon_fixtures.py

@@ -2342,20 +2342,6 @@ class NeonStorageController(MetricsGetter, LogUtils):
         response.raise_for_status()
         return response.json()
 
-    # HADRON
-    def tenant_timeline_describe(
-        self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-    ):
-        response = self.request(
-            "GET",
-            f"{self.api}/control/v1/tenant/{tenant_id}/timeline/{timeline_id}",
-            headers=self.headers(TokenScope.ADMIN),
-        )
-        response.raise_for_status()
-        return response.json()
-
     def nodes(self):
         """
         :return: list of {"id": ""}
@@ -2628,22 +2614,68 @@ class NeonStorageController(MetricsGetter, LogUtils):
         )
         return res.json()
 
-    def configure_failpoints(self, config_strings: tuple[str, str] | list[tuple[str, str]]):
-        if isinstance(config_strings, tuple):
-            pairs = [config_strings]
-        else:
-            pairs = config_strings
+    def configure_failpoints(
+        self,
+        config_strings: tuple[str, str]
+        | list[tuple[str, str]]
+        | list[dict[str, str | dict[str, str]]],
+    ):
+        """
+        Configure failpoints for testing purposes.
 
-        log.info(f"Requesting config failpoints: {repr(pairs)}")
+        Args:
+            config_strings: Can be one of:
+                - Single tuple of (name, actions)
+                - List of tuples [(name, actions), ...]
+                - List of dicts with keys: name, actions, and optionally context_matchers
+
+        Examples:
+            # Basic failpoint
+            client.configure_failpoints(("test_fp", "return(error)"))
+
+            # Multiple basic failpoints
+            client.configure_failpoints([("fp1", "return"), ("fp2", "sleep(1000)")])
+
+            # Probability-based failpoint
+            client.configure_failpoints(("test_fp", "50%return(error)"))
+
+            # Context-based failpoint
+            client.configure_failpoints([{
+                "name": "test_fp",
+                "actions": "return(error)",
+                "context_matchers": {"tenant_id": ".*test.*"}
+            }])
+        """
+        # Handle single tuple case
+        if isinstance(config_strings, tuple):
+            config_strings = [config_strings]
+
+        # Convert to server format
+        body: list[dict[str, str | dict[str, str]]] = []
+        for config in config_strings:
+            if isinstance(config, tuple):
+                # Simple (name, actions) tuple
+                body.append({"name": config[0], "actions": config[1]})
+            elif isinstance(config, dict):
+                # Dict with name, actions, and optional context_matchers
+                server_config: dict[str, str | dict[str, str]] = {
+                    "name": config["name"],
+                    "actions": config["actions"],
+                }
+                if "context_matchers" in config:
+                    server_config["context_matchers"] = config["context_matchers"]
+                body.append(server_config)
+            else:
+                raise ValueError(f"Invalid config format: {config}")
 
         res = self.request(
             "PUT",
-            f"{self.api}/debug/v1/failpoints",
-            json=[{"name": name, "actions": actions} for name, actions in pairs],
+            f"{self.api_root()}/debug/v1/failpoints",
+            json=body,
             headers=self.headers(TokenScope.ADMIN),
         )
-        log.info(f"Got failpoints request response code {res.status_code}")
-        res.raise_for_status()
+        if res.status_code != 200:
+            self.raise_api_exception(res)
 
     def get_tenants_placement(self) -> defaultdict[str, dict[str, Any]]:
         """
@@ -5409,7 +5441,6 @@ SKIP_FILES = frozenset(
     (
         "pg_internal.init",
         "pg.log",
-        "neon.signal",
         "zenith.signal",
         "pg_hba.conf",
         "postgresql.conf",

test_runner/fixtures/pageserver/allowed_errors.py

@@ -115,7 +115,8 @@ DEFAULT_PAGESERVER_ALLOWED_ERRORS = (
     ".*Local data loss suspected.*",
     # Too many frozen layers error is normal during intensive benchmarks
     ".*too many frozen layers.*",
-    ".*Failed to resolve tenant shard after.*",
+    # Transient errors when resolving tenant shards by page service
+    ".*Fail to resolve tenant shard in attempt.*",
     # Expected warnings when pageserver has not refreshed GC info yet
     ".*pitr LSN/interval not found, skipping force image creation LSN calculation.*",
     ".*No broker updates received for a while.*",

test_runner/fixtures/pageserver/http.py

@@ -309,25 +309,64 @@ class PageserverHttpClient(requests.Session, MetricsGetter):
     def check_status(self):
         self.get(f"http://localhost:{self.port}/v1/status").raise_for_status()
 
-    def configure_failpoints(self, config_strings: tuple[str, str] | list[tuple[str, str]]):
+    def configure_failpoints(
+        self,
+        config_strings: tuple[str, str]
+        | list[tuple[str, str]]
+        | list[dict[str, str | dict[str, str]]],
+    ):
+        """
+        Configure failpoints for testing purposes.
+
+        Args:
+            config_strings: Can be one of:
+                - Single tuple of (name, actions)
+                - List of tuples [(name, actions), ...]
+                - List of dicts with keys: name, actions, and optionally context_matchers
+
+        Examples:
+            # Basic failpoint
+            client.configure_failpoints(("test_fp", "return(error)"))
+
+            # Multiple basic failpoints
+            client.configure_failpoints([("fp1", "return"), ("fp2", "sleep(1000)")])
+
+            # Probability-based failpoint
+            client.configure_failpoints(("test_fp", "50%return(error)"))
+
+            # Context-based failpoint
+            client.configure_failpoints([{
+                "name": "test_fp",
+                "actions": "return(error)",
+                "context_matchers": {"tenant_id": ".*test.*"}
+            }])
+        """
         self.is_testing_enabled_or_skip()
 
+        # Handle single tuple case
         if isinstance(config_strings, tuple):
-            pairs = [config_strings]
-        else:
-            pairs = config_strings
+            config_strings = [config_strings]
 
-        log.info(f"Requesting config failpoints: {repr(pairs)}")
+        # Convert to server format
+        body: list[dict[str, str | dict[str, str]]] = []
+        for config in config_strings:
+            if isinstance(config, tuple):
+                # Simple (name, actions) tuple
+                body.append({"name": config[0], "actions": config[1]})
+            elif isinstance(config, dict):
+                # Dict with name, actions, and optional context_matchers
+                server_config = {"name": config["name"], "actions": config["actions"]}
+                if "context_matchers" in config:
+                    server_config["context_matchers"] = config["context_matchers"]
+                body.append(server_config)
+            else:
+                raise ValueError(f"Invalid config format: {config}")
 
-        res = self.put(
-            f"http://localhost:{self.port}/v1/failpoints",
-            json=[{"name": name, "actions": actions} for name, actions in pairs],
-        )
-        log.info(f"Got failpoints request response code {res.status_code}")
-        self.verbose_error(res)
-        res_json = res.json()
-        assert res_json is None
-        return res_json
+        res = self.post(f"{self.base_url}/v1/failpoints", json=body)
+        if res.status_code != 200:
+            raise PageserverApiException(
+                f"Failed to configure failpoints: {res.text}", res.status_code
+            )
 
     def reload_auth_validation_keys(self):
         res = self.post(f"http://localhost:{self.port}/v1/reload_auth_validation_keys")

test_runner/fixtures/safekeeper/http.py

@@ -8,7 +8,6 @@ import pytest
 import requests
 
 from fixtures.common_types import Lsn, TenantId, TenantTimelineId, TimelineId
-from fixtures.log_helper import log
 from fixtures.metrics import Metrics, MetricsGetter, parse_metrics
 from fixtures.utils import EnhancedJSONEncoder, wait_until
 
@@ -155,25 +154,62 @@ class SafekeeperHttpClient(requests.Session, MetricsGetter):
         if not self.is_testing_enabled:
             pytest.skip("safekeeper was built without 'testing' feature")
 
-    def configure_failpoints(self, config_strings: tuple[str, str] | list[tuple[str, str]]):
+    def configure_failpoints(
+        self,
+        config_strings: tuple[str, str]
+        | list[tuple[str, str]]
+        | list[dict[str, str | dict[str, str]]],
+    ):
+        """
+        Configure failpoints for testing purposes.
+
+        Args:
+            config_strings: Can be one of:
+                - Single tuple of (name, actions)
+                - List of tuples [(name, actions), ...]
+                - List of dicts with keys: name, actions, and optionally context_matchers
+
+        Examples:
+            # Basic failpoint
+            client.configure_failpoints(("test_fp", "return(error)"))
+
+            # Multiple basic failpoints
+            client.configure_failpoints([("fp1", "return"), ("fp2", "sleep(1000)")])
+
+            # Probability-based failpoint
+            client.configure_failpoints(("test_fp", "50%return(error)"))
+
+            # Context-based failpoint
+            client.configure_failpoints([{
+                "name": "test_fp",
+                "actions": "return(error)",
+                "context_matchers": {"tenant_id": ".*test.*"}
+            }])
+        """
         self.is_testing_enabled_or_skip()
 
+        # Handle single tuple case
         if isinstance(config_strings, tuple):
-            pairs = [config_strings]
-        else:
-            pairs = config_strings
+            config_strings = [config_strings]
 
-        log.info(f"Requesting config failpoints: {repr(pairs)}")
+        # Convert to server format
+        body: list[dict[str, str | dict[str, str]]] = []
+        for config in config_strings:
+            if isinstance(config, tuple):
+                # Simple (name, actions) tuple
+                body.append({"name": config[0], "actions": config[1]})
+            elif isinstance(config, dict):
+                # Dict with name, actions, and optional context_matchers
+                server_config = {"name": config["name"], "actions": config["actions"]}
+                if "context_matchers" in config:
+                    server_config["context_matchers"] = config["context_matchers"]
+                body.append(server_config)
+            else:
+                raise ValueError(f"Invalid config format: {config}")
 
-        res = self.put(
-            f"http://localhost:{self.port}/v1/failpoints",
-            json=[{"name": name, "actions": actions} for name, actions in pairs],
-        )
-        log.info(f"Got failpoints request response code {res.status_code}")
-        res.raise_for_status()
-        res_json = res.json()
-        assert res_json is None
-        return res_json
+        res = self.post(f"http://localhost:{self.port}/v1/failpoints", json=body)
+        if res.status_code != 200:
+            raise RuntimeError(f"Failed to configure failpoints: {res.text}")
 
     def tenant_delete_force(self, tenant_id: TenantId) -> dict[Any, Any]:
         res = self.delete(f"http://localhost:{self.port}/v1/tenant/{tenant_id}")

test_runner/regress/test_compaction.py

@@ -7,7 +7,6 @@ import time
 from enum import StrEnum
 
 import pytest
-from fixtures.common_types import TenantShardId
 from fixtures.log_helper import log
 from fixtures.neon_fixtures import (
     NeonEnvBuilder,
@@ -961,70 +960,9 @@ def get_layer_map(env, tenant_shard_id, timeline_id, ps_id):
     return image_layer_count, delta_layer_count
 
 
-def test_image_layer_creation_time_threshold(neon_env_builder: NeonEnvBuilder):
+def test_image_creation_timeout(neon_env_builder: NeonEnvBuilder):
     """
-    Tests that image layers can be created when the time threshold is reached on non-0 shards.
-    """
-    tenant_conf = {
-        "compaction_threshold": "100",
-        "image_creation_threshold": "100",
-        "image_layer_creation_check_threshold": "1",
-        # disable distance based image layer creation check
-        "checkpoint_distance": 10 * 1024 * 1024 * 1024,
-        "checkpoint_timeout": "100ms",
-        "image_layer_force_creation_period": "1s",
-        "pitr_interval": "10s",
-        "gc_period": "1s",
-        "compaction_period": "1s",
-        "lsn_lease_length": "1s",
-    }
-
-    # consider every tenant large to run the image layer generation check more eagerly
-    neon_env_builder.pageserver_config_override = (
-        "image_layer_generation_large_timeline_threshold=0"
-    )
-
-    neon_env_builder.num_pageservers = 1
-    neon_env_builder.num_safekeepers = 1
-    env = neon_env_builder.init_start(
-        initial_tenant_conf=tenant_conf,
-        initial_tenant_shard_count=2,
-        initial_tenant_shard_stripe_size=1,
-    )
-
-    tenant_id = env.initial_tenant
-    timeline_id = env.initial_timeline
-    endpoint = env.endpoints.create_start("main")
-    endpoint.safe_psql("CREATE TABLE foo (id INTEGER, val text)")
-
-    for v in range(10):
-        endpoint.safe_psql(f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))")
-
-    tenant_shard_id = TenantShardId(tenant_id, 1, 2)
-
-    # Generate some rows.
-    for v in range(20):
-        endpoint.safe_psql(f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))")
-
-    # restart page server so that logical size on non-0 shards is missing
-    env.pageserver.restart()
-
-    (old_images, old_deltas) = get_layer_map(env, tenant_shard_id, timeline_id, 0)
-    log.info(f"old images: {old_images}, old deltas: {old_deltas}")
-
-    def check_image_creation():
-        (new_images, old_deltas) = get_layer_map(env, tenant_shard_id, timeline_id, 0)
-        log.info(f"images: {new_images}, deltas: {old_deltas}")
-        assert new_images > old_images
-
-    wait_until(check_image_creation)
-
-    endpoint.stop_and_destroy()
-
-
-def test_image_layer_force_creation_period(neon_env_builder: NeonEnvBuilder):
-    """
-    Tests that page server can force creating new images if image_layer_force_creation_period is enabled
+    Tests that page server can force creating new images if image creation timeout is enabled
     """
     # use large knobs to disable L0 compaction/image creation except for the force image creation
     tenant_conf = {
@@ -1034,10 +972,10 @@ def test_image_layer_force_creation_period(neon_env_builder: NeonEnvBuilder):
         "checkpoint_distance": 10 * 1024,
         "checkpoint_timeout": "1s",
         "image_layer_force_creation_period": "1s",
-        "pitr_interval": "10s",
-        "gc_period": "1s",
-        "compaction_period": "1s",
-        "lsn_lease_length": "1s",
+        # The lsn for forced image layer creations is calculated once every 10 minutes.
+        # Hence, drive compaction manually such that the test doesn't compute it at the
+        # wrong time.
+        "compaction_period": "0s",
     }
 
     # consider every tenant large to run the image layer generation check more eagerly
@@ -1080,69 +1018,4 @@ def test_image_layer_force_creation_period(neon_env_builder: NeonEnvBuilder):
     )
 
 
-def test_image_consistent_lsn(neon_env_builder: NeonEnvBuilder):
-    """
-    Test the /v1/tenant/<tenant_id>/timeline/<timeline_id> endpoint and the computation of image_consistent_lsn
-    """
-    # use large knobs to disable L0 compaction/image creation except for the force image creation
-    tenant_conf = {
-        "compaction_threshold": "100",
-        "image_creation_threshold": "100",
-        "image_layer_creation_check_threshold": "1",
-        "checkpoint_distance": 10 * 1024,
-        "checkpoint_timeout": "1s",
-        "image_layer_force_creation_period": "1s",
-        "pitr_interval": "10s",
-        "gc_period": "1s",
-        "compaction_period": "1s",
-        "lsn_lease_length": "1s",
-    }
-
-    neon_env_builder.num_pageservers = 2
-    neon_env_builder.num_safekeepers = 1
-    env = neon_env_builder.init_start(
-        initial_tenant_conf=tenant_conf,
-        initial_tenant_shard_count=4,
-        initial_tenant_shard_stripe_size=1,
-    )
-
-    tenant_id = env.initial_tenant
-    timeline_id = env.initial_timeline
-
-    endpoint = env.endpoints.create_start("main")
-    endpoint.safe_psql("CREATE TABLE foo (id INTEGER, val text)")
-    for v in range(10):
-        endpoint.safe_psql(
-            f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))", log_query=False
-        )
-
-    response = env.storage_controller.tenant_timeline_describe(tenant_id, timeline_id)
-    shards = response["shards"]
-    for shard in shards:
-        assert shard["image_consistent_lsn"] is not None
-    image_consistent_lsn = response["image_consistent_lsn"]
-    assert image_consistent_lsn is not None
-
-    # do more writes and wait for image_consistent_lsn to advance
-    for v in range(100):
-        endpoint.safe_psql(
-            f"INSERT INTO foo (id, val) VALUES ({v}, repeat('abcde{v:0>3}', 500))", log_query=False
-        )
-
-    def check_image_consistent_lsn_advanced():
-        response = env.storage_controller.tenant_timeline_describe(tenant_id, timeline_id)
-        new_image_consistent_lsn = response["image_consistent_lsn"]
-        shards = response["shards"]
-        for shard in shards:
-            print(f"shard {shard['tenant_id']} image_consistent_lsn{shard['image_consistent_lsn']}")
-        assert new_image_consistent_lsn != image_consistent_lsn
-
-    wait_until(check_image_consistent_lsn_advanced)
-
-    endpoint.stop_and_destroy()
-
-    for ps in env.pageservers:
-        ps.allowed_errors.append(".*created delta file of size.*larger than double of target.*")
-
-
 # END_HADRON

test_runner/regress/test_safekeeper_migration.py

@@ -2,9 +2,6 @@ from __future__ import annotations
 
 from typing import TYPE_CHECKING
 
-import pytest
-from fixtures.neon_fixtures import StorageControllerApiException
-
 if TYPE_CHECKING:
     from fixtures.neon_fixtures import NeonEnvBuilder
 
@@ -78,38 +75,3 @@ def test_safekeeper_migration_simple(neon_env_builder: NeonEnvBuilder):
     ep.start(safekeeper_generation=1, safekeepers=[3])
 
     assert ep.safe_psql("SELECT * FROM t") == [(i,) for i in range(1, 4)]
-
-
-def test_new_sk_set_validation(neon_env_builder: NeonEnvBuilder):
-    """
-    Test that safekeeper_migrate validates the new_sk_set before starting the migration.
-    """
-    neon_env_builder.num_safekeepers = 3
-    neon_env_builder.storage_controller_config = {
-        "timelines_onto_safekeepers": True,
-        "timeline_safekeeper_count": 2,
-    }
-    env = neon_env_builder.init_start()
-
-    def expect_fail(sk_set: list[int], match: str):
-        with pytest.raises(StorageControllerApiException, match=match):
-            env.storage_controller.migrate_safekeepers(
-                env.initial_tenant, env.initial_timeline, sk_set
-            )
-        # Check that we failed before commiting to the database.
-        mconf = env.storage_controller.timeline_locate(env.initial_tenant, env.initial_timeline)
-        assert mconf["generation"] == 1
-
-    expect_fail([], "safekeeper set is empty")
-    expect_fail([1], "must have at least 2 safekeepers")
-    expect_fail([1, 1], "duplicate safekeeper")
-    expect_fail([1, 100500], "does not exist")
-
-    mconf = env.storage_controller.timeline_locate(env.initial_tenant, env.initial_timeline)
-    sk_set = mconf["sk_set"]
-    assert len(sk_set) == 2
-
-    decom_sk = [sk.id for sk in env.safekeepers if sk.id not in sk_set][0]
-    env.storage_controller.safekeeper_scheduling_policy(decom_sk, "Decomissioned")
-
-    expect_fail([sk_set[0], decom_sk], "decomissioned")

test_runner/regress/test_sharding.py

@@ -1673,91 +1673,6 @@ def test_shard_resolve_during_split_abort(neon_env_builder: NeonEnvBuilder):
 # END_HADRON
 
 
-# HADRON
-@pytest.mark.skip(reason="The backpressure change has not been merged yet.")
-def test_back_pressure_per_shard(neon_env_builder: NeonEnvBuilder):
-    """
-    Tests back pressure knobs are enforced on the per shard basis instead of at the tenant level.
-    """
-    init_shard_count = 4
-    neon_env_builder.num_pageservers = init_shard_count
-    stripe_size = 1
-
-    env = neon_env_builder.init_start(
-        initial_tenant_shard_count=init_shard_count,
-        initial_tenant_shard_stripe_size=stripe_size,
-        initial_tenant_conf={
-            # disable auto-flush of shards and set max_replication_flush_lag as 15MB.
-            # The backpressure parameters must be enforced at the shard level to avoid stalling PG.
-            "checkpoint_distance": 1 * 1024 * 1024 * 1024,
-            "checkpoint_timeout": "1h",
-        },
-    )
-
-    endpoint = env.endpoints.create(
-        "main",
-        config_lines=[
-            "max_replication_write_lag = 0",
-            "max_replication_apply_lag = 0",
-            "max_replication_flush_lag = 15MB",
-            "neon.max_cluster_size = 10GB",
-        ],
-    )
-    endpoint.respec(skip_pg_catalog_updates=False)  # Needed for databricks_system to get created.
-    endpoint.start()
-
-    # generate 20MB of data
-    endpoint.safe_psql(
-        "CREATE TABLE usertable AS SELECT s AS KEY, repeat('a', 1000) as VALUE from generate_series(1, 20000) s;"
-    )
-    res = endpoint.safe_psql(
-        "SELECT neon.backpressure_throttling_time() as throttling_time", dbname="databricks_system"
-    )[0]
-    assert res[0] == 0, f"throttling_time should be 0, but got {res[0]}"
-
-    endpoint.stop()
-
-
-# HADRON
-def test_shard_split_page_server_timeout(neon_env_builder: NeonEnvBuilder):
-    """
-    Tests that shard split can correctly handle page server timeouts and abort the split
-    """
-    init_shard_count = 2
-    neon_env_builder.num_pageservers = 1
-    stripe_size = 1
-
-    if neon_env_builder.storage_controller_config is None:
-        neon_env_builder.storage_controller_config = {"shard_split_request_timeout": "5s"}
-    else:
-        neon_env_builder.storage_controller_config["shard_split_request_timeout"] = "5s"
-
-    env = neon_env_builder.init_start(
-        initial_tenant_shard_count=init_shard_count,
-        initial_tenant_shard_stripe_size=stripe_size,
-    )
-
-    env.storage_controller.allowed_errors.extend(
-        [
-            ".*Enqueuing background abort.*",
-            ".*failpoint.*",
-            ".*Failed to abort.*",
-            ".*Exclusive lock by ShardSplit was held.*",
-        ]
-    )
-    env.pageserver.allowed_errors.extend([".*request was dropped before completing.*"])
-
-    endpoint1 = env.endpoints.create_start(branch_name="main")
-
-    env.pageserver.http_client().configure_failpoints(("shard-split-post-finish-pause", "pause"))
-
-    with pytest.raises(StorageControllerApiException):
-        env.storage_controller.tenant_shard_split(env.initial_tenant, shard_count=4)
-
-    env.pageserver.http_client().configure_failpoints(("shard-split-post-finish-pause", "off"))
-    endpoint1.stop_and_destroy()
-
-
 def test_sharding_backpressure(neon_env_builder: NeonEnvBuilder):
     """
     Check a scenario when one of the shards is much slower than others.

test_runner/regress/test_timeline_detach_ancestor.py

@@ -209,9 +209,9 @@ def test_ancestor_detach_branched_from(
     client.timeline_delete(env.initial_tenant, env.initial_timeline)
     wait_timeline_detail_404(client, env.initial_tenant, env.initial_timeline)
 
-    # because we do the fullbackup from ancestor at the branch_lsn, the neon.signal and/or zenith.signal is always
-    # different as there is always "PREV_LSN: invalid" for "before"
-    skip_files = {"zenith.signal", "neon.signal"}
+    # because we do the fullbackup from ancestor at the branch_lsn, the zenith.signal is always different
+    # as there is always "PREV_LSN: invalid" for "before"
+    skip_files = {"zenith.signal"}
 
     assert_pageserver_backups_equal(fullbackup_before, fullbackup_after, skip_files)
 
@@ -767,7 +767,7 @@ def test_compaction_induced_by_detaches_in_history(
         env.pageserver, env.initial_tenant, branch_timeline_id, branch_lsn, fullbackup_after
     )
 
-    # we don't need to skip any files, because neon.signal will be identical
+    # we don't need to skip any files, because zenith.signal will be identical
     assert_pageserver_backups_equal(fullbackup_before, fullbackup_after, set())
 
 

vendor/revisions.json

@@ -1,18 +1,18 @@
 {
   "v17": [
     "17.5",
-    "353c725b0c76cc82b15af21d8360d03391dc6814"
+    "db424d42d748f8ad91ac00e28db2c7f2efa42f7f"
   ],
   "v16": [
     "16.9",
-    "e08c8d5f1576ca0487d14d154510499c5f12adfb"
+    "7a4c0eacaeb9b97416542fa19103061c166460b1"
   ],
   "v15": [
     "15.13",
-    "afd46987f3da50c9146a8aa59380052df0862c06"
+    "8c3249f36c7df6ac0efb8ee9f1baf4aa1b83e5c9"
   ],
   "v14": [
     "14.18",
-    "8ce1f52303aec29e098309347b57c01a1962e221"
+    "9085654ee8022d5cc4ca719380a1dc53e5e3246f"
   ]
 }