diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index caa7ebee05..820848b4fb 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -404,7 +404,7 @@ jobs: uses: ./.github/actions/save-coverage-data regress-tests: - needs: [ check-permissions, build-neon ] + needs: [ check-permissions, build-neon, tag ] runs-on: [ self-hosted, gen3, large ] container: image: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/rust:pinned @@ -436,6 +436,7 @@ jobs: env: TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} CHECK_ONDISK_DATA_COMPATIBILITY: nonempty + BUILD_TAG: ${{ needs.tag.outputs.build-tag }} - name: Merge and upload coverage data if: matrix.build_type == 'debug' && matrix.pg_version == 'v14' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3d6fc75e08..ba37c5827a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Create Release Branch on: schedule: - - cron: '0 7 * * 5' + - cron: '0 6 * * 1' workflow_dispatch: jobs: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5de7842f1a..2692684006 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -9,6 +9,24 @@ refactoring, additional comments, and so forth. Let's try to raise the bar, and clean things up as we go. Try to leave code in a better shape than it was before. +## Pre-commit hook + +We have a sample pre-commit hook in `pre-commit.py`. +To set it up, run: + +```bash +ln -s ../../pre-commit.py .git/hooks/pre-commit +``` + +This will run following checks on staged files before each commit: +- `rustfmt` +- checks for python files, see [obligatory checks](/docs/sourcetree.md#obligatory-checks). + +There is also a separate script `./run_clippy.sh` that runs `cargo clippy` on the whole project +and `./scripts/reformat` that runs all formatting tools to ensure the project is up to date. + +If you want to skip the hook, run `git commit` with `--no-verify` option. + ## Submitting changes 1. Get at least one +1 on your PR before you push. diff --git a/Cargo.lock b/Cargo.lock index b0c97d64e5..270fea8b48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -210,6 +210,8 @@ dependencies = [ "memchr", "pin-project-lite", "tokio", + "zstd", + "zstd-safe", ] [[package]] @@ -1158,6 +1160,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-compression", + "bytes", "cfg-if", "chrono", "clap", @@ -3037,6 +3040,8 @@ dependencies = [ "git-version", "pageserver", "postgres_ffi", + "serde", + "serde_json", "svg_fmt", "tokio", "utils", @@ -3139,6 +3144,7 @@ dependencies = [ "serde_with", "strum", "strum_macros", + "thiserror", "utils", "workspace_hack", ] @@ -3354,7 +3360,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#988d0ddb4184c408fa7fc1bd0ecca7993c02978f" dependencies = [ "bytes", "fallible-iterator", @@ -3367,7 +3373,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#988d0ddb4184c408fa7fc1bd0ecca7993c02978f" dependencies = [ "native-tls", "tokio", @@ -3378,7 +3384,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#988d0ddb4184c408fa7fc1bd0ecca7993c02978f" dependencies = [ "base64 0.20.0", "byteorder", @@ -3396,7 +3402,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#988d0ddb4184c408fa7fc1bd0ecca7993c02978f" dependencies = [ "bytes", "fallible-iterator", @@ -3652,6 +3658,7 @@ dependencies = [ "pbkdf2", "pin-project-lite", "postgres-native-tls", + "postgres-protocol", "postgres_backend", "pq_proto", "prometheus", @@ -5132,7 +5139,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=6ce32f791526e27533cab0232a6bb243b2c32584#6ce32f791526e27533cab0232a6bb243b2c32584" +source = "git+https://github.com/neondatabase/rust-postgres.git?branch=neon#988d0ddb4184c408fa7fc1bd0ecca7993c02978f" dependencies = [ "async-trait", "byteorder", @@ -6235,7 +6242,10 @@ dependencies = [ "tracing-core", "tungstenite", "url", - "uuid 1.3.3", + "uuid", + "zstd", + "zstd-safe", + "zstd-sys", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 7d7d74993e..c50fb7be42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,7 +37,7 @@ license = "Apache-2.0" [workspace.dependencies] anyhow = { version = "1.0", features = ["backtrace"] } arc-swap = "1.6" -async-compression = { version = "0.4.0", features = ["tokio", "gzip"] } +async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] } azure_core = "0.16" azure_identity = "0.16" azure_storage = "0.16" @@ -165,11 +165,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -206,7 +206,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="6ce32f791526e27533cab0232a6bb243b2c32584" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch="neon" } ################# Binary contents sections diff --git a/Dockerfile.compute-node b/Dockerfile.compute-node index 36c3f874d4..8109091e76 100644 --- a/Dockerfile.compute-node +++ b/Dockerfile.compute-node @@ -714,6 +714,24 @@ RUN wget https://github.com/pksunkara/pgx_ulid/archive/refs/tags/v0.1.3.tar.gz - cargo pgrx install --release && \ echo "trusted = true" >> /usr/local/pgsql/share/extension/ulid.control +######################################################################################### +# +# Layer "wal2json-build" +# Compile "wal2json" extension +# +######################################################################################### + +FROM build-deps AS wal2json-pg-build +COPY --from=pg-build /usr/local/pgsql/ /usr/local/pgsql/ + +ENV PATH "/usr/local/pgsql/bin/:$PATH" +RUN wget https://github.com/eulerto/wal2json/archive/refs/tags/wal2json_2_5.tar.gz && \ + echo "b516653575541cf221b99cf3f8be9b6821f6dbcfc125675c85f35090f824f00e wal2json_2_5.tar.gz" | sha256sum --check && \ + mkdir wal2json-src && cd wal2json-src && tar xvzf ../wal2json_2_5.tar.gz --strip-components=1 -C . && \ + make -j $(getconf _NPROCESSORS_ONLN) && \ + make -j $(getconf _NPROCESSORS_ONLN) install && \ + echo 'trusted = true' >> /usr/local/pgsql/share/extension/wal2json.control + ######################################################################################### # # Layer "neon-pg-ext-build" @@ -750,6 +768,7 @@ COPY --from=rdkit-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-uuidv7-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-roaringbitmap-pg-build /usr/local/pgsql/ /usr/local/pgsql/ COPY --from=pg-embedding-pg-build /usr/local/pgsql/ /usr/local/pgsql/ +COPY --from=wal2json-pg-build /usr/local/pgsql /usr/local/pgsql COPY pgxn/ pgxn/ RUN make -j $(getconf _NPROCESSORS_ONLN) \ diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index 6c93befaa3..47378f1910 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -38,3 +38,4 @@ toml_edit.workspace = true remote_storage = { version = "0.1", path = "../libs/remote_storage/" } vm_monitor = { version = "0.1", path = "../libs/vm_monitor/" } zstd = "0.12.4" +bytes = "1.0" diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index 7f22bda13e..36e9ca0731 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -31,7 +31,7 @@ //! -C 'postgresql://cloud_admin@localhost/postgres' \ //! -S /var/db/postgres/specs/current.json \ //! -b /usr/local/bin/postgres \ -//! -r {"bucket": "neon-dev-extensions-eu-central-1", "region": "eu-central-1"} +//! -r http://pg-ext-s3-gateway //! ``` //! use std::collections::HashMap; @@ -51,7 +51,7 @@ use compute_api::responses::ComputeStatus; use compute_tools::compute::{ComputeNode, ComputeState, ParsedSpec}; use compute_tools::configurator::launch_configurator; -use compute_tools::extension_server::{get_pg_version, init_remote_storage}; +use compute_tools::extension_server::get_pg_version; use compute_tools::http::api::launch_http_server; use compute_tools::logger::*; use compute_tools::monitor::launch_monitor; @@ -60,7 +60,7 @@ use compute_tools::spec::*; // this is an arbitrary build tag. Fine as a default / for testing purposes // in-case of not-set environment var -const BUILD_TAG_DEFAULT: &str = "5670669815"; +const BUILD_TAG_DEFAULT: &str = "latest"; fn main() -> Result<()> { init_tracing_and_logging(DEFAULT_LOG_LEVEL)?; @@ -74,10 +74,18 @@ fn main() -> Result<()> { let pgbin_default = String::from("postgres"); let pgbin = matches.get_one::("pgbin").unwrap_or(&pgbin_default); - let remote_ext_config = matches.get_one::("remote-ext-config"); - let ext_remote_storage = remote_ext_config.map(|x| { - init_remote_storage(x).expect("cannot initialize remote extension storage from config") - }); + let ext_remote_storage = matches + .get_one::("remote-ext-config") + // Compatibility hack: if the control plane specified any remote-ext-config + // use the default value for extension storage proxy gateway. + // Remove this once the control plane is updated to pass the gateway URL + .map(|conf| { + if conf.starts_with("http") { + conf.trim_end_matches('/') + } else { + "http://pg-ext-s3-gateway" + } + }); let http_port = *matches .get_one::("http-port") @@ -198,7 +206,7 @@ fn main() -> Result<()> { live_config_allowed, state: Mutex::new(new_state), state_changed: Condvar::new(), - ext_remote_storage, + ext_remote_storage: ext_remote_storage.map(|s| s.to_string()), ext_download_progress: RwLock::new(HashMap::new()), build_tag, }; diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index 373f05ab2f..0dfacb615c 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -25,7 +25,7 @@ use compute_api::responses::{ComputeMetrics, ComputeStatus}; use compute_api::spec::{ComputeMode, ComputeSpec}; use utils::measured_stream::MeasuredReader; -use remote_storage::{DownloadError, GenericRemoteStorage, RemotePath}; +use remote_storage::{DownloadError, RemotePath}; use crate::checker::create_availability_check_data; use crate::pg_helpers::*; @@ -59,8 +59,8 @@ pub struct ComputeNode { pub state: Mutex, /// `Condvar` to allow notifying waiters about state changes. pub state_changed: Condvar, - /// the S3 bucket that we search for extensions in - pub ext_remote_storage: Option, + /// the address of extension storage proxy gateway + pub ext_remote_storage: Option, // key: ext_archive_name, value: started download time, download_completed? pub ext_download_progress: RwLock, bool)>>, pub build_tag: String, @@ -698,6 +698,7 @@ impl ComputeNode { handle_role_deletions(spec, self.connstr.as_str(), &mut client)?; handle_grants(spec, &mut client, self.connstr.as_str())?; handle_extensions(spec, &mut client)?; + handle_extension_neon(&mut client)?; create_availability_check_data(&mut client)?; // 'Close' connection @@ -742,6 +743,7 @@ impl ComputeNode { handle_role_deletions(&spec, self.connstr.as_str(), &mut client)?; handle_grants(&spec, &mut client, self.connstr.as_str())?; handle_extensions(&spec, &mut client)?; + handle_extension_neon(&mut client)?; } // 'Close' connection @@ -955,12 +957,12 @@ LIMIT 100", real_ext_name: String, ext_path: RemotePath, ) -> Result { - let remote_storage = self - .ext_remote_storage - .as_ref() - .ok_or(DownloadError::BadInput(anyhow::anyhow!( - "Remote extensions storage is not configured", - )))?; + let ext_remote_storage = + self.ext_remote_storage + .as_ref() + .ok_or(DownloadError::BadInput(anyhow::anyhow!( + "Remote extensions storage is not configured", + )))?; let ext_archive_name = ext_path.object_name().expect("bad path"); @@ -1016,7 +1018,7 @@ LIMIT 100", let download_size = extension_server::download_extension( &real_ext_name, &ext_path, - remote_storage, + ext_remote_storage, &self.pgbin, ) .await diff --git a/compute_tools/src/extension_server.rs b/compute_tools/src/extension_server.rs index 9732d8adea..2cec12119f 100644 --- a/compute_tools/src/extension_server.rs +++ b/compute_tools/src/extension_server.rs @@ -71,18 +71,16 @@ More specifically, here is an example ext_index.json } } */ -use anyhow::Context; use anyhow::{self, Result}; +use anyhow::{bail, Context}; +use bytes::Bytes; use compute_api::spec::RemoteExtSpec; use regex::Regex; use remote_storage::*; -use serde_json; -use std::io::Read; -use std::num::NonZeroUsize; +use reqwest::StatusCode; use std::path::Path; use std::str; use tar::Archive; -use tokio::io::AsyncReadExt; use tracing::info; use tracing::log::warn; use zstd::stream::read::Decoder; @@ -138,23 +136,31 @@ fn parse_pg_version(human_version: &str) -> &str { pub async fn download_extension( ext_name: &str, ext_path: &RemotePath, - remote_storage: &GenericRemoteStorage, + ext_remote_storage: &str, pgbin: &str, ) -> Result { info!("Download extension {:?} from {:?}", ext_name, ext_path); - let mut download = remote_storage.download(ext_path).await?; - let mut download_buffer = Vec::new(); - download - .download_stream - .read_to_end(&mut download_buffer) - .await?; + + // TODO add retry logic + let download_buffer = + match download_extension_tar(ext_remote_storage, &ext_path.to_string()).await { + Ok(buffer) => buffer, + Err(error_message) => { + return Err(anyhow::anyhow!( + "error downloading extension {:?}: {:?}", + ext_name, + error_message + )); + } + }; + let download_size = download_buffer.len() as u64; + info!("Download size {:?}", download_size); // it's unclear whether it is more performant to decompress into memory or not // TODO: decompressing into memory can be avoided - let mut decoder = Decoder::new(download_buffer.as_slice())?; - let mut decompress_buffer = Vec::new(); - decoder.read_to_end(&mut decompress_buffer)?; - let mut archive = Archive::new(decompress_buffer.as_slice()); + let decoder = Decoder::new(download_buffer.as_ref())?; + let mut archive = Archive::new(decoder); + let unzip_dest = pgbin .strip_suffix("/bin/postgres") .expect("bad pgbin") @@ -222,29 +228,32 @@ pub fn create_control_files(remote_extensions: &RemoteExtSpec, pgbin: &str) { } } -// This function initializes the necessary structs to use remote storage -pub fn init_remote_storage(remote_ext_config: &str) -> anyhow::Result { - #[derive(Debug, serde::Deserialize)] - struct RemoteExtJson { - bucket: String, - region: String, - endpoint: Option, - prefix: Option, - } - let remote_ext_json = serde_json::from_str::(remote_ext_config)?; +// Do request to extension storage proxy, i.e. +// curl http://pg-ext-s3-gateway/latest/v15/extensions/anon.tar.zst +// using HHTP GET +// and return the response body as bytes +// +async fn download_extension_tar(ext_remote_storage: &str, ext_path: &str) -> Result { + let uri = format!("{}/{}", ext_remote_storage, ext_path); - let config = S3Config { - bucket_name: remote_ext_json.bucket, - bucket_region: remote_ext_json.region, - prefix_in_bucket: remote_ext_json.prefix, - endpoint: remote_ext_json.endpoint, - concurrency_limit: NonZeroUsize::new(100).expect("100 != 0"), - max_keys_per_list_response: None, - }; - let config = RemoteStorageConfig { - storage: RemoteStorageKind::AwsS3(config), - }; - GenericRemoteStorage::from_config(&config) + info!("Download extension {:?} from uri {:?}", ext_path, uri); + + let resp = reqwest::get(uri).await?; + + match resp.status() { + StatusCode::OK => match resp.bytes().await { + Ok(resp) => { + info!("Download extension {:?} completed successfully", ext_path); + Ok(resp) + } + Err(e) => bail!("could not deserialize remote extension response: {}", e), + }, + StatusCode::SERVICE_UNAVAILABLE => bail!("remote extension is temporarily unavailable"), + _ => bail!( + "unexpected remote extension response status code: {}", + resp.status() + ), + } } #[cfg(test)] diff --git a/compute_tools/src/http/api.rs b/compute_tools/src/http/api.rs index 8851be1ec1..ef6ca6eee3 100644 --- a/compute_tools/src/http/api.rs +++ b/compute_tools/src/http/api.rs @@ -123,7 +123,7 @@ async fn routes(req: Request, compute: &Arc) -> Response { info!("serving {:?} POST request", route); info!("req.uri {:?}", req.uri()); diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index a85d6287b1..8c44c6d519 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -674,3 +674,33 @@ pub fn handle_extensions(spec: &ComputeSpec, client: &mut Client) -> Result<()> Ok(()) } + +/// Run CREATE and ALTER EXTENSION neon UPDATE for postgres database +#[instrument(skip_all)] +pub fn handle_extension_neon(client: &mut Client) -> Result<()> { + info!("handle extension neon"); + + let mut query = "CREATE SCHEMA IF NOT EXISTS neon"; + client.simple_query(query)?; + + query = "CREATE EXTENSION IF NOT EXISTS neon WITH SCHEMA neon"; + info!("create neon extension with query: {}", query); + client.simple_query(query)?; + + query = "UPDATE pg_extension SET extrelocatable = true WHERE extname = 'neon'"; + client.simple_query(query)?; + + query = "ALTER EXTENSION neon SET SCHEMA neon"; + info!("alter neon extension schema with query: {}", query); + client.simple_query(query)?; + + // this will be a no-op if extension is already up to date, + // which may happen in two cases: + // - extension was just installed + // - extension was already installed and is up to date + let query = "ALTER EXTENSION neon UPDATE"; + info!("update neon extension schema with query: {}", query); + client.simple_query(query)?; + + Ok(()) +} diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index ddd50ff51d..16577e27d6 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -286,6 +286,7 @@ async fn main() -> anyhow::Result<()> { logging::init( LogFormat::Plain, logging::TracingErrorLayerEnablement::Disabled, + logging::Output::Stdout, )?; let args = Cli::parse(); diff --git a/control_plane/src/bin/neon_local.rs b/control_plane/src/bin/neon_local.rs index 87ea519a9e..3053122f6a 100644 --- a/control_plane/src/bin/neon_local.rs +++ b/control_plane/src/bin/neon_local.rs @@ -487,8 +487,15 @@ fn handle_timeline(timeline_match: &ArgMatches, env: &mut local_env::LocalEnv) - .copied() .context("Failed to parse postgres version from the argument string")?; - let timeline_info = - pageserver.timeline_create(tenant_id, None, None, None, Some(pg_version))?; + let new_timeline_id_opt = parse_timeline_id(create_match)?; + + let timeline_info = pageserver.timeline_create( + tenant_id, + new_timeline_id_opt, + None, + None, + Some(pg_version), + )?; let new_timeline_id = timeline_info.timeline_id; let last_record_lsn = timeline_info.last_record_lsn; @@ -1245,7 +1252,7 @@ fn cli() -> Command { let remote_ext_config_args = Arg::new("remote-ext-config") .long("remote-ext-config") .num_args(1) - .help("Configure the S3 bucket that we search for extensions in.") + .help("Configure the remote extensions storage proxy gateway to request for extensions.") .required(false); let lsn_arg = Arg::new("lsn") @@ -1308,6 +1315,7 @@ fn cli() -> Command { .subcommand(Command::new("create") .about("Create a new blank timeline") .arg(tenant_id_arg.clone()) + .arg(timeline_id_arg.clone()) .arg(branch_name_arg.clone()) .arg(pg_version_arg.clone()) ) diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index 4443fd8704..ae45746925 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -45,6 +45,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, bail, Context, Result}; +use compute_api::spec::RemoteExtSpec; use serde::{Deserialize, Serialize}; use utils::id::{NodeId, TenantId, TimelineId}; @@ -476,6 +477,18 @@ impl Endpoint { } } + // check for file remote_extensions_spec.json + // if it is present, read it and pass to compute_ctl + let remote_extensions_spec_path = self.endpoint_path().join("remote_extensions_spec.json"); + let remote_extensions_spec = std::fs::File::open(remote_extensions_spec_path); + let remote_extensions: Option; + + if let Ok(spec_file) = remote_extensions_spec { + remote_extensions = serde_json::from_reader(spec_file).ok(); + } else { + remote_extensions = None; + }; + // Create spec file let spec = ComputeSpec { skip_pg_catalog_updates: self.skip_pg_catalog_updates, @@ -497,7 +510,7 @@ impl Endpoint { pageserver_connstring: Some(pageserver_connstring), safekeeper_connstrings, storage_auth_token: auth_token.clone(), - remote_extensions: None, + remote_extensions, }; let spec_path = self.endpoint_path().join("spec.json"); std::fs::write(spec_path, serde_json::to_string_pretty(&spec)?)?; diff --git a/control_plane/src/tenant_migration.rs b/control_plane/src/tenant_migration.rs index d28d1f9fe8..42780db85c 100644 --- a/control_plane/src/tenant_migration.rs +++ b/control_plane/src/tenant_migration.rs @@ -14,7 +14,6 @@ use pageserver_api::models::{ use std::collections::HashMap; use std::time::Duration; use utils::{ - generation::Generation, id::{TenantId, TimelineId}, lsn::Lsn, }; @@ -93,6 +92,22 @@ pub fn migrate_tenant( // Get a new generation let attachment_service = AttachmentService::from_env(env); + fn build_location_config( + mode: LocationConfigMode, + generation: Option, + secondary_conf: Option, + ) -> LocationConfig { + LocationConfig { + mode, + generation, + secondary_conf, + tenant_conf: TenantConfig::default(), + shard_number: 0, + shard_count: 0, + shard_stripe_size: 0, + } + } + let previous = attachment_service.inspect(tenant_id)?; let mut baseline_lsns = None; if let Some((generation, origin_ps_id)) = &previous { @@ -101,12 +116,7 @@ pub fn migrate_tenant( if origin_ps_id == &dest_ps.conf.id { println!("🔁 Already attached to {origin_ps_id}, freshening..."); let gen = attachment_service.attach_hook(tenant_id, dest_ps.conf.id)?; - let dest_conf = LocationConfig { - mode: LocationConfigMode::AttachedSingle, - generation: gen.map(Generation::new), - secondary_conf: None, - tenant_conf: TenantConfig::default(), - }; + let dest_conf = build_location_config(LocationConfigMode::AttachedSingle, gen, None); dest_ps.location_config(tenant_id, dest_conf)?; println!("✅ Migration complete"); return Ok(()); @@ -114,24 +124,15 @@ pub fn migrate_tenant( println!("🔁 Switching origin pageserver {origin_ps_id} to stale mode"); - let stale_conf = LocationConfig { - mode: LocationConfigMode::AttachedStale, - generation: Some(Generation::new(*generation)), - secondary_conf: None, - tenant_conf: TenantConfig::default(), - }; + let stale_conf = + build_location_config(LocationConfigMode::AttachedStale, Some(*generation), None); origin_ps.location_config(tenant_id, stale_conf)?; baseline_lsns = Some(get_lsns(tenant_id, &origin_ps)?); } let gen = attachment_service.attach_hook(tenant_id, dest_ps.conf.id)?; - let dest_conf = LocationConfig { - mode: LocationConfigMode::AttachedMulti, - generation: gen.map(Generation::new), - secondary_conf: None, - tenant_conf: TenantConfig::default(), - }; + let dest_conf = build_location_config(LocationConfigMode::AttachedMulti, gen, None); println!("🔁 Attaching to pageserver {}", dest_ps.conf.id); dest_ps.location_config(tenant_id, dest_conf)?; @@ -170,12 +171,11 @@ pub fn migrate_tenant( } // Downgrade to a secondary location - let secondary_conf = LocationConfig { - mode: LocationConfigMode::Secondary, - generation: None, - secondary_conf: Some(LocationConfigSecondary { warm: true }), - tenant_conf: TenantConfig::default(), - }; + let secondary_conf = build_location_config( + LocationConfigMode::Secondary, + None, + Some(LocationConfigSecondary { warm: true }), + ); println!( "💤 Switching to secondary mode on pageserver {}", @@ -188,12 +188,7 @@ pub fn migrate_tenant( "🔁 Switching to AttachedSingle mode on pageserver {}", dest_ps.conf.id ); - let dest_conf = LocationConfig { - mode: LocationConfigMode::AttachedSingle, - generation: gen.map(Generation::new), - secondary_conf: None, - tenant_conf: TenantConfig::default(), - }; + let dest_conf = build_location_config(LocationConfigMode::AttachedSingle, gen, None); dest_ps.location_config(tenant_id, dest_conf)?; println!("✅ Migration complete"); diff --git a/libs/pageserver_api/Cargo.toml b/libs/pageserver_api/Cargo.toml index df9796b039..4d08d78e87 100644 --- a/libs/pageserver_api/Cargo.toml +++ b/libs/pageserver_api/Cargo.toml @@ -18,6 +18,7 @@ enum-map.workspace = true strum.workspace = true strum_macros.workspace = true hex.workspace = true +thiserror.workspace = true workspace_hack.workspace = true diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 71e32e479f..c792a5eff7 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -10,7 +10,6 @@ use serde_with::serde_as; use strum_macros; use utils::{ completion, - generation::Generation, history_buffer::HistoryBufferWithDropCounter, id::{NodeId, TenantId, TimelineId}, lsn::Lsn, @@ -262,10 +261,19 @@ pub struct LocationConfig { pub mode: LocationConfigMode, /// If attaching, in what generation? #[serde(default)] - pub generation: Option, + pub generation: Option, #[serde(default)] pub secondary_conf: Option, + // Shard parameters: if shard_count is nonzero, then other shard_* fields + // must be set accurately. + #[serde(default)] + pub shard_number: u8, + #[serde(default)] + pub shard_count: u8, + #[serde(default)] + pub shard_stripe_size: u32, + // If requesting mode `Secondary`, configuration for that. // Custom storage configuration for the tenant, if any pub tenant_conf: TenantConfig, diff --git a/libs/pageserver_api/src/shard.rs b/libs/pageserver_api/src/shard.rs index 32a834a26a..688b911425 100644 --- a/libs/pageserver_api/src/shard.rs +++ b/libs/pageserver_api/src/shard.rs @@ -2,6 +2,7 @@ use std::{ops::RangeInclusive, str::FromStr}; use hex::FromHex; use serde::{Deserialize, Serialize}; +use thiserror; use utils::id::TenantId; #[derive(Ord, PartialOrd, Eq, PartialEq, Clone, Copy, Serialize, Deserialize, Debug)] @@ -139,6 +140,89 @@ impl From<[u8; 18]> for TenantShardId { } } +/// For use within the context of a particular tenant, when we need to know which +/// shard we're dealing with, but do not need to know the full ShardIdentity (because +/// we won't be doing any page->shard mapping), and do not need to know the fully qualified +/// TenantShardId. +#[derive(Eq, PartialEq, PartialOrd, Ord, Clone, Copy)] +pub struct ShardIndex { + pub shard_number: ShardNumber, + pub shard_count: ShardCount, +} + +impl ShardIndex { + pub fn new(number: ShardNumber, count: ShardCount) -> Self { + Self { + shard_number: number, + shard_count: count, + } + } + pub fn unsharded() -> Self { + Self { + shard_number: ShardNumber(0), + shard_count: ShardCount(0), + } + } + + pub fn is_unsharded(&self) -> bool { + self.shard_number == ShardNumber(0) && self.shard_count == ShardCount(0) + } + + /// For use in constructing remote storage paths: concatenate this with a TenantId + /// to get a fully qualified TenantShardId. + /// + /// Backward compat: this function returns an empty string if Self::is_unsharded, such + /// that the legacy pre-sharding remote key format is preserved. + pub fn get_suffix(&self) -> String { + if self.is_unsharded() { + "".to_string() + } else { + format!("-{:02x}{:02x}", self.shard_number.0, self.shard_count.0) + } + } +} + +impl std::fmt::Display for ShardIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:02x}{:02x}", self.shard_number.0, self.shard_count.0) + } +} + +impl std::fmt::Debug for ShardIndex { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // Debug is the same as Display: the compact hex representation + write!(f, "{}", self) + } +} + +impl std::str::FromStr for ShardIndex { + type Err = hex::FromHexError; + + fn from_str(s: &str) -> Result { + // Expect format: 1 byte shard number, 1 byte shard count + if s.len() == 4 { + let bytes = s.as_bytes(); + let mut shard_parts: [u8; 2] = [0u8; 2]; + hex::decode_to_slice(bytes, &mut shard_parts)?; + Ok(Self { + shard_number: ShardNumber(shard_parts[0]), + shard_count: ShardCount(shard_parts[1]), + }) + } else { + Err(hex::FromHexError::InvalidStringLength) + } + } +} + +impl From<[u8; 2]> for ShardIndex { + fn from(b: [u8; 2]) -> Self { + Self { + shard_number: ShardNumber(b[0]), + shard_count: ShardCount(b[1]), + } + } +} + impl Serialize for TenantShardId { fn serialize(&self, serializer: S) -> Result where @@ -209,6 +293,151 @@ impl<'de> Deserialize<'de> for TenantShardId { } } +/// Stripe size in number of pages +#[derive(Clone, Copy, Serialize, Deserialize, Eq, PartialEq, Debug)] +pub struct ShardStripeSize(pub u32); + +/// Layout version: for future upgrades where we might change how the key->shard mapping works +#[derive(Clone, Copy, Serialize, Deserialize, Eq, PartialEq, Debug)] +pub struct ShardLayout(u8); + +const LAYOUT_V1: ShardLayout = ShardLayout(1); + +/// Default stripe size in pages: 256MiB divided by 8kiB page size. +const DEFAULT_STRIPE_SIZE: ShardStripeSize = ShardStripeSize(256 * 1024 / 8); + +/// The ShardIdentity contains the information needed for one member of map +/// to resolve a key to a shard, and then check whether that shard is ==self. +#[derive(Clone, Copy, Serialize, Deserialize, Eq, PartialEq, Debug)] +pub struct ShardIdentity { + pub layout: ShardLayout, + pub number: ShardNumber, + pub count: ShardCount, + pub stripe_size: ShardStripeSize, +} + +#[derive(thiserror::Error, Debug, PartialEq, Eq)] +pub enum ShardConfigError { + #[error("Invalid shard count")] + InvalidCount, + #[error("Invalid shard number")] + InvalidNumber, + #[error("Invalid stripe size")] + InvalidStripeSize, +} + +impl ShardIdentity { + /// An identity with number=0 count=0 is a "none" identity, which represents legacy + /// tenants. Modern single-shard tenants should not use this: they should + /// have number=0 count=1. + pub fn unsharded() -> Self { + Self { + number: ShardNumber(0), + count: ShardCount(0), + layout: LAYOUT_V1, + stripe_size: DEFAULT_STRIPE_SIZE, + } + } + + pub fn is_unsharded(&self) -> bool { + self.number == ShardNumber(0) && self.count == ShardCount(0) + } + + /// Count must be nonzero, and number must be < count. To construct + /// the legacy case (count==0), use Self::unsharded instead. + pub fn new( + number: ShardNumber, + count: ShardCount, + stripe_size: ShardStripeSize, + ) -> Result { + if count.0 == 0 { + Err(ShardConfigError::InvalidCount) + } else if number.0 > count.0 - 1 { + Err(ShardConfigError::InvalidNumber) + } else if stripe_size.0 == 0 { + Err(ShardConfigError::InvalidStripeSize) + } else { + Ok(Self { + number, + count, + layout: LAYOUT_V1, + stripe_size, + }) + } + } +} + +impl Serialize for ShardIndex { + fn serialize(&self, serializer: S) -> Result + where + S: serde::Serializer, + { + if serializer.is_human_readable() { + serializer.collect_str(self) + } else { + // Binary encoding is not used in index_part.json, but is included in anticipation of + // switching various structures (e.g. inter-process communication, remote metadata) to more + // compact binary encodings in future. + let mut packed: [u8; 2] = [0; 2]; + packed[0] = self.shard_number.0; + packed[1] = self.shard_count.0; + packed.serialize(serializer) + } + } +} + +impl<'de> Deserialize<'de> for ShardIndex { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + struct IdVisitor { + is_human_readable_deserializer: bool, + } + + impl<'de> serde::de::Visitor<'de> for IdVisitor { + type Value = ShardIndex; + + fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.is_human_readable_deserializer { + formatter.write_str("value in form of hex string") + } else { + formatter.write_str("value in form of integer array([u8; 2])") + } + } + + fn visit_seq(self, seq: A) -> Result + where + A: serde::de::SeqAccess<'de>, + { + let s = serde::de::value::SeqAccessDeserializer::new(seq); + let id: [u8; 2] = Deserialize::deserialize(s)?; + Ok(ShardIndex::from(id)) + } + + fn visit_str(self, v: &str) -> Result + where + E: serde::de::Error, + { + ShardIndex::from_str(v).map_err(E::custom) + } + } + + if deserializer.is_human_readable() { + deserializer.deserialize_str(IdVisitor { + is_human_readable_deserializer: true, + }) + } else { + deserializer.deserialize_tuple( + 2, + IdVisitor { + is_human_readable_deserializer: false, + }, + ) + } + } +} + #[cfg(test)] mod tests { use std::str::FromStr; @@ -318,4 +547,66 @@ mod tests { Ok(()) } + + #[test] + fn shard_identity_validation() -> Result<(), ShardConfigError> { + // Happy cases + ShardIdentity::new(ShardNumber(0), ShardCount(1), DEFAULT_STRIPE_SIZE)?; + ShardIdentity::new(ShardNumber(0), ShardCount(1), ShardStripeSize(1))?; + ShardIdentity::new(ShardNumber(254), ShardCount(255), ShardStripeSize(1))?; + + assert_eq!( + ShardIdentity::new(ShardNumber(0), ShardCount(0), DEFAULT_STRIPE_SIZE), + Err(ShardConfigError::InvalidCount) + ); + assert_eq!( + ShardIdentity::new(ShardNumber(10), ShardCount(10), DEFAULT_STRIPE_SIZE), + Err(ShardConfigError::InvalidNumber) + ); + assert_eq!( + ShardIdentity::new(ShardNumber(11), ShardCount(10), DEFAULT_STRIPE_SIZE), + Err(ShardConfigError::InvalidNumber) + ); + assert_eq!( + ShardIdentity::new(ShardNumber(255), ShardCount(255), DEFAULT_STRIPE_SIZE), + Err(ShardConfigError::InvalidNumber) + ); + assert_eq!( + ShardIdentity::new(ShardNumber(0), ShardCount(1), ShardStripeSize(0)), + Err(ShardConfigError::InvalidStripeSize) + ); + + Ok(()) + } + + #[test] + fn shard_index_human_encoding() -> Result<(), hex::FromHexError> { + let example = ShardIndex { + shard_number: ShardNumber(13), + shard_count: ShardCount(17), + }; + let expected: String = "0d11".to_string(); + let encoded = format!("{example}"); + assert_eq!(&encoded, &expected); + + let decoded = ShardIndex::from_str(&encoded)?; + assert_eq!(example, decoded); + Ok(()) + } + + #[test] + fn shard_index_binary_encoding() -> Result<(), hex::FromHexError> { + let example = ShardIndex { + shard_number: ShardNumber(13), + shard_count: ShardCount(17), + }; + let expected: [u8; 2] = [0x0d, 0x11]; + + let encoded = bincode::serialize(&example).unwrap(); + assert_eq!(Hex(&encoded), Hex(&expected)); + let decoded = bincode::deserialize(&encoded).unwrap(); + assert_eq!(example, decoded); + + Ok(()) + } } diff --git a/libs/remote_storage/tests/test_real_azure.rs b/libs/remote_storage/tests/test_real_azure.rs index 59b92b48fb..b631079bc5 100644 --- a/libs/remote_storage/tests/test_real_azure.rs +++ b/libs/remote_storage/tests/test_real_azure.rs @@ -281,6 +281,7 @@ fn ensure_logging_ready() { utils::logging::init( utils::logging::LogFormat::Test, utils::logging::TracingErrorLayerEnablement::Disabled, + utils::logging::Output::Stdout, ) .expect("logging init failed"); }); diff --git a/libs/remote_storage/tests/test_real_s3.rs b/libs/remote_storage/tests/test_real_s3.rs index e2360b7533..48f00e0106 100644 --- a/libs/remote_storage/tests/test_real_s3.rs +++ b/libs/remote_storage/tests/test_real_s3.rs @@ -210,6 +210,7 @@ fn ensure_logging_ready() { utils::logging::init( utils::logging::LogFormat::Test, utils::logging::TracingErrorLayerEnablement::Disabled, + utils::logging::Output::Stdout, ) .expect("logging init failed"); }); diff --git a/libs/utils/scripts/restore_from_wal_initdb.sh b/libs/utils/scripts/restore_from_wal_initdb.sh new file mode 100755 index 0000000000..c6277ebc60 --- /dev/null +++ b/libs/utils/scripts/restore_from_wal_initdb.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# like restore_from_wal.sh, but takes existing initdb.tar.zst + +set -euxo pipefail + +PG_BIN=$1 +WAL_PATH=$2 +DATA_DIR=$3 +PORT=$4 +echo "port=$PORT" >> "$DATA_DIR"/postgresql.conf +echo "shared_preload_libraries='\$libdir/neon_rmgr.so'" >> "$DATA_DIR"/postgresql.conf +REDO_POS=0x$("$PG_BIN"/pg_controldata -D "$DATA_DIR" | grep -F "REDO location"| cut -c 42-) +declare -i WAL_SIZE=$REDO_POS+114 +"$PG_BIN"/pg_ctl -D "$DATA_DIR" -l "$DATA_DIR/logfile.log" start +"$PG_BIN"/pg_ctl -D "$DATA_DIR" -l "$DATA_DIR/logfile.log" stop -m immediate +cp "$DATA_DIR"/pg_wal/000000010000000000000001 . +cp "$WAL_PATH"/* "$DATA_DIR"/pg_wal/ +for partial in "$DATA_DIR"/pg_wal/*.partial ; do mv "$partial" "${partial%.partial}" ; done +dd if=000000010000000000000001 of="$DATA_DIR"/pg_wal/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc +rm -f 000000010000000000000001 diff --git a/libs/utils/src/logging.rs b/libs/utils/src/logging.rs index 502e02dc71..2f09c2f3ea 100644 --- a/libs/utils/src/logging.rs +++ b/libs/utils/src/logging.rs @@ -66,9 +66,17 @@ pub enum TracingErrorLayerEnablement { EnableWithRustLogFilter, } +/// Where the logging should output to. +#[derive(Clone, Copy)] +pub enum Output { + Stdout, + Stderr, +} + pub fn init( log_format: LogFormat, tracing_error_layer_enablement: TracingErrorLayerEnablement, + output: Output, ) -> anyhow::Result<()> { // We fall back to printing all spans at info-level or above if // the RUST_LOG environment variable is not set. @@ -85,7 +93,12 @@ pub fn init( let log_layer = tracing_subscriber::fmt::layer() .with_target(false) .with_ansi(false) - .with_writer(std::io::stdout); + .with_writer(move || -> Box { + match output { + Output::Stdout => Box::new(std::io::stdout()), + Output::Stderr => Box::new(std::io::stderr()), + } + }); let log_layer = match log_format { LogFormat::Json => log_layer.json().boxed(), LogFormat::Plain => log_layer.boxed(), diff --git a/pageserver/ctl/Cargo.toml b/pageserver/ctl/Cargo.toml index ff0c530125..c5cd451e8d 100644 --- a/pageserver/ctl/Cargo.toml +++ b/pageserver/ctl/Cargo.toml @@ -18,3 +18,5 @@ tokio.workspace = true utils.workspace = true svg_fmt.workspace = true workspace_hack.workspace = true +serde.workspace = true +serde_json.workspace = true diff --git a/pageserver/ctl/src/index_part.rs b/pageserver/ctl/src/index_part.rs new file mode 100644 index 0000000000..20e5572914 --- /dev/null +++ b/pageserver/ctl/src/index_part.rs @@ -0,0 +1,38 @@ +use std::collections::HashMap; + +use anyhow::Context; +use camino::Utf8PathBuf; +use pageserver::tenant::remote_timeline_client::index::IndexLayerMetadata; +use pageserver::tenant::storage_layer::LayerFileName; +use pageserver::tenant::{metadata::TimelineMetadata, IndexPart}; +use utils::lsn::Lsn; + +#[derive(clap::Subcommand)] +pub(crate) enum IndexPartCmd { + Dump { path: Utf8PathBuf }, +} + +pub(crate) async fn main(cmd: &IndexPartCmd) -> anyhow::Result<()> { + match cmd { + IndexPartCmd::Dump { path } => { + let bytes = tokio::fs::read(path).await.context("read file")?; + let des: IndexPart = IndexPart::from_s3_bytes(&bytes).context("deserialize")?; + #[derive(serde::Serialize)] + struct Output<'a> { + layer_metadata: &'a HashMap, + disk_consistent_lsn: Lsn, + timeline_metadata: &'a TimelineMetadata, + } + + let output = Output { + layer_metadata: &des.layer_metadata, + disk_consistent_lsn: des.get_disk_consistent_lsn(), + timeline_metadata: &des.metadata, + }; + + let output = serde_json::to_string_pretty(&output).context("serialize output")?; + println!("{output}"); + Ok(()) + } + } +} diff --git a/pageserver/ctl/src/main.rs b/pageserver/ctl/src/main.rs index c4d6e8d883..fb42d6d2f1 100644 --- a/pageserver/ctl/src/main.rs +++ b/pageserver/ctl/src/main.rs @@ -5,11 +5,13 @@ //! Separate, `metadata` subcommand allows to print and update pageserver's metadata file. mod draw_timeline_dir; +mod index_part; mod layer_map_analyzer; mod layers; use camino::{Utf8Path, Utf8PathBuf}; use clap::{Parser, Subcommand}; +use index_part::IndexPartCmd; use layers::LayerCmd; use pageserver::{ context::{DownloadBehavior, RequestContext}, @@ -38,6 +40,8 @@ struct CliOpts { #[derive(Subcommand)] enum Commands { Metadata(MetadataCmd), + #[command(subcommand)] + IndexPart(IndexPartCmd), PrintLayerFile(PrintLayerFileCmd), DrawTimeline {}, AnalyzeLayerMap(AnalyzeLayerMapCmd), @@ -83,6 +87,9 @@ async fn main() -> anyhow::Result<()> { Commands::Metadata(cmd) => { handle_metadata(&cmd)?; } + Commands::IndexPart(cmd) => { + index_part::main(&cmd).await?; + } Commands::DrawTimeline {} => { draw_timeline_dir::main()?; } diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 68865b1684..093a5ed6d4 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -105,7 +105,11 @@ fn main() -> anyhow::Result<()> { } else { TracingErrorLayerEnablement::Disabled }; - logging::init(conf.log_format, tracing_error_layer_enablement)?; + logging::init( + conf.log_format, + tracing_error_layer_enablement, + logging::Output::Stdout, + )?; // mind the order required here: 1. logging, 2. panic_hook, 3. sentry. // disarming this hook on pageserver, because we never tear down tracing. @@ -623,6 +627,7 @@ fn start_pageserver( conf.synthetic_size_calculation_interval, conf.id, local_disk_storage, + cancel, metrics_ctx, ) .instrument(info_span!("metrics_collection")) diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 9e8377c1f1..7ad6a0f890 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -3,7 +3,7 @@ use crate::context::{DownloadBehavior, RequestContext}; use crate::task_mgr::{self, TaskKind, BACKGROUND_RUNTIME}; use crate::tenant::tasks::BackgroundLoopKind; -use crate::tenant::{mgr, LogicalSizeCalculationCause}; +use crate::tenant::{mgr, LogicalSizeCalculationCause, PageReconstructError}; use camino::Utf8PathBuf; use consumption_metrics::EventType; use pageserver_api::models::TenantState; @@ -12,6 +12,7 @@ use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, SystemTime}; use tokio::time::Instant; +use tokio_util::sync::CancellationToken; use tracing::*; use utils::id::NodeId; @@ -37,6 +38,7 @@ type RawMetric = (MetricsKey, (EventType, u64)); type Cache = HashMap; /// Main thread that serves metrics collection +#[allow(clippy::too_many_arguments)] pub async fn collect_metrics( metric_collection_endpoint: &Url, metric_collection_interval: Duration, @@ -44,6 +46,7 @@ pub async fn collect_metrics( synthetic_size_calculation_interval: Duration, node_id: NodeId, local_disk_storage: Utf8PathBuf, + cancel: CancellationToken, ctx: RequestContext, ) -> anyhow::Result<()> { if _cached_metric_collection_interval != Duration::ZERO { @@ -63,9 +66,13 @@ pub async fn collect_metrics( "synthetic size calculation", false, async move { - calculate_synthetic_size_worker(synthetic_size_calculation_interval, &worker_ctx) - .instrument(info_span!("synthetic_size_worker")) - .await?; + calculate_synthetic_size_worker( + synthetic_size_calculation_interval, + &cancel, + &worker_ctx, + ) + .instrument(info_span!("synthetic_size_worker")) + .await?; Ok(()) }, ); @@ -241,6 +248,7 @@ async fn reschedule( /// Caclculate synthetic size for each active tenant async fn calculate_synthetic_size_worker( synthetic_size_calculation_interval: Duration, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result<()> { info!("starting calculate_synthetic_size_worker"); @@ -272,7 +280,12 @@ async fn calculate_synthetic_size_worker( // Same for the loop that fetches computed metrics. // By using the same limiter, we centralize metrics collection for "start" and "finished" counters, // which turns out is really handy to understand the system. - if let Err(e) = tenant.calculate_synthetic_size(cause, ctx).await { + if let Err(e) = tenant.calculate_synthetic_size(cause, cancel, ctx).await { + if let Some(PageReconstructError::Cancelled) = + e.downcast_ref::() + { + return Ok(()); + } error!("failed to calculate synthetic size for tenant {tenant_id}: {e:#}"); } } diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 6a732d1029..ad95254a65 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -10,6 +10,7 @@ use crate::control_plane_client::ControlPlaneGenerationsApi; use crate::metrics; use crate::tenant::remote_timeline_client::remote_layer_path; use crate::tenant::remote_timeline_client::remote_timeline_path; +use crate::tenant::remote_timeline_client::LayerFileMetadata; use crate::virtual_file::MaybeFatalIo; use crate::virtual_file::VirtualFile; use anyhow::Context; @@ -509,17 +510,19 @@ impl DeletionQueueClient { tenant_id: TenantId, timeline_id: TimelineId, current_generation: Generation, - layers: Vec<(LayerFileName, Generation)>, + layers: Vec<(LayerFileName, LayerFileMetadata)>, ) -> Result<(), DeletionQueueError> { if current_generation.is_none() { debug!("Enqueuing deletions in legacy mode, skipping queue"); + let mut layer_paths = Vec::new(); - for (layer, generation) in layers { + for (layer, meta) in layers { layer_paths.push(remote_layer_path( &tenant_id, &timeline_id, + meta.shard, &layer, - generation, + meta.generation, )); } self.push_immediate(layer_paths).await?; @@ -539,7 +542,7 @@ impl DeletionQueueClient { tenant_id: TenantId, timeline_id: TimelineId, current_generation: Generation, - layers: Vec<(LayerFileName, Generation)>, + layers: Vec<(LayerFileName, LayerFileMetadata)>, ) -> Result<(), DeletionQueueError> { metrics::DELETION_QUEUE .keys_submitted @@ -750,6 +753,7 @@ impl DeletionQueue { mod test { use camino::Utf8Path; use hex_literal::hex; + use pageserver_api::shard::ShardIndex; use std::{io::ErrorKind, time::Duration}; use tracing::info; @@ -989,6 +993,8 @@ mod test { // we delete, and the generation of the running Tenant. let layer_generation = Generation::new(0xdeadbeef); let now_generation = Generation::new(0xfeedbeef); + let layer_metadata = + LayerFileMetadata::new(0xf00, layer_generation, ShardIndex::unsharded()); let remote_layer_file_name_1 = format!("{}{}", layer_file_name_1, layer_generation.get_suffix()); @@ -1012,7 +1018,7 @@ mod test { tenant_id, TIMELINE_ID, now_generation, - [(layer_file_name_1.clone(), layer_generation)].to_vec(), + [(layer_file_name_1.clone(), layer_metadata)].to_vec(), ) .await?; assert_remote_files(&[&remote_layer_file_name_1], &remote_timeline_path); @@ -1051,6 +1057,8 @@ mod test { let stale_generation = latest_generation.previous(); // Generation that our example layer file was written with let layer_generation = stale_generation.previous(); + let layer_metadata = + LayerFileMetadata::new(0xf00, layer_generation, ShardIndex::unsharded()); ctx.set_latest_generation(latest_generation); @@ -1068,7 +1076,7 @@ mod test { tenant_id, TIMELINE_ID, stale_generation, - [(EXAMPLE_LAYER_NAME.clone(), layer_generation)].to_vec(), + [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(), ) .await?; @@ -1083,7 +1091,7 @@ mod test { tenant_id, TIMELINE_ID, latest_generation, - [(EXAMPLE_LAYER_NAME.clone(), layer_generation)].to_vec(), + [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(), ) .await?; @@ -1110,6 +1118,8 @@ mod test { let layer_generation = Generation::new(0xdeadbeef); let now_generation = Generation::new(0xfeedbeef); + let layer_metadata = + LayerFileMetadata::new(0xf00, layer_generation, ShardIndex::unsharded()); // Inject a deletion in the generation before generation_now: after restart, // this deletion should _not_ get executed (only the immediately previous @@ -1121,7 +1131,7 @@ mod test { tenant_id, TIMELINE_ID, now_generation.previous(), - [(EXAMPLE_LAYER_NAME.clone(), layer_generation)].to_vec(), + [(EXAMPLE_LAYER_NAME.clone(), layer_metadata.clone())].to_vec(), ) .await?; @@ -1135,7 +1145,7 @@ mod test { tenant_id, TIMELINE_ID, now_generation, - [(EXAMPLE_LAYER_NAME_ALT.clone(), layer_generation)].to_vec(), + [(EXAMPLE_LAYER_NAME_ALT.clone(), layer_metadata.clone())].to_vec(), ) .await?; @@ -1225,12 +1235,13 @@ pub(crate) mod mock { match msg { ListWriterQueueMessage::Delete(op) => { let mut objects = op.objects; - for (layer, generation) in op.layers { + for (layer, meta) in op.layers { objects.push(remote_layer_path( &op.tenant_id, &op.timeline_id, + meta.shard, &layer, - generation, + meta.generation, )); } diff --git a/pageserver/src/deletion_queue/list_writer.rs b/pageserver/src/deletion_queue/list_writer.rs index 28daae2da5..5d52b680e4 100644 --- a/pageserver/src/deletion_queue/list_writer.rs +++ b/pageserver/src/deletion_queue/list_writer.rs @@ -33,6 +33,7 @@ use crate::config::PageServerConf; use crate::deletion_queue::TEMP_SUFFIX; use crate::metrics; use crate::tenant::remote_timeline_client::remote_layer_path; +use crate::tenant::remote_timeline_client::LayerFileMetadata; use crate::tenant::storage_layer::LayerFileName; use crate::virtual_file::on_fatal_io_error; use crate::virtual_file::MaybeFatalIo; @@ -58,7 +59,7 @@ pub(super) struct DeletionOp { // `layers` and `objects` are both just lists of objects. `layers` is used if you do not // have a config object handy to project it to a remote key, and need the consuming worker // to do it for you. - pub(super) layers: Vec<(LayerFileName, Generation)>, + pub(super) layers: Vec<(LayerFileName, LayerFileMetadata)>, pub(super) objects: Vec, /// The _current_ generation of the Tenant attachment in which we are enqueuing @@ -387,12 +388,13 @@ impl ListWriter { ); let mut layer_paths = Vec::new(); - for (layer, generation) in op.layers { + for (layer, meta) in op.layers { layer_paths.push(remote_layer_path( &op.tenant_id, &op.timeline_id, + meta.shard, &layer, - generation, + meta.generation, )); } layer_paths.extend(op.objects); diff --git a/pageserver/src/deletion_queue/validator.rs b/pageserver/src/deletion_queue/validator.rs index 72bdbdefd6..bf06c78e67 100644 --- a/pageserver/src/deletion_queue/validator.rs +++ b/pageserver/src/deletion_queue/validator.rs @@ -178,7 +178,14 @@ where .unwrap_or(false); if valid && *validated_generation == tenant_lsn_state.generation { - for (_timeline_id, pending_lsn) in tenant_lsn_state.timelines { + for (timeline_id, pending_lsn) in tenant_lsn_state.timelines { + tracing::debug!( + %tenant_id, + %timeline_id, + current = %pending_lsn.result_slot.load(), + projected = %pending_lsn.projected, + "advancing validated remote_consistent_lsn", + ); pending_lsn.result_slot.store(pending_lsn.projected); } } else { diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index aa2b017471..5ce09500ee 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -6,6 +6,7 @@ use std::str::FromStr; use std::sync::Arc; use anyhow::{anyhow, Context, Result}; +use enumset::EnumSet; use futures::TryFutureExt; use humantime::format_rfc3339; use hyper::header; @@ -42,6 +43,7 @@ use crate::tenant::mgr::{ }; use crate::tenant::size::ModelInputs; use crate::tenant::storage_layer::LayerAccessStatsReset; +use crate::tenant::timeline::CompactFlags; use crate::tenant::timeline::Timeline; use crate::tenant::{LogicalSizeCalculationCause, PageReconstructError, TenantSharedResources}; use crate::{config::PageServerConf, tenant::mgr}; @@ -548,7 +550,7 @@ async fn timeline_detail_handler( async fn get_lsn_by_timestamp_handler( request: Request, - _cancel: CancellationToken, + cancel: CancellationToken, ) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -564,7 +566,9 @@ async fn get_lsn_by_timestamp_handler( let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; - let result = timeline.find_lsn_for_timestamp(timestamp_pg, &ctx).await?; + let result = timeline + .find_lsn_for_timestamp(timestamp_pg, &cancel, &ctx) + .await?; if version.unwrap_or(0) > 1 { #[derive(serde::Serialize)] @@ -840,7 +844,7 @@ async fn tenant_delete_handler( /// without modifying anything anyway. async fn tenant_size_handler( request: Request, - _cancel: CancellationToken, + cancel: CancellationToken, ) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; check_permission(&request, Some(tenant_id))?; @@ -856,6 +860,7 @@ async fn tenant_size_handler( .gather_size_inputs( retention_period, LogicalSizeCalculationCause::TenantSizeHandler, + &cancel, &ctx, ) .await @@ -1240,7 +1245,7 @@ async fn failpoints_handler( // Run GC immediately on given timeline. async fn timeline_gc_handler( mut request: Request, - _cancel: CancellationToken, + cancel: CancellationToken, ) -> Result, ApiError> { let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; @@ -1249,7 +1254,7 @@ async fn timeline_gc_handler( let gc_req: TimelineGcRequest = json_request(&mut request).await?; let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); - let wait_task_done = mgr::immediate_gc(tenant_id, timeline_id, gc_req, &ctx).await?; + let wait_task_done = mgr::immediate_gc(tenant_id, timeline_id, gc_req, cancel, &ctx).await?; let gc_result = wait_task_done .await .context("wait for gc task") @@ -1268,11 +1273,15 @@ async fn timeline_compact_handler( let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; + let mut flags = EnumSet::empty(); + if Some(true) == parse_query_param::<_, bool>(&request, "force_repartition")? { + flags |= CompactFlags::ForceRepartition; + } async { let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; timeline - .compact(&cancel, &ctx) + .compact(&cancel, flags, &ctx) .await .map_err(|e| ApiError::InternalServerError(e.into()))?; json_response(StatusCode::OK, ()) @@ -1289,6 +1298,11 @@ async fn timeline_checkpoint_handler( let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; check_permission(&request, Some(tenant_id))?; + + let mut flags = EnumSet::empty(); + if Some(true) == parse_query_param::<_, bool>(&request, "force_repartition")? { + flags |= CompactFlags::ForceRepartition; + } async { let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; @@ -1297,7 +1311,7 @@ async fn timeline_checkpoint_handler( .await .map_err(ApiError::InternalServerError)?; timeline - .compact(&cancel, &ctx) + .compact(&cancel, flags, &ctx) .await .map_err(|e| ApiError::InternalServerError(e.into()))?; @@ -1675,8 +1689,24 @@ where let token_cloned = token.clone(); let result = handler(r, token).await; if token_cloned.is_cancelled() { - info!("Cancelled request finished"); + // dropguard has executed: we will never turn this result into response. + // + // at least temporarily do {:?} logging; these failures are rare enough but + // could hide difficult errors. + match &result { + Ok(response) => { + let status = response.status(); + info!(%status, "Cancelled request finished successfully") + } + Err(e) => error!("Cancelled request finished with an error: {e:?}"), + } } + // only logging for cancelled panicked request handlers is the tracing_panic_hook, + // which should suffice. + // + // there is still a chance to lose the result due to race between + // returning from here and the actual connection closing happening + // before outer task gets to execute. leaving that up for #5815. result } .in_current_span(), diff --git a/pageserver/src/import_datadir.rs b/pageserver/src/import_datadir.rs index 30975c1fc9..770458e02e 100644 --- a/pageserver/src/import_datadir.rs +++ b/pageserver/src/import_datadir.rs @@ -3,18 +3,25 @@ //! a neon Timeline. //! use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::task::{self, Poll}; use anyhow::{bail, ensure, Context, Result}; +use async_compression::{tokio::write::ZstdEncoder, zstd::CParameter, Level}; use bytes::Bytes; use camino::Utf8Path; use futures::StreamExt; -use tokio::io::{AsyncRead, AsyncReadExt}; +use nix::NixPath; +use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite, AsyncWriteExt}; use tokio_tar::Archive; +use tokio_tar::Builder; +use tokio_tar::HeaderMode; use tracing::*; use walkdir::WalkDir; use crate::context::RequestContext; use crate::pgdatadir_mapping::*; +use crate::tenant::remote_timeline_client::INITDB_PATH; use crate::tenant::Timeline; use crate::walingest::WalIngest; use crate::walrecord::DecodedWALRecord; @@ -33,7 +40,9 @@ use utils::lsn::Lsn; pub fn get_lsn_from_controlfile(path: &Utf8Path) -> Result { // Read control file to extract the LSN let controlfile_path = path.join("global").join("pg_control"); - let controlfile = ControlFileData::decode(&std::fs::read(controlfile_path)?)?; + let controlfile_buf = std::fs::read(&controlfile_path) + .with_context(|| format!("reading controlfile: {controlfile_path}"))?; + let controlfile = ControlFileData::decode(&controlfile_buf)?; let lsn = controlfile.checkPoint; Ok(Lsn(lsn)) @@ -618,3 +627,108 @@ async fn read_all_bytes(reader: &mut (impl AsyncRead + Unpin)) -> Result reader.read_to_end(&mut buf).await?; Ok(Bytes::from(buf)) } + +/// An in-memory buffer implementing `AsyncWrite`, inserting yields every now and then +/// +/// The number of yields is bounded by above by the number of times poll_write is called, +/// so calling it with 8 KB chunks and 8 MB chunks gives the same number of yields in total. +/// This is an explicit choice as the `YieldingVec` is meant to give the async executor +/// breathing room between units of CPU intensive preparation of buffers to be written. +/// Once a write call is issued, the whole buffer has been prepared already, so there is no +/// gain in splitting up the memcopy further. +struct YieldingVec { + yield_budget: usize, + // the buffer written into + buf: Vec, +} + +impl YieldingVec { + fn new() -> Self { + Self { + yield_budget: 0, + buf: Vec::new(), + } + } + // Whether we should yield for a read operation of given size + fn should_yield(&mut self, add_buf_len: usize) -> bool { + // Set this limit to a small value so that we are a + // good async citizen and yield repeatedly (but not + // too often for many small writes to cause many yields) + const YIELD_DIST: usize = 1024; + + let target_buf_len = self.buf.len() + add_buf_len; + let ret = self.yield_budget / YIELD_DIST < target_buf_len / YIELD_DIST; + if self.yield_budget < target_buf_len { + self.yield_budget += add_buf_len; + } + ret + } +} + +impl AsyncWrite for YieldingVec { + fn poll_write( + mut self: Pin<&mut Self>, + cx: &mut task::Context<'_>, + buf: &[u8], + ) -> Poll> { + if self.should_yield(buf.len()) { + cx.waker().wake_by_ref(); + return Poll::Pending; + } + self.get_mut().buf.extend_from_slice(buf); + Poll::Ready(Ok(buf.len())) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut task::Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown( + self: Pin<&mut Self>, + _cx: &mut task::Context<'_>, + ) -> Poll> { + Poll::Ready(Ok(())) + } +} + +pub async fn create_tar_zst(pgdata_path: &Utf8Path) -> Result> { + let mut paths = Vec::new(); + for entry in WalkDir::new(pgdata_path) { + let entry = entry?; + let metadata = entry.metadata().expect("error getting dir entry metadata"); + // Also allow directories so that we also get empty directories + if !(metadata.is_file() || metadata.is_dir()) { + continue; + } + let path = entry.into_path(); + paths.push(path); + } + // Do a sort to get a more consistent listing + paths.sort_unstable(); + let zstd = ZstdEncoder::with_quality_and_params( + YieldingVec::new(), + Level::Default, + &[CParameter::enable_long_distance_matching(true)], + ); + let mut builder = Builder::new(zstd); + // Use reproducible header mode + builder.mode(HeaderMode::Deterministic); + for path in paths { + let rel_path = path.strip_prefix(pgdata_path)?; + if rel_path.is_empty() { + // The top directory should not be compressed, + // the tar crate doesn't like that + continue; + } + builder.append_path_with_name(&path, rel_path).await?; + } + let mut zstd = builder.into_inner().await?; + zstd.shutdown().await?; + let compressed = zstd.into_inner(); + let compressed_len = compressed.buf.len(); + const INITDB_TAR_ZST_WARN_LIMIT: usize = 2_000_000; + if compressed_len > INITDB_TAR_ZST_WARN_LIMIT { + warn!("compressed {INITDB_PATH} size of {compressed_len} is above limit {INITDB_TAR_ZST_WARN_LIMIT}."); + } + Ok(compressed.buf) +} diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 827278af72..15d5609ceb 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -21,6 +21,7 @@ use serde::{Deserialize, Serialize}; use std::collections::{hash_map, HashMap, HashSet}; use std::ops::ControlFlow; use std::ops::Range; +use tokio_util::sync::CancellationToken; use tracing::{debug, trace, warn}; use utils::bin_ser::DeserializeError; use utils::{bin_ser::BeSer, lsn::Lsn}; @@ -365,6 +366,7 @@ impl Timeline { pub async fn find_lsn_for_timestamp( &self, search_timestamp: TimestampTz, + cancel: &CancellationToken, ctx: &RequestContext, ) -> Result { let gc_cutoff_lsn_guard = self.get_latest_gc_cutoff_lsn(); @@ -383,6 +385,9 @@ impl Timeline { let mut found_smaller = false; let mut found_larger = false; while low < high { + if cancel.is_cancelled() { + return Err(PageReconstructError::Cancelled); + } // cannot overflow, high and low are both smaller than u64::MAX / 2 let mid = (high + low) / 2; diff --git a/pageserver/src/repository.rs b/pageserver/src/repository.rs index 24f47df92e..97d731bf49 100644 --- a/pageserver/src/repository.rs +++ b/pageserver/src/repository.rs @@ -138,6 +138,14 @@ pub struct GcResult { #[serde(serialize_with = "serialize_duration_as_millis")] pub elapsed: Duration, + + /// The layers which were garbage collected. + /// + /// Used in `/v1/tenant/:tenant_id/timeline/:timeline_id/do_gc` to wait for the layers to be + /// dropped in tests. + #[cfg(feature = "testing")] + #[serde(skip)] + pub(crate) doomed_layers: Vec, } // helper function for `GcResult`, serializing a `Duration` as an integer number of milliseconds @@ -158,5 +166,11 @@ impl AddAssign for GcResult { self.layers_removed += other.layers_removed; self.elapsed += other.elapsed; + + #[cfg(feature = "testing")] + { + let mut other = other; + self.doomed_layers.append(&mut other.doomed_layers); + } } } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index b112dbae9e..bc8779b26f 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -12,42 +12,26 @@ //! use anyhow::{bail, Context}; +use bytes::Bytes; use camino::{Utf8Path, Utf8PathBuf}; +use enumset::EnumSet; use futures::FutureExt; use pageserver_api::models::TimelineState; use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; +use std::fmt; use storage_broker::BrokerClientChannel; use tokio::runtime::Handle; use tokio::sync::watch; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; use tracing::*; +use utils::backoff; use utils::completion; use utils::crashsafe::path_with_suffix_extension; use utils::fs_ext; use utils::sync::gate::Gate; -use std::cmp::min; -use std::collections::hash_map::Entry; -use std::collections::BTreeSet; -use std::collections::HashMap; -use std::collections::HashSet; -use std::fmt::Debug; -use std::fmt::Display; -use std::fs; -use std::fs::File; -use std::io; -use std::ops::Bound::Included; -use std::process::Command; -use std::process::Stdio; -use std::sync::atomic::AtomicU64; -use std::sync::atomic::Ordering; -use std::sync::Arc; -use std::sync::MutexGuard; -use std::sync::{Mutex, RwLock}; -use std::time::{Duration, Instant}; - use self::config::AttachedLocationConfig; use self::config::AttachmentMode; use self::config::LocationConf; @@ -81,14 +65,35 @@ use crate::tenant::remote_timeline_client::MaybeDeletedIndexPart; use crate::tenant::storage_layer::DeltaLayer; use crate::tenant::storage_layer::ImageLayer; use crate::InitializationOrder; +use std::cmp::min; +use std::collections::hash_map::Entry; +use std::collections::BTreeSet; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fmt::Debug; +use std::fmt::Display; +use std::fs; +use std::fs::File; +use std::io; +use std::ops::Bound::Included; +use std::process::Stdio; +use std::sync::atomic::AtomicU64; +use std::sync::atomic::Ordering; +use std::sync::Arc; +use std::sync::MutexGuard; +use std::sync::{Mutex, RwLock}; +use std::time::{Duration, Instant}; use crate::tenant::timeline::delete::DeleteTimelineFlow; use crate::tenant::timeline::uninit::cleanup_timeline_directory; use crate::virtual_file::VirtualFile; use crate::walredo::PostgresRedoManager; use crate::TEMP_FILE_SUFFIX; +use once_cell::sync::Lazy; pub use pageserver_api::models::TenantState; +use tokio::sync::Semaphore; +static INIT_DB_SEMAPHORE: Lazy = Lazy::new(|| Semaphore::new(8)); use toml_edit; use utils::{ crashsafe, @@ -400,6 +405,36 @@ pub enum CreateTimelineError { Other(#[from] anyhow::Error), } +#[derive(thiserror::Error, Debug)] +enum InitdbError { + Other(anyhow::Error), + Cancelled, + Spawn(std::io::Result<()>), + Failed(std::process::ExitStatus, Vec), +} + +impl fmt::Display for InitdbError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + InitdbError::Cancelled => write!(f, "Operation was cancelled"), + InitdbError::Spawn(e) => write!(f, "Spawn error: {:?}", e), + InitdbError::Failed(status, stderr) => write!( + f, + "Command failed with status {:?}: {}", + status, + String::from_utf8_lossy(stderr) + ), + InitdbError::Other(e) => write!(f, "Error: {:?}", e), + } + } +} + +impl From for InitdbError { + fn from(error: std::io::Error) -> Self { + InitdbError::Spawn(Err(error)) + } +} + struct TenantDirectoryScan { sorted_timelines_to_load: Vec<(TimelineId, TimelineMetadata)>, timelines_to_resume_deletion: Vec<(TimelineId, Option)>, @@ -730,7 +765,7 @@ impl Tenant { /// async fn attach( self: &Arc, - mut init_order: Option, + init_order: Option, preload: Option, ctx: &RequestContext, ) -> anyhow::Result<()> { @@ -747,11 +782,6 @@ impl Tenant { } }; - // Signal that we have completed remote phase - init_order - .as_mut() - .and_then(|x| x.initial_tenant_load_remote.take()); - let mut timelines_to_resume_deletions = vec![]; let mut remote_index_and_client = HashMap::new(); @@ -1629,6 +1659,7 @@ impl Tenant { target_timeline_id: Option, horizon: u64, pitr: Duration, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result { // Don't start doing work during shutdown @@ -1651,7 +1682,7 @@ impl Tenant { } } - self.gc_iteration_internal(target_timeline_id, horizon, pitr, ctx) + self.gc_iteration_internal(target_timeline_id, horizon, pitr, cancel, ctx) .await } @@ -1699,7 +1730,7 @@ impl Tenant { for (timeline_id, timeline) in &timelines_to_compact { timeline - .compact(cancel, ctx) + .compact(cancel, EnumSet::empty(), ctx) .instrument(info_span!("compact_timeline", %timeline_id)) .await?; } @@ -1854,6 +1885,7 @@ impl Tenant { }); }) }; + // test_long_timeline_create_then_tenant_delete is leaning on this message tracing::info!("Waiting for timelines..."); while let Some(res) = js.join_next().await { match res { @@ -2568,14 +2600,30 @@ impl Tenant { target_timeline_id: Option, horizon: u64, pitr: Duration, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result { let mut totals: GcResult = Default::default(); let now = Instant::now(); - let gc_timelines = self - .refresh_gc_info_internal(target_timeline_id, horizon, pitr, ctx) - .await?; + let gc_timelines = match self + .refresh_gc_info_internal(target_timeline_id, horizon, pitr, cancel, ctx) + .await + { + Ok(result) => result, + Err(e) => { + if let Some(PageReconstructError::Cancelled) = + e.downcast_ref::() + { + // Handle cancellation + totals.elapsed = now.elapsed(); + return Ok(totals); + } else { + // Propagate other errors + return Err(e); + } + } + }; crate::failpoint_support::sleep_millis_async!( "gc_iteration_internal_after_getting_gc_timelines" @@ -2590,16 +2638,14 @@ impl Tenant { // Perform GC for each timeline. // - // Note that we don't hold the GC lock here because we don't want - // to delay the branch creation task, which requires the GC lock. - // A timeline GC iteration can be slow because it may need to wait for - // compaction (both require `layer_removal_cs` lock), - // but the GC iteration can run concurrently with branch creation. + // Note that we don't hold the `Tenant::gc_cs` lock here because we don't want to delay the + // branch creation task, which requires the GC lock. A GC iteration can run concurrently + // with branch creation. // - // See comments in [`Tenant::branch_timeline`] for more information - // about why branch creation task can run concurrently with timeline's GC iteration. + // See comments in [`Tenant::branch_timeline`] for more information about why branch + // creation task can run concurrently with timeline's GC iteration. for timeline in gc_timelines { - if task_mgr::is_shutdown_requested() { + if task_mgr::is_shutdown_requested() || cancel.is_cancelled() { // We were requested to shut down. Stop and return with the progress we // made. break; @@ -2619,6 +2665,7 @@ impl Tenant { /// This is usually executed as part of periodic gc, but can now be triggered more often. pub async fn refresh_gc_info( &self, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result>> { // since this method can now be called at different rates than the configured gc loop, it @@ -2630,7 +2677,7 @@ impl Tenant { // refresh all timelines let target_timeline_id = None; - self.refresh_gc_info_internal(target_timeline_id, horizon, pitr, ctx) + self.refresh_gc_info_internal(target_timeline_id, horizon, pitr, cancel, ctx) .await } @@ -2639,6 +2686,7 @@ impl Tenant { target_timeline_id: Option, horizon: u64, pitr: Duration, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result>> { // grab mutex to prevent new timelines from being created here. @@ -2712,7 +2760,7 @@ impl Tenant { .map(|&x| x.1) .collect(); timeline - .update_gc_info(branchpoints, cutoff, pitr, ctx) + .update_gc_info(branchpoints, cutoff, pitr, cancel, ctx) .await?; gc_timelines.push(timeline); @@ -2875,7 +2923,7 @@ impl Tenant { } /// - run initdb to init temporary instance and get bootstrap data - /// - after initialization complete, remove the temp dir. + /// - after initialization completes, tar up the temp dir and upload it to S3. /// /// The caller is responsible for activating the returned timeline. pub(crate) async fn bootstrap_timeline( @@ -2890,7 +2938,7 @@ impl Tenant { }; // create a `tenant/{tenant_id}/timelines/basebackup-{timeline_id}.{TEMP_FILE_SUFFIX}/` // temporary directory for basebackup files for the given timeline. - let initdb_path = path_with_suffix_extension( + let pgdata_path = path_with_suffix_extension( self.conf .timelines_path(&self.tenant_id) .join(format!("basebackup-{timeline_id}")), @@ -2899,22 +2947,45 @@ impl Tenant { // an uninit mark was placed before, nothing else can access this timeline files // current initdb was not run yet, so remove whatever was left from the previous runs - if initdb_path.exists() { - fs::remove_dir_all(&initdb_path).with_context(|| { - format!("Failed to remove already existing initdb directory: {initdb_path}") + if pgdata_path.exists() { + fs::remove_dir_all(&pgdata_path).with_context(|| { + format!("Failed to remove already existing initdb directory: {pgdata_path}") })?; } // Init temporarily repo to get bootstrap data, this creates a directory in the `initdb_path` path - run_initdb(self.conf, &initdb_path, pg_version)?; + run_initdb(self.conf, &pgdata_path, pg_version, &self.cancel).await?; // this new directory is very temporary, set to remove it immediately after bootstrap, we don't need it scopeguard::defer! { - if let Err(e) = fs::remove_dir_all(&initdb_path) { + if let Err(e) = fs::remove_dir_all(&pgdata_path) { // this is unlikely, but we will remove the directory on pageserver restart or another bootstrap call - error!("Failed to remove temporary initdb directory '{initdb_path}': {e}"); + error!("Failed to remove temporary initdb directory '{pgdata_path}': {e}"); } } - let pgdata_path = &initdb_path; - let pgdata_lsn = import_datadir::get_lsn_from_controlfile(pgdata_path)?.align(); + let pgdata_lsn = import_datadir::get_lsn_from_controlfile(&pgdata_path)?.align(); + + // Upload the created data dir to S3 + if let Some(storage) = &self.remote_storage { + let pgdata_zstd = import_datadir::create_tar_zst(&pgdata_path).await?; + let pgdata_zstd = Bytes::from(pgdata_zstd); + backoff::retry( + || async { + self::remote_timeline_client::upload_initdb_dir( + storage, + &self.tenant_id, + &timeline_id, + pgdata_zstd.clone(), + ) + .await + }, + |_| false, + 3, + u32::MAX, + "persist_initdb_tar_zst", + // TODO: use a cancellation token (https://github.com/neondatabase/neon/issues/5066) + backoff::Cancel::new(CancellationToken::new(), || unreachable!()), + ) + .await?; + } // Import the contents of the data directory at the initial checkpoint // LSN, and any WAL after that. @@ -2944,7 +3015,7 @@ impl Tenant { import_datadir::import_timeline_from_postgres_datadir( unfinished_timeline, - pgdata_path, + &pgdata_path, pgdata_lsn, ctx, ) @@ -3125,6 +3196,7 @@ impl Tenant { // (only if it is shorter than the real cutoff). max_retention_period: Option, cause: LogicalSizeCalculationCause, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result { let logical_sizes_at_once = self @@ -3147,6 +3219,7 @@ impl Tenant { max_retention_period, &mut shared_cache, cause, + cancel, ctx, ) .await @@ -3159,9 +3232,10 @@ impl Tenant { pub async fn calculate_synthetic_size( &self, cause: LogicalSizeCalculationCause, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result { - let inputs = self.gather_size_inputs(None, cause, ctx).await?; + let inputs = self.gather_size_inputs(None, cause, cancel, ctx).await?; let size = inputs.calculate()?; @@ -3343,42 +3417,54 @@ fn rebase_directory( /// Create the cluster temporarily in 'initdbpath' directory inside the repository /// to get bootstrap data for timeline initialization. -fn run_initdb( +async fn run_initdb( conf: &'static PageServerConf, initdb_target_dir: &Utf8Path, pg_version: u32, -) -> anyhow::Result<()> { - let initdb_bin_path = conf.pg_bin_dir(pg_version)?.join("initdb"); - let initdb_lib_dir = conf.pg_lib_dir(pg_version)?; + cancel: &CancellationToken, +) -> Result<(), InitdbError> { + let initdb_bin_path = conf + .pg_bin_dir(pg_version) + .map_err(InitdbError::Other)? + .join("initdb"); + let initdb_lib_dir = conf.pg_lib_dir(pg_version).map_err(InitdbError::Other)?; info!( "running {} in {}, libdir: {}", initdb_bin_path, initdb_target_dir, initdb_lib_dir, ); - let initdb_output = Command::new(&initdb_bin_path) + let _permit = INIT_DB_SEMAPHORE.acquire().await; + + let initdb_command = tokio::process::Command::new(&initdb_bin_path) .args(["-D", initdb_target_dir.as_ref()]) .args(["-U", &conf.superuser]) .args(["-E", "utf8"]) .arg("--no-instructions") - // This is only used for a temporary installation that is deleted shortly after, - // so no need to fsync it .arg("--no-sync") .env_clear() .env("LD_LIBRARY_PATH", &initdb_lib_dir) .env("DYLD_LIBRARY_PATH", &initdb_lib_dir) - .stdout(Stdio::null()) - .output() - .with_context(|| { - format!( - "failed to execute {} at target dir {}", - initdb_bin_path, initdb_target_dir, - ) - })?; - if !initdb_output.status.success() { - bail!( - "initdb failed: '{}'", - String::from_utf8_lossy(&initdb_output.stderr) - ); + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + // If the `select!` below doesn't finish the `wait_with_output`, + // let the task get `wait()`ed for asynchronously by tokio. + // This means there is a slim chance we can go over the INIT_DB_SEMAPHORE. + // TODO: fix for this is non-trivial, see + // https://github.com/neondatabase/neon/pull/5921#pullrequestreview-1750858021 + // + .kill_on_drop(true) + .spawn()?; + + tokio::select! { + initdb_output = initdb_command.wait_with_output() => { + let initdb_output = initdb_output?; + if !initdb_output.status.success() { + return Err(InitdbError::Failed(initdb_output.status, initdb_output.stderr)); + } + } + _ = cancel.cancelled() => { + return Err(InitdbError::Cancelled); + } } Ok(()) @@ -3424,6 +3510,7 @@ pub async fn dump_layerfile_from_path( pub(crate) mod harness { use bytes::{Bytes, BytesMut}; use once_cell::sync::OnceCell; + use pageserver_api::shard::ShardIndex; use std::fs; use std::sync::Arc; use utils::logging; @@ -3490,6 +3577,7 @@ pub(crate) mod harness { pub tenant_conf: TenantConf, pub tenant_id: TenantId, pub generation: Generation, + pub shard: ShardIndex, pub remote_storage: GenericRemoteStorage, pub remote_fs_dir: Utf8PathBuf, pub deletion_queue: MockDeletionQueue, @@ -3504,6 +3592,7 @@ pub(crate) mod harness { // enable it in case the tests exercise code paths that use // debug_assert_current_span_has_tenant_and_timeline_id logging::TracingErrorLayerEnablement::EnableWithRustLogFilter, + logging::Output::Stdout, ) .expect("Failed to init test logging") }); @@ -3548,6 +3637,7 @@ pub(crate) mod harness { tenant_conf, tenant_id, generation: Generation::new(0xdeadbeef), + shard: ShardIndex::unsharded(), remote_storage, remote_fs_dir, deletion_queue, @@ -3932,7 +4022,13 @@ mod tests { // and compaction works. But it does set the 'cutoff' point so that the cross check // below should fail. tenant - .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) + .gc_iteration( + Some(TIMELINE_ID), + 0x10, + Duration::ZERO, + &CancellationToken::new(), + &ctx, + ) .await?; // try to branch at lsn 25, should fail because we already garbage collected the data @@ -4035,7 +4131,13 @@ mod tests { tline.set_broken("test".to_owned()); tenant - .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) + .gc_iteration( + Some(TIMELINE_ID), + 0x10, + Duration::ZERO, + &CancellationToken::new(), + &ctx, + ) .await?; // The branchpoints should contain all timelines, even ones marked @@ -4081,7 +4183,13 @@ mod tests { .expect("Should have a local timeline"); // this removes layers before lsn 40 (50 minus 10), so there are two remaining layers, image and delta for 31-50 tenant - .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) + .gc_iteration( + Some(TIMELINE_ID), + 0x10, + Duration::ZERO, + &CancellationToken::new(), + &ctx, + ) .await?; assert!(newtline.get(*TEST_KEY, Lsn(0x25), &ctx).await.is_ok()); @@ -4109,7 +4217,13 @@ mod tests { // run gc on parent tenant - .gc_iteration(Some(TIMELINE_ID), 0x10, Duration::ZERO, &ctx) + .gc_iteration( + Some(TIMELINE_ID), + 0x10, + Duration::ZERO, + &CancellationToken::new(), + &ctx, + ) .await?; // Check that the data is still accessible on the branch. @@ -4298,7 +4412,9 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact(&CancellationToken::new(), &ctx).await?; + tline + .compact(&CancellationToken::new(), EnumSet::empty(), &ctx) + .await?; let writer = tline.writer().await; writer @@ -4313,7 +4429,9 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact(&CancellationToken::new(), &ctx).await?; + tline + .compact(&CancellationToken::new(), EnumSet::empty(), &ctx) + .await?; let writer = tline.writer().await; writer @@ -4328,7 +4446,9 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact(&CancellationToken::new(), &ctx).await?; + tline + .compact(&CancellationToken::new(), EnumSet::empty(), &ctx) + .await?; let writer = tline.writer().await; writer @@ -4343,7 +4463,9 @@ mod tests { drop(writer); tline.freeze_and_flush().await?; - tline.compact(&CancellationToken::new(), &ctx).await?; + tline + .compact(&CancellationToken::new(), EnumSet::empty(), &ctx) + .await?; assert_eq!( tline.get(*TEST_KEY, Lsn(0x10), &ctx).await?, @@ -4411,10 +4533,18 @@ mod tests { let cutoff = tline.get_last_record_lsn(); tline - .update_gc_info(Vec::new(), cutoff, Duration::ZERO, &ctx) + .update_gc_info( + Vec::new(), + cutoff, + Duration::ZERO, + &CancellationToken::new(), + &ctx, + ) .await?; tline.freeze_and_flush().await?; - tline.compact(&CancellationToken::new(), &ctx).await?; + tline + .compact(&CancellationToken::new(), EnumSet::empty(), &ctx) + .await?; tline.gc().await?; } @@ -4491,10 +4621,18 @@ mod tests { // Perform a cycle of flush, compact, and GC let cutoff = tline.get_last_record_lsn(); tline - .update_gc_info(Vec::new(), cutoff, Duration::ZERO, &ctx) + .update_gc_info( + Vec::new(), + cutoff, + Duration::ZERO, + &CancellationToken::new(), + &ctx, + ) .await?; tline.freeze_and_flush().await?; - tline.compact(&CancellationToken::new(), &ctx).await?; + tline + .compact(&CancellationToken::new(), EnumSet::empty(), &ctx) + .await?; tline.gc().await?; } @@ -4581,10 +4719,18 @@ mod tests { // Perform a cycle of flush, compact, and GC let cutoff = tline.get_last_record_lsn(); tline - .update_gc_info(Vec::new(), cutoff, Duration::ZERO, &ctx) + .update_gc_info( + Vec::new(), + cutoff, + Duration::ZERO, + &CancellationToken::new(), + &ctx, + ) .await?; tline.freeze_and_flush().await?; - tline.compact(&CancellationToken::new(), &ctx).await?; + tline + .compact(&CancellationToken::new(), EnumSet::empty(), &ctx) + .await?; tline.gc().await?; } diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index 5f8c7f6c59..4ad6a71f67 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -10,6 +10,7 @@ //! use anyhow::Context; use pageserver_api::models; +use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize}; use serde::{Deserialize, Serialize}; use std::num::NonZeroU64; use std::time::Duration; @@ -88,6 +89,14 @@ pub(crate) struct LocationConf { /// The location-specific part of the configuration, describes the operating /// mode of this pageserver for this tenant. pub(crate) mode: LocationMode, + + /// The detailed shard identity. This structure is already scoped within + /// a TenantShardId, but we need the full ShardIdentity to enable calculating + /// key->shard mappings. + #[serde(default = "ShardIdentity::unsharded")] + #[serde(skip_serializing_if = "ShardIdentity::is_unsharded")] + pub(crate) shard: ShardIdentity, + /// The pan-cluster tenant configuration, the same on all locations pub(crate) tenant_conf: TenantConfOpt, } @@ -160,6 +169,8 @@ impl LocationConf { generation, attach_mode: AttachmentMode::Single, }), + // Legacy configuration loads are always from tenants created before sharding existed. + shard: ShardIdentity::unsharded(), tenant_conf, } } @@ -187,6 +198,7 @@ impl LocationConf { fn get_generation(conf: &'_ models::LocationConfig) -> Result { conf.generation + .map(Generation::new) .ok_or_else(|| anyhow::anyhow!("Generation must be set when attaching")) } @@ -226,7 +238,21 @@ impl LocationConf { } }; - Ok(Self { mode, tenant_conf }) + let shard = if conf.shard_count == 0 { + ShardIdentity::unsharded() + } else { + ShardIdentity::new( + ShardNumber(conf.shard_number), + ShardCount(conf.shard_count), + ShardStripeSize(conf.shard_stripe_size), + )? + }; + + Ok(Self { + shard, + mode, + tenant_conf, + }) } } @@ -241,6 +267,7 @@ impl Default for LocationConf { attach_mode: AttachmentMode::Single, }), tenant_conf: TenantConfOpt::default(), + shard: ShardIdentity::unsharded(), } } } diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index a766cca0c5..5988dcd45f 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -792,8 +792,6 @@ pub(crate) async fn set_new_tenant_config( impl TenantManager { /// Gets the attached tenant from the in-memory data, erroring if it's absent, in secondary mode, or is not fitting to the query. /// `active_only = true` allows to query only tenants that are ready for operations, erroring on other kinds of tenants. - /// - /// This method is cancel-safe. pub(crate) fn get_attached_tenant_shard( &self, tenant_shard_id: TenantShardId, @@ -1944,6 +1942,7 @@ pub(crate) async fn immediate_gc( tenant_id: TenantId, timeline_id: TimelineId, gc_req: TimelineGcRequest, + cancel: CancellationToken, ctx: &RequestContext, ) -> Result>, ApiError> { let guard = TENANTS.read().unwrap(); @@ -1960,6 +1959,7 @@ pub(crate) async fn immediate_gc( // Run in task_mgr to avoid race with tenant_detach operation let ctx = ctx.detached_child(TaskKind::GarbageCollector, DownloadBehavior::Download); let (task_done, wait_task_done) = tokio::sync::oneshot::channel(); + // TODO: spawning is redundant now, need to hold the gate task_mgr::spawn( &tokio::runtime::Handle::current(), TaskKind::GarbageCollector, @@ -1969,12 +1969,40 @@ pub(crate) async fn immediate_gc( false, async move { fail::fail_point!("immediate_gc_task_pre"); - let result = tenant - .gc_iteration(Some(timeline_id), gc_horizon, pitr, &ctx) + + #[allow(unused_mut)] + let mut result = tenant + .gc_iteration(Some(timeline_id), gc_horizon, pitr, &cancel, &ctx) .instrument(info_span!("manual_gc", %tenant_id, %timeline_id)) .await; // FIXME: `gc_iteration` can return an error for multiple reasons; we should handle it // better once the types support it. + + #[cfg(feature = "testing")] + { + if let Ok(result) = result.as_mut() { + // why not futures unordered? it seems it needs very much the same task structure + // but would only run on single task. + let mut js = tokio::task::JoinSet::new(); + for layer in std::mem::take(&mut result.doomed_layers) { + js.spawn(layer.wait_drop()); + } + tracing::info!(total = js.len(), "starting to wait for the gc'd layers to be dropped"); + while let Some(res) = js.join_next().await { + res.expect("wait_drop should not panic"); + } + } + + let timeline = tenant.get_timeline(timeline_id, false).ok(); + let rtc = timeline.as_ref().and_then(|x| x.remote_client.as_ref()); + + if let Some(rtc) = rtc { + // layer drops schedule actions on remote timeline client to actually do the + // deletions; don't care just exit fast about the shutdown error + drop(rtc.wait_completion().await); + } + } + match task_done.send(result) { Ok(_) => (), Err(result) => error!("failed to send gc result: {result:?}"), diff --git a/pageserver/src/tenant/remote_timeline_client.rs b/pageserver/src/tenant/remote_timeline_client.rs index bbf6a0c5c5..01c60ca8f8 100644 --- a/pageserver/src/tenant/remote_timeline_client.rs +++ b/pageserver/src/tenant/remote_timeline_client.rs @@ -188,8 +188,10 @@ use anyhow::Context; use camino::Utf8Path; use chrono::{NaiveDateTime, Utc}; +use pageserver_api::shard::ShardIndex; use scopeguard::ScopeGuard; use tokio_util::sync::CancellationToken; +pub(crate) use upload::upload_initdb_dir; use utils::backoff::{ self, exponential_backoff, DEFAULT_BASE_BACKOFF_SECONDS, DEFAULT_MAX_BACKOFF_SECONDS, }; @@ -249,6 +251,8 @@ pub(crate) const FAILED_REMOTE_OP_RETRIES: u32 = 10; // retries. Uploads and deletions are retried forever, though. pub(crate) const FAILED_UPLOAD_WARN_THRESHOLD: u32 = 3; +pub(crate) const INITDB_PATH: &str = "initdb.tar.zst"; + pub enum MaybeDeletedIndexPart { IndexPart(IndexPart), Deleted(IndexPart), @@ -399,6 +403,11 @@ impl RemoteTimelineClient { Ok(()) } + pub(crate) fn get_shard_index(&self) -> ShardIndex { + // TODO: carry this on the struct + ShardIndex::unsharded() + } + pub fn remote_consistent_lsn_projected(&self) -> Option { match &mut *self.upload_queue.lock().unwrap() { UploadQueue::Uninitialized => None, @@ -462,6 +471,7 @@ impl RemoteTimelineClient { &self.storage_impl, &self.tenant_id, &self.timeline_id, + self.get_shard_index(), self.generation, cancel, ) @@ -654,10 +664,10 @@ impl RemoteTimelineClient { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; - let with_generations = + let with_metadata = self.schedule_unlinking_of_layers_from_index_part0(upload_queue, names.iter().cloned()); - self.schedule_deletion_of_unlinked0(upload_queue, with_generations); + self.schedule_deletion_of_unlinked0(upload_queue, with_metadata); // Launch the tasks immediately, if possible self.launch_queued_tasks(upload_queue); @@ -692,7 +702,7 @@ impl RemoteTimelineClient { self: &Arc, upload_queue: &mut UploadQueueInitialized, names: I, - ) -> Vec<(LayerFileName, Generation)> + ) -> Vec<(LayerFileName, LayerFileMetadata)> where I: IntoIterator, { @@ -700,16 +710,17 @@ impl RemoteTimelineClient { // so we don't need update it. Just serialize it. let metadata = upload_queue.latest_metadata.clone(); - // Decorate our list of names with each name's generation, dropping - // names that are unexpectedly missing from our metadata. - let with_generations: Vec<_> = names + // Decorate our list of names with each name's metadata, dropping + // names that are unexpectedly missing from our metadata. This metadata + // is later used when physically deleting layers, to construct key paths. + let with_metadata: Vec<_> = names .into_iter() .filter_map(|name| { let meta = upload_queue.latest_files.remove(&name); if let Some(meta) = meta { upload_queue.latest_files_changes_since_metadata_upload_scheduled += 1; - Some((name, meta.generation)) + Some((name, meta)) } else { // This can only happen if we forgot to to schedule the file upload // before scheduling the delete. Log it because it is a rare/strange @@ -722,9 +733,10 @@ impl RemoteTimelineClient { .collect(); #[cfg(feature = "testing")] - for (name, gen) in &with_generations { - if let Some(unexpected) = upload_queue.dangling_files.insert(name.to_owned(), *gen) { - if &unexpected == gen { + for (name, metadata) in &with_metadata { + let gen = metadata.generation; + if let Some(unexpected) = upload_queue.dangling_files.insert(name.to_owned(), gen) { + if unexpected == gen { tracing::error!("{name} was unlinked twice with same generation"); } else { tracing::error!("{name} was unlinked twice with different generations {gen:?} and {unexpected:?}"); @@ -739,14 +751,14 @@ impl RemoteTimelineClient { self.schedule_index_upload(upload_queue, metadata); } - with_generations + with_metadata } /// Schedules deletion for layer files which have previously been unlinked from the /// `index_part.json` with [`Self::schedule_gc_update`] or [`Self::schedule_compaction_update`]. pub(crate) fn schedule_deletion_of_unlinked( self: &Arc, - layers: Vec<(LayerFileName, Generation)>, + layers: Vec<(LayerFileName, LayerFileMetadata)>, ) -> anyhow::Result<()> { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; @@ -759,16 +771,22 @@ impl RemoteTimelineClient { fn schedule_deletion_of_unlinked0( self: &Arc, upload_queue: &mut UploadQueueInitialized, - with_generations: Vec<(LayerFileName, Generation)>, + with_metadata: Vec<(LayerFileName, LayerFileMetadata)>, ) { - for (name, gen) in &with_generations { - info!("scheduling deletion of layer {}{}", name, gen.get_suffix()); + for (name, meta) in &with_metadata { + info!( + "scheduling deletion of layer {}{} (shard {})", + name, + meta.generation.get_suffix(), + meta.shard + ); } #[cfg(feature = "testing")] - for (name, gen) in &with_generations { + for (name, meta) in &with_metadata { + let gen = meta.generation; match upload_queue.dangling_files.remove(name) { - Some(same) if &same == gen => { /* expected */ } + Some(same) if same == gen => { /* expected */ } Some(other) => { tracing::error!("{name} was unlinked with {other:?} but deleted with {gen:?}"); } @@ -780,7 +798,7 @@ impl RemoteTimelineClient { // schedule the actual deletions let op = UploadOp::Delete(Delete { - layers: with_generations, + layers: with_metadata, }); self.calls_unfinished_metric_begin(&op); upload_queue.queued_operations.push_back(op); @@ -809,23 +827,29 @@ impl RemoteTimelineClient { Ok(()) } - /// /// Wait for all previously scheduled uploads/deletions to complete - /// - pub async fn wait_completion(self: &Arc) -> anyhow::Result<()> { + pub(crate) async fn wait_completion(self: &Arc) -> anyhow::Result<()> { let mut receiver = { let mut guard = self.upload_queue.lock().unwrap(); let upload_queue = guard.initialized_mut()?; - self.schedule_barrier(upload_queue) + self.schedule_barrier0(upload_queue) }; if receiver.changed().await.is_err() { anyhow::bail!("wait_completion aborted because upload queue was stopped"); } + Ok(()) } - fn schedule_barrier( + pub(crate) fn schedule_barrier(self: &Arc) -> anyhow::Result<()> { + let mut guard = self.upload_queue.lock().unwrap(); + let upload_queue = guard.initialized_mut()?; + self.schedule_barrier0(upload_queue); + Ok(()) + } + + fn schedule_barrier0( self: &Arc, upload_queue: &mut UploadQueueInitialized, ) -> tokio::sync::watch::Receiver<()> { @@ -841,6 +865,56 @@ impl RemoteTimelineClient { receiver } + /// Wait for all previously scheduled operations to complete, and then stop. + /// + /// Not cancellation safe + pub(crate) async fn shutdown(self: &Arc) -> Result<(), StopError> { + // On cancellation the queue is left in ackward state of refusing new operations but + // proper stop is yet to be called. On cancel the original or some later task must call + // `stop` or `shutdown`. + let sg = scopeguard::guard((), |_| { + tracing::error!("RemoteTimelineClient::shutdown was cancelled; this should not happen, do not make this into an allowed_error") + }); + + let fut = { + let mut guard = self.upload_queue.lock().unwrap(); + let upload_queue = match &mut *guard { + UploadQueue::Stopped(_) => return Ok(()), + UploadQueue::Uninitialized => return Err(StopError::QueueUninitialized), + UploadQueue::Initialized(ref mut init) => init, + }; + + // if the queue is already stuck due to a shutdown operation which was cancelled, then + // just don't add more of these as they would never complete. + // + // TODO: if launch_queued_tasks were to be refactored to accept a &mut UploadQueue + // in every place we would not have to jump through this hoop, and this method could be + // made cancellable. + if !upload_queue.shutting_down { + upload_queue.shutting_down = true; + upload_queue.queued_operations.push_back(UploadOp::Shutdown); + // this operation is not counted similar to Barrier + + self.launch_queued_tasks(upload_queue); + } + + upload_queue.shutdown_ready.clone().acquire_owned() + }; + + let res = fut.await; + + scopeguard::ScopeGuard::into_inner(sg); + + match res { + Ok(_permit) => unreachable!("shutdown_ready should not have been added permits"), + Err(_closed) => { + // expected + } + } + + self.stop() + } + /// Set the deleted_at field in the remote index file. /// /// This fails if the upload queue has not been `stop()`ed. @@ -894,6 +968,7 @@ impl RemoteTimelineClient { &self.storage_impl, &self.tenant_id, &self.timeline_id, + self.get_shard_index(), self.generation, &index_part_with_deleted_at, ) @@ -952,6 +1027,7 @@ impl RemoteTimelineClient { remote_layer_path( &self.tenant_id, &self.timeline_id, + meta.shard, &file_name, meta.generation, ) @@ -1000,7 +1076,12 @@ impl RemoteTimelineClient { .unwrap_or( // No generation-suffixed indices, assume we are dealing with // a legacy index. - remote_index_path(&self.tenant_id, &self.timeline_id, Generation::none()), + remote_index_path( + &self.tenant_id, + &self.timeline_id, + self.get_shard_index(), + Generation::none(), + ), ); let remaining_layers: Vec = remaining @@ -1071,7 +1152,9 @@ impl RemoteTimelineClient { upload_queue.num_inprogress_deletions == upload_queue.inprogress_tasks.len() } - UploadOp::Barrier(_) => upload_queue.inprogress_tasks.is_empty(), + UploadOp::Barrier(_) | UploadOp::Shutdown => { + upload_queue.inprogress_tasks.is_empty() + } }; // If we cannot launch this task, don't look any further. @@ -1084,6 +1167,13 @@ impl RemoteTimelineClient { break; } + if let UploadOp::Shutdown = next_op { + // leave the op in the queue but do not start more tasks; it will be dropped when + // the stop is called. + upload_queue.shutdown_ready.close(); + break; + } + // We can launch this task. Remove it from the queue first. let next_op = upload_queue.queued_operations.pop_front().unwrap(); @@ -1104,6 +1194,7 @@ impl RemoteTimelineClient { sender.send_replace(()); continue; } + UploadOp::Shutdown => unreachable!("shutdown is intentionally never popped off"), }; // Assign unique ID to this task @@ -1209,6 +1300,7 @@ impl RemoteTimelineClient { &self.storage_impl, &self.tenant_id, &self.timeline_id, + self.get_shard_index(), self.generation, index_part, ) @@ -1229,20 +1321,22 @@ impl RemoteTimelineClient { } res } - UploadOp::Delete(delete) => self - .deletion_queue_client - .push_layers( - self.tenant_id, - self.timeline_id, - self.generation, - delete.layers.clone(), - ) - .await - .map_err(|e| anyhow::anyhow!(e)), - UploadOp::Barrier(_) => { + UploadOp::Delete(delete) => { + pausable_failpoint!("before-delete-layer-pausable"); + self.deletion_queue_client + .push_layers( + self.tenant_id, + self.timeline_id, + self.generation, + delete.layers.clone(), + ) + .await + .map_err(|e| anyhow::anyhow!(e)) + } + unexpected @ UploadOp::Barrier(_) | unexpected @ UploadOp::Shutdown => { // unreachable. Barrier operations are handled synchronously in // launch_queued_tasks - warn!("unexpected Barrier operation in perform_upload_task"); + warn!("unexpected {unexpected:?} operation in perform_upload_task"); break; } }; @@ -1336,7 +1430,7 @@ impl RemoteTimelineClient { upload_queue.num_inprogress_deletions -= 1; None } - UploadOp::Barrier(_) => unreachable!(), + UploadOp::Barrier(..) | UploadOp::Shutdown => unreachable!(), }; // Launch any queued tasks that were unblocked by this one. @@ -1391,7 +1485,7 @@ impl RemoteTimelineClient { reason: "should we track deletes? positive or negative sign?", }, ), - UploadOp::Barrier(_) => { + UploadOp::Barrier(..) | UploadOp::Shutdown => { // we do not account these return None; } @@ -1417,10 +1511,13 @@ impl RemoteTimelineClient { } /// Close the upload queue for new operations and cancel queued operations. + /// + /// Use [`RemoteTimelineClient::shutdown`] for graceful stop. + /// /// In-progress operations will still be running after this function returns. /// Use `task_mgr::shutdown_tasks(None, Some(self.tenant_id), Some(timeline_id))` /// to wait for them to complete, after calling this function. - pub fn stop(&self) -> Result<(), StopError> { + pub(crate) fn stop(&self) -> Result<(), StopError> { // Whichever *task* for this RemoteTimelineClient grabs the mutex first will transition the queue // into stopped state, thereby dropping all off the queued *ops* which haven't become *tasks* yet. // The other *tasks* will come here and observe an already shut down queue and hence simply wrap up their business. @@ -1458,6 +1555,8 @@ impl RemoteTimelineClient { queued_operations: VecDeque::default(), #[cfg(feature = "testing")] dangling_files: HashMap::default(), + shutting_down: false, + shutdown_ready: Arc::new(tokio::sync::Semaphore::new(0)), }; let upload_queue = std::mem::replace( @@ -1515,12 +1614,14 @@ pub fn remote_timeline_path(tenant_id: &TenantId, timeline_id: &TimelineId) -> R pub fn remote_layer_path( tenant_id: &TenantId, timeline_id: &TimelineId, + shard: ShardIndex, layer_file_name: &LayerFileName, generation: Generation, ) -> RemotePath { // Generation-aware key format let path = format!( - "tenants/{tenant_id}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{0}{1}", + "tenants/{tenant_id}{0}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{1}{2}", + shard.get_suffix(), layer_file_name.file_name(), generation.get_suffix() ); @@ -1528,13 +1629,22 @@ pub fn remote_layer_path( RemotePath::from_string(&path).expect("Failed to construct path") } +pub fn remote_initdb_archive_path(tenant_id: &TenantId, timeline_id: &TimelineId) -> RemotePath { + RemotePath::from_string(&format!( + "tenants/{tenant_id}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{INITDB_PATH}" + )) + .expect("Failed to construct path") +} + pub fn remote_index_path( tenant_id: &TenantId, timeline_id: &TimelineId, + shard: ShardIndex, generation: Generation, ) -> RemotePath { RemotePath::from_string(&format!( - "tenants/{tenant_id}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{0}{1}", + "tenants/{tenant_id}{0}/{TIMELINES_SEGMENT_NAME}/{timeline_id}/{1}{2}", + shard.get_suffix(), IndexPart::FILE_NAME, generation.get_suffix() )) @@ -1759,6 +1869,7 @@ mod tests { println!("remote_timeline_dir: {remote_timeline_dir}"); let generation = harness.generation; + let shard = harness.shard; // Create a couple of dummy files, schedule upload for them @@ -1775,7 +1886,7 @@ mod tests { harness.conf, &timeline, name, - LayerFileMetadata::new(contents.len() as u64, generation), + LayerFileMetadata::new(contents.len() as u64, generation, shard), ) }).collect::>(); @@ -1924,7 +2035,7 @@ mod tests { harness.conf, &timeline, layer_file_name_1.clone(), - LayerFileMetadata::new(content_1.len() as u64, harness.generation), + LayerFileMetadata::new(content_1.len() as u64, harness.generation, harness.shard), ); #[derive(Debug, PartialEq, Clone, Copy)] @@ -1989,7 +2100,11 @@ mod tests { assert_eq!(actual_c, expected_c); } - async fn inject_index_part(test_state: &TestSetup, generation: Generation) -> IndexPart { + async fn inject_index_part( + test_state: &TestSetup, + generation: Generation, + shard: ShardIndex, + ) -> IndexPart { // An empty IndexPart, just sufficient to ensure deserialization will succeed let example_metadata = TimelineMetadata::example(); let example_index_part = IndexPart::new( @@ -2010,7 +2125,13 @@ mod tests { std::fs::create_dir_all(remote_timeline_dir).expect("creating test dir should work"); let index_path = test_state.harness.remote_fs_dir.join( - remote_index_path(&test_state.harness.tenant_id, &TIMELINE_ID, generation).get_path(), + remote_index_path( + &test_state.harness.tenant_id, + &TIMELINE_ID, + shard, + generation, + ) + .get_path(), ); eprintln!("Writing {index_path}"); std::fs::write(&index_path, index_part_bytes).unwrap(); @@ -2047,7 +2168,12 @@ mod tests { // Simple case: we are in generation N, load the index from generation N - 1 let generation_n = 5; - let injected = inject_index_part(&test_state, Generation::new(generation_n - 1)).await; + let injected = inject_index_part( + &test_state, + Generation::new(generation_n - 1), + ShardIndex::unsharded(), + ) + .await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected).await; @@ -2065,22 +2191,34 @@ mod tests { // A generation-less IndexPart exists in the bucket, we should find it let generation_n = 5; - let injected_none = inject_index_part(&test_state, Generation::none()).await; + let injected_none = + inject_index_part(&test_state, Generation::none(), ShardIndex::unsharded()).await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected_none).await; // If a more recent-than-none generation exists, we should prefer to load that - let injected_1 = inject_index_part(&test_state, Generation::new(1)).await; + let injected_1 = + inject_index_part(&test_state, Generation::new(1), ShardIndex::unsharded()).await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected_1).await; // If a more-recent-than-me generation exists, we should ignore it. - let _injected_10 = inject_index_part(&test_state, Generation::new(10)).await; + let _injected_10 = + inject_index_part(&test_state, Generation::new(10), ShardIndex::unsharded()).await; assert_got_index_part(&test_state, Generation::new(generation_n), &injected_1).await; // If a directly previous generation exists, _and_ an index exists in my own // generation, I should prefer my own generation. - let _injected_prev = - inject_index_part(&test_state, Generation::new(generation_n - 1)).await; - let injected_current = inject_index_part(&test_state, Generation::new(generation_n)).await; + let _injected_prev = inject_index_part( + &test_state, + Generation::new(generation_n - 1), + ShardIndex::unsharded(), + ) + .await; + let injected_current = inject_index_part( + &test_state, + Generation::new(generation_n), + ShardIndex::unsharded(), + ) + .await; assert_got_index_part( &test_state, Generation::new(generation_n), diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index 6039b01ab8..3b2cb5b599 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -9,6 +9,7 @@ use std::time::Duration; use anyhow::{anyhow, Context}; use camino::Utf8Path; +use pageserver_api::shard::ShardIndex; use tokio::fs; use tokio::io::AsyncWriteExt; use tokio_util::sync::CancellationToken; @@ -53,6 +54,7 @@ pub async fn download_layer_file<'a>( let remote_path = remote_layer_path( &tenant_id, &timeline_id, + layer_metadata.shard, layer_file_name, layer_metadata.generation, ); @@ -213,10 +215,11 @@ async fn do_download_index_part( storage: &GenericRemoteStorage, tenant_id: &TenantId, timeline_id: &TimelineId, + shard: ShardIndex, index_generation: Generation, cancel: CancellationToken, ) -> Result { - let remote_path = remote_index_path(tenant_id, timeline_id, index_generation); + let remote_path = remote_index_path(tenant_id, timeline_id, shard, index_generation); let index_part_bytes = download_retry_forever( || async { @@ -254,6 +257,7 @@ pub(super) async fn download_index_part( storage: &GenericRemoteStorage, tenant_id: &TenantId, timeline_id: &TimelineId, + shard: ShardIndex, my_generation: Generation, cancel: CancellationToken, ) -> Result { @@ -261,8 +265,15 @@ pub(super) async fn download_index_part( if my_generation.is_none() { // Operating without generations: just fetch the generation-less path - return do_download_index_part(storage, tenant_id, timeline_id, my_generation, cancel) - .await; + return do_download_index_part( + storage, + tenant_id, + timeline_id, + shard, + my_generation, + cancel, + ) + .await; } // Stale case: If we were intentionally attached in a stale generation, there may already be a remote @@ -273,6 +284,7 @@ pub(super) async fn download_index_part( storage, tenant_id, timeline_id, + shard, my_generation, cancel.clone(), ) @@ -300,6 +312,7 @@ pub(super) async fn download_index_part( storage, tenant_id, timeline_id, + shard, my_generation.previous(), cancel.clone(), ) @@ -320,8 +333,9 @@ pub(super) async fn download_index_part( } // General case/fallback: if there is no index at my_generation or prev_generation, then list all index_part.json - // objects, and select the highest one with a generation <= my_generation. - let index_prefix = remote_index_path(tenant_id, timeline_id, Generation::none()); + // objects, and select the highest one with a generation <= my_generation. Constructing the prefix is equivalent + // to constructing a full index path with no generation, because the generation is a suffix. + let index_prefix = remote_index_path(tenant_id, timeline_id, shard, Generation::none()); let indices = backoff::retry( || async { storage.list_files(Some(&index_prefix)).await }, |_| false, @@ -347,14 +361,21 @@ pub(super) async fn download_index_part( match max_previous_generation { Some(g) => { tracing::debug!("Found index_part in generation {g:?}"); - do_download_index_part(storage, tenant_id, timeline_id, g, cancel).await + do_download_index_part(storage, tenant_id, timeline_id, shard, g, cancel).await } None => { // Migration from legacy pre-generation state: we have a generation but no prior // attached pageservers did. Try to load from a no-generation path. tracing::info!("No index_part.json* found"); - do_download_index_part(storage, tenant_id, timeline_id, Generation::none(), cancel) - .await + do_download_index_part( + storage, + tenant_id, + timeline_id, + shard, + Generation::none(), + cancel, + ) + .await } } } diff --git a/pageserver/src/tenant/remote_timeline_client/index.rs b/pageserver/src/tenant/remote_timeline_client/index.rs index fa0679c7a2..0abfdeef02 100644 --- a/pageserver/src/tenant/remote_timeline_client/index.rs +++ b/pageserver/src/tenant/remote_timeline_client/index.rs @@ -12,6 +12,7 @@ use crate::tenant::metadata::TimelineMetadata; use crate::tenant::storage_layer::LayerFileName; use crate::tenant::upload_queue::UploadQueueInitialized; use crate::tenant::Generation; +use pageserver_api::shard::ShardIndex; use utils::lsn::Lsn; @@ -25,6 +26,8 @@ pub struct LayerFileMetadata { file_size: u64, pub(crate) generation: Generation, + + pub(crate) shard: ShardIndex, } impl From<&'_ IndexLayerMetadata> for LayerFileMetadata { @@ -32,15 +35,17 @@ impl From<&'_ IndexLayerMetadata> for LayerFileMetadata { LayerFileMetadata { file_size: other.file_size, generation: other.generation, + shard: other.shard, } } } impl LayerFileMetadata { - pub fn new(file_size: u64, generation: Generation) -> Self { + pub fn new(file_size: u64, generation: Generation, shard: ShardIndex) -> Self { LayerFileMetadata { file_size, generation, + shard, } } @@ -128,6 +133,14 @@ impl IndexPart { pub fn get_disk_consistent_lsn(&self) -> Lsn { self.disk_consistent_lsn } + + pub fn from_s3_bytes(bytes: &[u8]) -> Result { + serde_json::from_slice::(bytes) + } + + pub fn to_s3_bytes(&self) -> serde_json::Result> { + serde_json::to_vec(self) + } } impl TryFrom<&UploadQueueInitialized> for IndexPart { @@ -153,6 +166,10 @@ pub struct IndexLayerMetadata { #[serde(default = "Generation::none")] #[serde(skip_serializing_if = "Generation::is_none")] pub generation: Generation, + + #[serde(default = "ShardIndex::unsharded")] + #[serde(skip_serializing_if = "ShardIndex::is_unsharded")] + pub shard: ShardIndex, } impl From for IndexLayerMetadata { @@ -160,6 +177,7 @@ impl From for IndexLayerMetadata { IndexLayerMetadata { file_size: other.file_size, generation: other.generation, + shard: other.shard, } } } @@ -187,13 +205,15 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), @@ -201,7 +221,7 @@ mod tests { deleted_at: None, }; - let part = serde_json::from_str::(example).unwrap(); + let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap(); assert_eq!(part, expected); } @@ -225,13 +245,15 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), @@ -239,7 +261,7 @@ mod tests { deleted_at: None, }; - let part = serde_json::from_str::(example).unwrap(); + let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap(); assert_eq!(part, expected); } @@ -264,13 +286,15 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), @@ -279,7 +303,7 @@ mod tests { "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap()) }; - let part = serde_json::from_str::(example).unwrap(); + let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap(); assert_eq!(part, expected); } @@ -323,7 +347,7 @@ mod tests { deleted_at: None, }; - let empty_layers_parsed = serde_json::from_str::(empty_layers_json).unwrap(); + let empty_layers_parsed = IndexPart::from_s3_bytes(empty_layers_json.as_bytes()).unwrap(); assert_eq!(empty_layers_parsed, expected); } @@ -346,22 +370,24 @@ mod tests { layer_metadata: HashMap::from([ ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__0000000001696070-00000000016960E9".parse().unwrap(), IndexLayerMetadata { file_size: 25600000, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }), ("000000000000000000000000000000000000-FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF__00000000016B59D8-00000000016B5A51".parse().unwrap(), IndexLayerMetadata { // serde_json should always parse this but this might be a double with jq for // example. file_size: 9007199254741001, - generation: Generation::none() + generation: Generation::none(), + shard: ShardIndex::unsharded() }) ]), disk_consistent_lsn: "0/16960E8".parse::().unwrap(), metadata: TimelineMetadata::from_bytes(&[113,11,159,210,0,54,0,4,0,0,0,0,1,105,96,232,1,0,0,0,0,1,105,96,112,0,0,0,0,0,0,0,0,0,0,0,0,0,1,105,96,112,0,0,0,0,1,105,96,112,0,0,0,14,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]).unwrap(), deleted_at: Some(chrono::NaiveDateTime::parse_from_str( - "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap()) + "2023-07-31T09:00:00.123000000", "%Y-%m-%dT%H:%M:%S.%f").unwrap()), }; - let part = serde_json::from_str::(example).unwrap(); + let part = IndexPart::from_s3_bytes(example.as_bytes()).unwrap(); assert_eq!(part, expected); } } diff --git a/pageserver/src/tenant/remote_timeline_client/upload.rs b/pageserver/src/tenant/remote_timeline_client/upload.rs index 0a37a8f283..789a10cf54 100644 --- a/pageserver/src/tenant/remote_timeline_client/upload.rs +++ b/pageserver/src/tenant/remote_timeline_client/upload.rs @@ -1,15 +1,19 @@ //! Helper functions to upload files to remote storage with a RemoteStorage use anyhow::{bail, Context}; +use bytes::Bytes; use camino::Utf8Path; use fail::fail_point; +use pageserver_api::shard::ShardIndex; use std::io::ErrorKind; use tokio::fs; use super::Generation; use crate::{ config::PageServerConf, - tenant::remote_timeline_client::{index::IndexPart, remote_index_path, remote_path}, + tenant::remote_timeline_client::{ + index::IndexPart, remote_index_path, remote_initdb_archive_path, remote_path, + }, }; use remote_storage::GenericRemoteStorage; use utils::id::{TenantId, TimelineId}; @@ -23,6 +27,7 @@ pub(super) async fn upload_index_part<'a>( storage: &'a GenericRemoteStorage, tenant_id: &TenantId, timeline_id: &TimelineId, + shard: ShardIndex, generation: Generation, index_part: &'a IndexPart, ) -> anyhow::Result<()> { @@ -33,12 +38,13 @@ pub(super) async fn upload_index_part<'a>( }); pausable_failpoint!("before-upload-index-pausable"); - let index_part_bytes = - serde_json::to_vec(&index_part).context("serialize index part file into bytes")?; + let index_part_bytes = index_part + .to_s3_bytes() + .context("serialize index part file into bytes")?; let index_part_size = index_part_bytes.len(); let index_part_bytes = tokio::io::BufReader::new(std::io::Cursor::new(index_part_bytes)); - let remote_path = remote_index_path(tenant_id, timeline_id, generation); + let remote_path = remote_index_path(tenant_id, timeline_id, shard, generation); storage .upload_storage_object(Box::new(index_part_bytes), index_part_size, &remote_path) .await @@ -103,3 +109,22 @@ pub(super) async fn upload_timeline_layer<'a>( Ok(()) } + +/// Uploads the given `initdb` data to the remote storage. +pub(crate) async fn upload_initdb_dir( + storage: &GenericRemoteStorage, + tenant_id: &TenantId, + timeline_id: &TimelineId, + initdb_dir: Bytes, +) -> anyhow::Result<()> { + tracing::trace!("uploading initdb dir"); + + let size = initdb_dir.len(); + let bytes = tokio::io::BufReader::new(std::io::Cursor::new(initdb_dir)); + + let remote_path = remote_initdb_archive_path(tenant_id, timeline_id); + storage + .upload_storage_object(bytes, size, &remote_path) + .await + .with_context(|| format!("upload initdb dir for '{tenant_id} / {timeline_id}'")) +} diff --git a/pageserver/src/tenant/size.rs b/pageserver/src/tenant/size.rs index a85dc9231c..e0b1652d98 100644 --- a/pageserver/src/tenant/size.rs +++ b/pageserver/src/tenant/size.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use anyhow::{bail, Context}; use tokio::sync::oneshot::error::RecvError; use tokio::sync::Semaphore; +use tokio_util::sync::CancellationToken; use crate::context::RequestContext; use crate::pgdatadir_mapping::CalculateLogicalSizeError; @@ -113,11 +114,12 @@ pub(super) async fn gather_inputs( max_retention_period: Option, logical_size_cache: &mut HashMap<(TimelineId, Lsn), u64>, cause: LogicalSizeCalculationCause, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result { // refresh is needed to update gc related pitr_cutoff and horizon_cutoff tenant - .refresh_gc_info(ctx) + .refresh_gc_info(cancel, ctx) .await .context("Failed to refresh gc_info before gathering inputs")?; diff --git a/pageserver/src/tenant/storage_layer/layer.rs b/pageserver/src/tenant/storage_layer/layer.rs index f28f1c9444..c27c3e69ed 100644 --- a/pageserver/src/tenant/storage_layer/layer.rs +++ b/pageserver/src/tenant/storage_layer/layer.rs @@ -3,6 +3,7 @@ use camino::{Utf8Path, Utf8PathBuf}; use pageserver_api::models::{ HistoricLayerInfo, LayerAccessKind, LayerResidenceEventReason, LayerResidenceStatus, }; +use pageserver_api::shard::ShardIndex; use std::ops::Range; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Weak}; @@ -96,6 +97,7 @@ impl Layer { desc, None, metadata.generation, + metadata.shard, ))); debug_assert!(owner.0.needs_download_blocking().unwrap().is_some()); @@ -136,6 +138,7 @@ impl Layer { desc, Some(inner), metadata.generation, + metadata.shard, ) })); @@ -179,6 +182,7 @@ impl Layer { desc, Some(inner), timeline.generation, + timeline.get_shard_index(), ) })); @@ -322,6 +326,24 @@ impl Layer { Ok(()) } + + /// Waits until this layer has been dropped (and if needed, local garbage collection and remote + /// deletion scheduling has completed). + /// + /// Does not start garbage collection, use [`Self::garbage_collect_on_drop`] for that + /// separatedly. + #[cfg(feature = "testing")] + pub(crate) fn wait_drop(&self) -> impl std::future::Future + 'static { + let mut rx = self.0.status.subscribe(); + + async move { + loop { + if let Err(tokio::sync::broadcast::error::RecvError::Closed) = rx.recv().await { + break; + } + } + } + } } /// The download-ness ([`DownloadedLayer`]) can be either resident or wanted evicted. @@ -426,6 +448,15 @@ struct LayerInner { /// For loaded layers (resident or evicted) this comes from [`LayerFileMetadata::generation`], /// for created layers from [`Timeline::generation`]. generation: Generation, + + /// The shard of this Layer. + /// + /// For layers created in this process, this will always be the [`ShardIndex`] of the + /// current `ShardIdentity`` (TODO: add link once it's introduced). + /// + /// For loaded layers, this may be some other value if the tenant has undergone + /// a shard split since the layer was originally written. + shard: ShardIndex, } impl std::fmt::Display for LayerInner { @@ -459,13 +490,17 @@ impl Drop for LayerInner { let path = std::mem::take(&mut self.path); let file_name = self.layer_desc().filename(); - let gen = self.generation; let file_size = self.layer_desc().file_size; let timeline = self.timeline.clone(); + let meta = self.metadata(); + let status = self.status.clone(); crate::task_mgr::BACKGROUND_RUNTIME.spawn_blocking(move || { let _g = span.entered(); + // carry this until we are finished for [`Layer::wait_drop`] support + let _status = status; + let removed = match std::fs::remove_file(path) { Ok(()) => true, Err(e) if e.kind() == std::io::ErrorKind::NotFound => { @@ -489,7 +524,7 @@ impl Drop for LayerInner { timeline.metrics.resident_physical_size_sub(file_size); } if let Some(remote_client) = timeline.remote_client.as_ref() { - let res = remote_client.schedule_deletion_of_unlinked(vec![(file_name, gen)]); + let res = remote_client.schedule_deletion_of_unlinked(vec![(file_name, meta)]); if let Err(e) = res { // test_timeline_deletion_with_files_stuck_in_upload_queue is good at @@ -523,6 +558,7 @@ impl LayerInner { desc: PersistentLayerDesc, downloaded: Option>, generation: Generation, + shard: ShardIndex, ) -> Self { let path = conf .timeline_path(&timeline.tenant_id, &timeline.timeline_id) @@ -550,6 +586,7 @@ impl LayerInner { status: tokio::sync::broadcast::channel(1).0, consecutive_failures: AtomicUsize::new(0), generation, + shard, } } @@ -1077,7 +1114,7 @@ impl LayerInner { } fn metadata(&self) -> LayerFileMetadata { - LayerFileMetadata::new(self.desc.file_size, self.generation) + LayerFileMetadata::new(self.desc.file_size, self.generation, self.shard) } } @@ -1401,6 +1438,7 @@ impl Default for LayerImplMetrics { ) .unwrap(); + // reminder: this will be pageserver_layer_gcs_count_total with "_total" suffix let gcs = metrics::register_int_counter_vec!( "pageserver_layer_gcs_count", "Garbage collections started and completed in the Layer implementation", diff --git a/pageserver/src/tenant/tasks.rs b/pageserver/src/tenant/tasks.rs index 27dff9f54a..860bb255ca 100644 --- a/pageserver/src/tenant/tasks.rs +++ b/pageserver/src/tenant/tasks.rs @@ -261,7 +261,7 @@ async fn gc_loop(tenant: Arc, cancel: CancellationToken) { } else { // Run gc let res = tenant - .gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), &ctx) + .gc_iteration(None, gc_horizon, tenant.get_pitr_interval(), &cancel, &ctx) .await; if let Err(e) = res { let wait_duration = backoff::exponential_backoff_duration_seconds( diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 7ae7b7e7e4..a7cf427de5 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -10,6 +10,7 @@ mod walreceiver; use anyhow::{anyhow, bail, ensure, Context, Result}; use bytes::Bytes; use camino::{Utf8Path, Utf8PathBuf}; +use enumset::EnumSet; use fail::fail_point; use itertools::Itertools; use pageserver_api::models::{ @@ -61,6 +62,7 @@ use crate::pgdatadir_mapping::{is_rel_fsm_block_key, is_rel_vm_block_key}; use crate::pgdatadir_mapping::{BlockNumber, CalculateLogicalSizeError}; use crate::tenant::config::{EvictionPolicy, TenantConfOpt}; use pageserver_api::reltag::RelTag; +use pageserver_api::shard::ShardIndex; use postgres_connection::PgConnectionConfig; use postgres_ffi::to_pg_timestamp; @@ -249,14 +251,6 @@ pub struct Timeline { /// to be notified when layer flushing has finished, subscribe to the layer_flush_done channel layer_flush_done_tx: tokio::sync::watch::Sender<(u64, Result<(), FlushLayerError>)>, - /// Layer removal lock. - /// A lock to ensure that no layer of the timeline is removed concurrently by other tasks. - /// This lock is acquired in [`Timeline::gc`] and [`Timeline::compact`]. - /// This is an `Arc` lock because we need an owned - /// lock guard in functions that will be spawned to tokio I/O pool (which requires `'static`). - /// Note that [`DeleteTimelineFlow`] uses `delete_progress` field. - pub(super) layer_removal_cs: Arc>, - // Needed to ensure that we can't create a branch at a point that was already garbage collected pub latest_gc_cutoff_lsn: Rcu, @@ -317,6 +311,24 @@ pub struct Timeline { /// Cancellation token scoped to this timeline: anything doing long-running work relating /// to the timeline should drop out when this token fires. pub(crate) cancel: CancellationToken, + + /// Make sure we only have one running compaction at a time in tests. + /// + /// Must only be taken in two places: + /// - [`Timeline::compact`] (this file) + /// - [`delete::delete_local_layer_files`] + /// + /// Timeline deletion will acquire both compaction and gc locks in whatever order. + compaction_lock: tokio::sync::Mutex<()>, + + /// Make sure we only have one running gc at a time. + /// + /// Must only be taken in two places: + /// - [`Timeline::gc`] (this file) + /// - [`delete::delete_local_layer_files`] + /// + /// Timeline deletion will acquire both compaction and gc locks in whatever order. + gc_lock: tokio::sync::Mutex<()>, } pub struct WalReceiverInfo { @@ -437,6 +449,11 @@ pub enum LogicalSizeCalculationCause { TenantSizeHandler, } +#[derive(enumset::EnumSetType)] +pub(crate) enum CompactFlags { + ForceRepartition, +} + /// Public interface functions impl Timeline { /// Get the LSN where this branch was created @@ -694,8 +711,11 @@ impl Timeline { pub(crate) async fn compact( self: &Arc, cancel: &CancellationToken, + flags: EnumSet, ctx: &RequestContext, ) -> Result<(), CompactionError> { + let _g = self.compaction_lock.lock().await; + // this wait probably never needs any "long time spent" logging, because we already nag if // compaction task goes over it's period (20s) which is quite often in production. let _permit = match super::tasks::concurrent_background_tasks_rate_limit( @@ -750,7 +770,7 @@ impl Timeline { // Below are functions compact_level0() and create_image_layers() // but they are a bit ad hoc and don't quite work like it's explained // above. Rewrite it. - let layer_removal_cs = Arc::new(self.layer_removal_cs.clone().lock_owned().await); + // Is the timeline being deleted? if self.is_stopping() { trace!("Dropping out of compaction on timeline shutdown"); @@ -766,6 +786,7 @@ impl Timeline { .repartition( self.get_last_record_lsn(), self.get_compaction_target_size(), + flags, ctx, ) .await @@ -790,8 +811,7 @@ impl Timeline { // 3. Compact let timer = self.metrics.compact_time_histo.start_timer(); - self.compact_level0(layer_removal_cs.clone(), target_file_size, ctx) - .await?; + self.compact_level0(target_file_size, ctx).await?; timer.stop_and_record(); if let Some(remote_client) = &self.remote_client { @@ -937,7 +957,7 @@ impl Timeline { // what is problematic is the shutting down of RemoteTimelineClient, because // obviously it does not make sense to stop while we wait for it, but what // about corner cases like s3 suddenly hanging up? - if let Err(e) = client.wait_completion().await { + if let Err(e) = client.shutdown().await { // Non-fatal. Shutdown is infallible. Failures to flush just mean that // we have some extra WAL replay to do next time the timeline starts. warn!("failed to flush to remote storage: {e:#}"); @@ -1192,16 +1212,6 @@ impl Timeline { remote_client: &Arc, layers_to_evict: &[Layer], ) -> anyhow::Result>>> { - // ensure that the layers have finished uploading - // (don't hold the layer_removal_cs while we do it, we're not removing anything yet) - remote_client - .wait_completion() - .await - .context("wait for layer upload ops to complete")?; - - // now lock out layer removal (compaction, gc, timeline deletion) - let _layer_removal_guard = self.layer_removal_cs.lock().await; - { // to avoid racing with detach and delete_timeline let state = self.current_state(); @@ -1412,7 +1422,6 @@ impl Timeline { layer_flush_done_tx, write_lock: tokio::sync::Mutex::new(()), - layer_removal_cs: Default::default(), gc_info: std::sync::RwLock::new(GcInfo { retain_lsns: Vec::new(), @@ -1451,6 +1460,9 @@ impl Timeline { initial_logical_size_attempt: Mutex::new(initial_logical_size_attempt), cancel, gate: Gate::new(format!("Timeline<{tenant_id}/{timeline_id}>")), + + compaction_lock: tokio::sync::Mutex::default(), + gc_lock: tokio::sync::Mutex::default(), }; result.repartition_threshold = result.get_checkpoint_distance() / REPARTITION_FREQ_IN_CHECKPOINT_DISTANCE; @@ -1589,6 +1601,7 @@ impl Timeline { // Copy to move into the task we're about to spawn let generation = self.generation; + let shard = self.get_shard_index(); let this = self.myself.upgrade().expect("&self method holds the arc"); let (loaded_layers, needs_cleanup, total_physical_size) = tokio::task::spawn_blocking({ @@ -1637,6 +1650,7 @@ impl Timeline { index_part.as_ref(), disk_consistent_lsn, generation, + shard, ); let mut loaded_layers = Vec::new(); @@ -1711,6 +1725,30 @@ impl Timeline { if let Some(rtc) = self.remote_client.as_ref() { rtc.schedule_layer_file_deletion(&needs_cleanup)?; rtc.schedule_index_upload_for_file_changes()?; + // This barrier orders above DELETEs before any later operations. + // This is critical because code executing after the barrier might + // create again objects with the same key that we just scheduled for deletion. + // For example, if we just scheduled deletion of an image layer "from the future", + // later compaction might run again and re-create the same image layer. + // "from the future" here means an image layer whose LSN is > IndexPart::disk_consistent_lsn. + // "same" here means same key range and LSN. + // + // Without a barrier between above DELETEs and the re-creation's PUTs, + // the upload queue may execute the PUT first, then the DELETE. + // In our example, we will end up with an IndexPart referencing a non-existent object. + // + // 1. a future image layer is created and uploaded + // 2. ps restart + // 3. the future layer from (1) is deleted during load layer map + // 4. image layer is re-created and uploaded + // 5. deletion queue would like to delete (1) but actually deletes (4) + // 6. delete by name works as expected, but it now deletes the wrong (later) version + // + // See https://github.com/neondatabase/neon/issues/5878 + // + // NB: generation numbers naturally protect against this because they disambiguate + // (1) and (4) + rtc.schedule_barrier()?; // Tenant::create_timeline will wait for these uploads to happen before returning, or // on retry. } @@ -2525,7 +2563,12 @@ impl Timeline { // Note: The 'ctx' in use here has DownloadBehavior::Error. We should not // require downloading anything during initial import. let (partitioning, _lsn) = self - .repartition(self.initdb_lsn, self.get_compaction_target_size(), ctx) + .repartition( + self.initdb_lsn, + self.get_compaction_target_size(), + EnumSet::empty(), + ctx, + ) .await?; if self.cancel.is_cancelled() { @@ -2563,6 +2606,8 @@ impl Timeline { ) }; + pausable_failpoint!("flush-layer-cancel-after-writing-layer-out-pausable"); + if self.cancel.is_cancelled() { return Err(FlushLayerError::Cancelled); } @@ -2744,12 +2789,16 @@ impl Timeline { &self, lsn: Lsn, partition_size: u64, + flags: EnumSet, ctx: &RequestContext, ) -> anyhow::Result<(KeyPartitioning, Lsn)> { { let partitioning_guard = self.partitioning.lock().unwrap(); let distance = lsn.0 - partitioning_guard.1 .0; - if partitioning_guard.1 != Lsn(0) && distance <= self.repartition_threshold { + if partitioning_guard.1 != Lsn(0) + && distance <= self.repartition_threshold + && !flags.contains(CompactFlags::ForceRepartition) + { debug!( distance, threshold = self.repartition_threshold, @@ -3104,13 +3153,8 @@ impl TryFrom for CompactLevel0Phase1Stats { impl Timeline { /// Level0 files first phase of compaction, explained in the [`Self::compact`] comment. - /// - /// This method takes the `_layer_removal_cs` guard to highlight it required downloads are - /// returned as an error. If the `layer_removal_cs` boundary is changed not to be taken in the - /// start of level0 files compaction, the on-demand download should be revisited as well. async fn compact_level0_phase1( self: &Arc, - _layer_removal_cs: Arc>, guard: tokio::sync::OwnedRwLockReadGuard, mut stats: CompactLevel0Phase1StatsBuilder, target_file_size: u64, @@ -3197,8 +3241,6 @@ impl Timeline { let mut prev_lsn_end = first_level0_delta.layer_desc().lsn_range.end; let mut deltas_to_compact = Vec::with_capacity(level0_deltas.len()); - // FIXME: downloading while holding layer_removal_cs is not great, but we will remove that - // soon deltas_to_compact.push(first_level0_delta.download_and_keep_resident().await?); for l in level0_deltas_iter { let lsn_range = &l.layer_desc().lsn_range; @@ -3548,7 +3590,6 @@ impl Timeline { /// async fn compact_level0( self: &Arc, - layer_removal_cs: Arc>, target_file_size: u64, ctx: &RequestContext, ) -> Result<(), CompactionError> { @@ -3570,16 +3611,9 @@ impl Timeline { let now = tokio::time::Instant::now(); stats.read_lock_acquisition_micros = DurationRecorder::Recorded(RecordedDuration(now - begin), now); - let layer_removal_cs = layer_removal_cs.clone(); - self.compact_level0_phase1( - layer_removal_cs, - phase1_layers_locked, - stats, - target_file_size, - &ctx, - ) - .instrument(phase1_span) - .await? + self.compact_level0_phase1(phase1_layers_locked, stats, target_file_size, &ctx) + .instrument(phase1_span) + .await? }; if new_layers.is_empty() && deltas_to_compact.is_empty() { @@ -3587,17 +3621,6 @@ impl Timeline { return Ok(()); } - // Before deleting any layers, we need to wait for their upload ops to finish. - // See remote_timeline_client module level comment on consistency. - // Do it here because we don't want to hold self.layers.write() while waiting. - if let Some(remote_client) = &self.remote_client { - debug!("waiting for upload ops to complete"); - remote_client - .wait_completion() - .await - .context("wait for layer upload ops to complete")?; - } - let mut guard = self.layers.write().await; let mut duplicated_layers = HashSet::new(); @@ -3629,12 +3652,7 @@ impl Timeline { }; // deletion will happen later, the layer file manager calls garbage_collect_on_drop - guard.finish_compact_l0( - &layer_removal_cs, - &remove_layers, - &insert_layers, - &self.metrics, - ); + guard.finish_compact_l0(&remove_layers, &insert_layers, &self.metrics); if let Some(remote_client) = self.remote_client.as_ref() { remote_client.schedule_compaction_update(&remove_layers, &new_layers)?; @@ -3685,6 +3703,7 @@ impl Timeline { retain_lsns: Vec, cutoff_horizon: Lsn, pitr: Duration, + cancel: &CancellationToken, ctx: &RequestContext, ) -> anyhow::Result<()> { // First, calculate pitr_cutoff_timestamp and then convert it to LSN. @@ -3698,7 +3717,10 @@ impl Timeline { if let Some(pitr_cutoff_timestamp) = now.checked_sub(pitr) { let pitr_timestamp = to_pg_timestamp(pitr_cutoff_timestamp); - match self.find_lsn_for_timestamp(pitr_timestamp, ctx).await? { + match self + .find_lsn_for_timestamp(pitr_timestamp, cancel, ctx) + .await? + { LsnForTimestamp::Present(lsn) => lsn, LsnForTimestamp::Future(lsn) => { // The timestamp is in the future. That sounds impossible, @@ -3741,19 +3763,17 @@ impl Timeline { Ok(()) } - /// /// Garbage collect layer files on a timeline that are no longer needed. /// /// Currently, we don't make any attempt at removing unneeded page versions /// within a layer file. We can only remove the whole file if it's fully /// obsolete. - /// pub(super) async fn gc(&self) -> anyhow::Result { + let _g = self.gc_lock.lock().await; let timer = self.metrics.garbage_collect_histo.start_timer(); fail_point!("before-timeline-gc"); - let layer_removal_cs = Arc::new(self.layer_removal_cs.clone().lock_owned().await); // Is the timeline being deleted? if self.is_stopping() { anyhow::bail!("timeline is Stopping"); @@ -3771,13 +3791,7 @@ impl Timeline { let new_gc_cutoff = Lsn::min(horizon_cutoff, pitr_cutoff); let res = self - .gc_timeline( - layer_removal_cs.clone(), - horizon_cutoff, - pitr_cutoff, - retain_lsns, - new_gc_cutoff, - ) + .gc_timeline(horizon_cutoff, pitr_cutoff, retain_lsns, new_gc_cutoff) .instrument( info_span!("gc_timeline", timeline_id = %self.timeline_id, cutoff = %new_gc_cutoff), ) @@ -3791,7 +3805,6 @@ impl Timeline { async fn gc_timeline( &self, - layer_removal_cs: Arc>, horizon_cutoff: Lsn, pitr_cutoff: Lsn, retain_lsns: Vec, @@ -3829,17 +3842,6 @@ impl Timeline { debug!("retain_lsns: {:?}", retain_lsns); - // Before deleting any layers, we need to wait for their upload ops to finish. - // See storage_sync module level comment on consistency. - // Do it here because we don't want to hold self.layers.write() while waiting. - if let Some(remote_client) = &self.remote_client { - debug!("waiting for upload ops to complete"); - remote_client - .wait_completion() - .await - .context("wait for layer upload ops to complete")?; - } - let mut layers_to_remove = Vec::new(); let mut wanted_image_layers = KeySpaceRandomAccum::default(); @@ -3955,6 +3957,11 @@ impl Timeline { // // This does not in fact have any effect as we no longer consider local metadata unless // running without remote storage. + // + // This unconditionally schedules also an index_part.json update, even though, we will + // be doing one a bit later with the unlinked gc'd layers. + // + // TODO: remove when implementing . self.update_metadata_file(self.disk_consistent_lsn.load(), None) .await?; @@ -3969,11 +3976,16 @@ impl Timeline { remote_client.schedule_gc_update(&gc_layers)?; } - guard.finish_gc_timeline(&layer_removal_cs, gc_layers); + guard.finish_gc_timeline(&gc_layers); if result.layers_removed != 0 { fail_point!("after-timeline-gc-removed-layers"); } + + #[cfg(feature = "testing")] + { + result.doomed_layers = gc_layers; + } } info!( @@ -3985,9 +3997,7 @@ impl Timeline { Ok(result) } - /// /// Reconstruct a value, using the given base image and WAL records in 'data'. - /// async fn reconstruct_value( &self, key: Key, @@ -4317,6 +4327,11 @@ impl Timeline { resident_layers, } } + + pub(crate) fn get_shard_index(&self) -> ShardIndex { + // TODO: carry this on the struct + ShardIndex::unsharded() + } } type TraversalPathItem = ( diff --git a/pageserver/src/tenant/timeline/delete.rs b/pageserver/src/tenant/timeline/delete.rs index 56a99a25cf..fefeafb7d3 100644 --- a/pageserver/src/tenant/timeline/delete.rs +++ b/pageserver/src/tenant/timeline/delete.rs @@ -110,40 +110,11 @@ async fn set_deleted_in_remote_index(timeline: &Timeline) -> Result<(), DeleteTi Ok(()) } -// We delete local files first, so if pageserver restarts after local files deletion then remote deletion is not continued. -// This can be solved with inversion of these steps. But even if these steps are inverted then, when index_part.json -// gets deleted there is no way to distinguish between "this timeline is good, we just didnt upload it to remote" -// and "this timeline is deleted we should continue with removal of local state". So to avoid the ambiguity we use a mark file. -// After index part is deleted presence of this mark file indentifies that it was a deletion intention. -// So we can just remove the mark file. -async fn create_delete_mark( - conf: &PageServerConf, - tenant_id: TenantId, - timeline_id: TimelineId, -) -> Result<(), DeleteTimelineError> { - fail::fail_point!("timeline-delete-before-delete-mark", |_| { - Err(anyhow::anyhow!( - "failpoint: timeline-delete-before-delete-mark" - ))? - }); - let marker_path = conf.timeline_delete_mark_file_path(tenant_id, timeline_id); - - // Note: we're ok to replace existing file. - let _ = std::fs::OpenOptions::new() - .write(true) - .create(true) - .open(&marker_path) - .with_context(|| format!("could not create delete marker file {marker_path:?}"))?; - - crashsafe::fsync_file_and_parent(&marker_path).context("sync_mark")?; - Ok(()) -} - -/// Grab the layer_removal_cs lock, and actually perform the deletion. +/// Grab the compaction and gc locks, and actually perform the deletion. /// -/// This lock prevents prevents GC or compaction from running at the same time. -/// The GC task doesn't register itself with the timeline it's operating on, -/// so it might still be running even though we called `shutdown_tasks`. +/// The locks prevent GC or compaction from running at the same time. The background tasks do not +/// register themselves with the timeline it's operating on, so it might still be running even +/// though we called `shutdown_tasks`. /// /// Note that there are still other race conditions between /// GC, compaction and timeline deletion. See @@ -151,14 +122,19 @@ async fn create_delete_mark( /// /// No timeout here, GC & Compaction should be responsive to the /// `TimelineState::Stopping` change. -async fn delete_local_layer_files( +// pub(super): documentation link +pub(super) async fn delete_local_layer_files( conf: &PageServerConf, tenant_id: TenantId, timeline: &Timeline, ) -> anyhow::Result<()> { - info!("waiting for layer_removal_cs.lock()"); - let layer_removal_guard = timeline.layer_removal_cs.lock().await; - info!("got layer_removal_cs.lock(), deleting layer files"); + let guards = async { tokio::join!(timeline.gc_lock.lock(), timeline.compaction_lock.lock()) }; + let guards = crate::timed( + guards, + "acquire gc and compaction locks", + std::time::Duration::from_secs(5), + ) + .await; // NB: storage_sync upload tasks that reference these layers have been cancelled // by the caller. @@ -179,8 +155,8 @@ async fn delete_local_layer_files( // because of a previous failure/cancellation at/after // failpoint timeline-delete-after-rm. // - // It can also happen if we race with tenant detach, because, - // it doesn't grab the layer_removal_cs lock. + // ErrorKind::NotFound can also happen if we race with tenant detach, because, + // no locks are shared. // // For now, log and continue. // warn! level is technically not appropriate for the @@ -248,8 +224,8 @@ async fn delete_local_layer_files( .with_context(|| format!("Failed to remove: {}", entry.path().display()))?; } - info!("finished deleting layer files, releasing layer_removal_cs.lock()"); - drop(layer_removal_guard); + info!("finished deleting layer files, releasing locks"); + drop(guards); fail::fail_point!("timeline-delete-after-rm", |_| { Err(anyhow::anyhow!("failpoint: timeline-delete-after-rm"))? @@ -311,6 +287,8 @@ async fn cleanup_remaining_timeline_fs_traces( .context("fsync_pre_mark_remove")?; // Remove delete mark + // TODO: once we are confident that no more exist in the field, remove this + // line. It cleans up a legacy marker file that might in rare cases be present. tokio::fs::remove_file(conf.timeline_delete_mark_file_path(tenant_id, timeline_id)) .await .or_else(fs_ext::ignore_not_found) @@ -391,8 +369,6 @@ impl DeleteTimelineFlow { set_deleted_in_remote_index(&timeline).await?; - create_delete_mark(tenant.conf, timeline.tenant_id, timeline.timeline_id).await?; - fail::fail_point!("timeline-delete-before-schedule", |_| { Err(anyhow::anyhow!( "failpoint: timeline-delete-before-schedule" @@ -464,10 +440,6 @@ impl DeleteTimelineFlow { guard.mark_in_progress()?; - // Note that delete mark can be missing on resume - // because we create delete mark after we set deleted_at in the index part. - create_delete_mark(tenant.conf, tenant.tenant_id, timeline_id).await?; - Self::schedule_background(guard, tenant.conf, tenant, timeline); Ok(()) diff --git a/pageserver/src/tenant/timeline/eviction_task.rs b/pageserver/src/tenant/timeline/eviction_task.rs index 79bc434a2a..2d0f1c609b 100644 --- a/pageserver/src/tenant/timeline/eviction_task.rs +++ b/pageserver/src/tenant/timeline/eviction_task.rs @@ -296,7 +296,6 @@ impl Timeline { stats.evicted += 1; } Some(Err(EvictionError::NotFound | EvictionError::Downloaded)) => { - // compaction/gc removed the file while we were waiting on layer_removal_cs stats.not_evictable += 1; } } @@ -351,7 +350,7 @@ impl Timeline { match state.last_layer_access_imitation { Some(ts) if ts.elapsed() < inter_imitate_period => { /* no need to run */ } _ => { - self.imitate_synthetic_size_calculation_worker(&tenant, ctx, cancel) + self.imitate_synthetic_size_calculation_worker(&tenant, cancel, ctx) .await; state.last_layer_access_imitation = Some(tokio::time::Instant::now()); } @@ -417,8 +416,8 @@ impl Timeline { async fn imitate_synthetic_size_calculation_worker( &self, tenant: &Arc, - ctx: &RequestContext, cancel: &CancellationToken, + ctx: &RequestContext, ) { if self.conf.metric_collection_endpoint.is_none() { // We don't start the consumption metrics task if this is not set in the config. @@ -457,6 +456,7 @@ impl Timeline { None, &mut throwaway_cache, LogicalSizeCalculationCause::EvictionTaskImitation, + cancel, ctx, ) .instrument(info_span!("gather_inputs")); diff --git a/pageserver/src/tenant/timeline/init.rs b/pageserver/src/tenant/timeline/init.rs index 96bf847fbb..916ebfc6d9 100644 --- a/pageserver/src/tenant/timeline/init.rs +++ b/pageserver/src/tenant/timeline/init.rs @@ -13,6 +13,7 @@ use crate::{ }; use anyhow::Context; use camino::Utf8Path; +use pageserver_api::shard::ShardIndex; use std::{collections::HashMap, str::FromStr}; use utils::lsn::Lsn; @@ -107,6 +108,7 @@ pub(super) fn reconcile( index_part: Option<&IndexPart>, disk_consistent_lsn: Lsn, generation: Generation, + shard: ShardIndex, ) -> Vec<(LayerFileName, Result)> { use Decision::*; @@ -118,10 +120,13 @@ pub(super) fn reconcile( .map(|(name, file_size)| { ( name, - // The generation here will be corrected to match IndexPart in the merge below, unless + // The generation and shard here will be corrected to match IndexPart in the merge below, unless // it is not in IndexPart, in which case using our current generation makes sense // because it will be uploaded in this generation. - (Some(LayerFileMetadata::new(file_size, generation)), None), + ( + Some(LayerFileMetadata::new(file_size, generation, shard)), + None, + ), ) }) .collect::(); diff --git a/pageserver/src/tenant/timeline/layer_manager.rs b/pageserver/src/tenant/timeline/layer_manager.rs index e4991e0865..7e1aa279d3 100644 --- a/pageserver/src/tenant/timeline/layer_manager.rs +++ b/pageserver/src/tenant/timeline/layer_manager.rs @@ -190,7 +190,6 @@ impl LayerManager { /// Called when compaction is completed. pub(crate) fn finish_compact_l0( &mut self, - layer_removal_cs: &Arc>, compact_from: &[Layer], compact_to: &[ResidentLayer], metrics: &TimelineMetrics, @@ -201,25 +200,16 @@ impl LayerManager { metrics.record_new_file_metrics(l.layer_desc().file_size); } for l in compact_from { - Self::delete_historic_layer(layer_removal_cs, l, &mut updates, &mut self.layer_fmgr); + Self::delete_historic_layer(l, &mut updates, &mut self.layer_fmgr); } updates.flush(); } - /// Called when garbage collect the timeline. Returns a guard that will apply the updates to the layer map. - pub(crate) fn finish_gc_timeline( - &mut self, - layer_removal_cs: &Arc>, - gc_layers: Vec, - ) { + /// Called when garbage collect has selected the layers to be removed. + pub(crate) fn finish_gc_timeline(&mut self, gc_layers: &[Layer]) { let mut updates = self.layer_map.batch_update(); for doomed_layer in gc_layers { - Self::delete_historic_layer( - layer_removal_cs, - &doomed_layer, - &mut updates, - &mut self.layer_fmgr, - ); + Self::delete_historic_layer(doomed_layer, &mut updates, &mut self.layer_fmgr); } updates.flush() } @@ -238,7 +228,6 @@ impl LayerManager { /// Remote storage is not affected by this operation. fn delete_historic_layer( // we cannot remove layers otherwise, since gc and compaction will race - _layer_removal_cs: &Arc>, layer: &Layer, updates: &mut BatchedUpdates<'_>, mapping: &mut LayerFileManager, diff --git a/pageserver/src/tenant/timeline/uninit.rs b/pageserver/src/tenant/timeline/uninit.rs index 6b68fdeb84..f9bb6ca419 100644 --- a/pageserver/src/tenant/timeline/uninit.rs +++ b/pageserver/src/tenant/timeline/uninit.rs @@ -45,12 +45,20 @@ impl<'t> UninitializedTimeline<'t> { let timeline_id = self.timeline_id; let tenant_id = self.owning_tenant.tenant_id; - let (new_timeline, uninit_mark) = self.raw_timeline.take().with_context(|| { - format!("No timeline for initalization found for {tenant_id}/{timeline_id}") - })?; + if self.raw_timeline.is_none() { + return Err(anyhow::anyhow!( + "No timeline for initialization found for {tenant_id}/{timeline_id}" + )); + } // Check that the caller initialized disk_consistent_lsn - let new_disk_consistent_lsn = new_timeline.get_disk_consistent_lsn(); + let new_disk_consistent_lsn = self + .raw_timeline + .as_ref() + .expect("checked above") + .0 + .get_disk_consistent_lsn(); + anyhow::ensure!( new_disk_consistent_lsn.is_valid(), "new timeline {tenant_id}/{timeline_id} has invalid disk_consistent_lsn" @@ -62,6 +70,13 @@ impl<'t> UninitializedTimeline<'t> { "Found freshly initialized timeline {tenant_id}/{timeline_id} in the tenant map" ), Entry::Vacant(v) => { + // after taking here should be no fallible operations, because the drop guard will not + // cleanup after and would block for example the tenant deletion + let (new_timeline, uninit_mark) = + self.raw_timeline.take().expect("already checked"); + + // this is the mutual exclusion between different retries to create the timeline; + // this should be an assertion. uninit_mark.remove_uninit_mark().with_context(|| { format!( "Failed to remove uninit mark file for timeline {tenant_id}/{timeline_id}" @@ -70,10 +85,10 @@ impl<'t> UninitializedTimeline<'t> { v.insert(Arc::clone(&new_timeline)); new_timeline.maybe_spawn_flush_loop(); + + Ok(new_timeline) } } - - Ok(new_timeline) } /// Prepares timeline data by loading it from the basebackup archive. diff --git a/pageserver/src/tenant/upload_queue.rs b/pageserver/src/tenant/upload_queue.rs index b47878aacc..32f14f40c5 100644 --- a/pageserver/src/tenant/upload_queue.rs +++ b/pageserver/src/tenant/upload_queue.rs @@ -1,6 +1,5 @@ use super::storage_layer::LayerFileName; use super::storage_layer::ResidentLayer; -use super::Generation; use crate::tenant::metadata::TimelineMetadata; use crate::tenant::remote_timeline_client::index::IndexPart; use crate::tenant::remote_timeline_client::index::LayerFileMetadata; @@ -15,6 +14,9 @@ use utils::lsn::AtomicLsn; use std::sync::atomic::AtomicU32; use utils::lsn::Lsn; +#[cfg(feature = "testing")] +use utils::generation::Generation; + // clippy warns that Uninitialized is much smaller than Initialized, which wastes // memory for Uninitialized variants. Doesn't matter in practice, there are not // that many upload queues in a running pageserver, and most of them are initialized @@ -88,6 +90,14 @@ pub(crate) struct UploadQueueInitialized { /// bug causing leaks, then it's better to not leave this enabled for production builds. #[cfg(feature = "testing")] pub(crate) dangling_files: HashMap, + + /// Set to true when we have inserted the `UploadOp::Shutdown` into the `inprogress_tasks`. + pub(crate) shutting_down: bool, + + /// Permitless semaphore on which any number of `RemoteTimelineClient::shutdown` futures can + /// wait on until one of them stops the queue. The semaphore is closed when + /// `RemoteTimelineClient::launch_queued_tasks` encounters `UploadOp::Shutdown`. + pub(crate) shutdown_ready: Arc, } impl UploadQueueInitialized { @@ -146,6 +156,8 @@ impl UploadQueue { queued_operations: VecDeque::new(), #[cfg(feature = "testing")] dangling_files: HashMap::new(), + shutting_down: false, + shutdown_ready: Arc::new(tokio::sync::Semaphore::new(0)), }; *self = UploadQueue::Initialized(state); @@ -193,6 +205,8 @@ impl UploadQueue { queued_operations: VecDeque::new(), #[cfg(feature = "testing")] dangling_files: HashMap::new(), + shutting_down: false, + shutdown_ready: Arc::new(tokio::sync::Semaphore::new(0)), }; *self = UploadQueue::Initialized(state); @@ -204,7 +218,13 @@ impl UploadQueue { UploadQueue::Uninitialized | UploadQueue::Stopped(_) => { anyhow::bail!("queue is in state {}", self.as_str()) } - UploadQueue::Initialized(x) => Ok(x), + UploadQueue::Initialized(x) => { + if !x.shutting_down { + Ok(x) + } else { + anyhow::bail!("queue is shutting down") + } + } } } @@ -232,7 +252,7 @@ pub(crate) struct UploadTask { /// for timeline deletion, which skips this queue and goes directly to DeletionQueue. #[derive(Debug)] pub(crate) struct Delete { - pub(crate) layers: Vec<(LayerFileName, Generation)>, + pub(crate) layers: Vec<(LayerFileName, LayerFileMetadata)>, } #[derive(Debug)] @@ -248,6 +268,10 @@ pub(crate) enum UploadOp { /// Barrier. When the barrier operation is reached, Barrier(tokio::sync::watch::Sender<()>), + + /// Shutdown; upon encountering this operation no new operations will be spawned, otherwise + /// this is the same as a Barrier. + Shutdown, } impl std::fmt::Display for UploadOp { @@ -269,6 +293,7 @@ impl std::fmt::Display for UploadOp { write!(f, "Delete({} layers)", delete.layers.len()) } UploadOp::Barrier(_) => write!(f, "Barrier"), + UploadOp::Shutdown => write!(f, "Shutdown"), } } } diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 1f802af66d..49a7eb2a0e 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -2091,10 +2091,28 @@ mod tests { use postgres_ffi::waldecoder::WalStreamDecoder; use postgres_ffi::WAL_SEGMENT_SIZE; + // Define test data path and constants. + // + // Steps to reconstruct the data, if needed: + // 1. Run the pgbench python test + // 2. Take the first wal segment file from safekeeper + // 3. Compress it using `zstd --long input_file` + // 4. Copy initdb.tar.zst from local_fs_remote_storage + // 5. Grep sk logs for "restart decoder" to get startpoint + // 6. Run just the decoder from this test to get the endpoint. + // It's the last LSN the decoder will output. + let pg_version = 15; // The test data was generated by pg15 + let path = "test_data/sk_wal_segment_from_pgbench"; + let wal_segment_path = format!("{path}/000000010000000000000001.zst"); + let startpoint = Lsn::from_hex("14AEC08").unwrap(); + let endpoint = Lsn::from_hex("1FFFF98").unwrap(); + // Bootstrap a real timeline. We can't use create_test_timeline because // it doesn't create a real checkpoint, and Walingest::new tries to parse // the garbage data. - let pg_version = 15; + // + // TODO use the initdb.tar.zst file stored with the test data to avoid + // problems with inconsistent initdb results after pg minor version bumps. let (tenant, ctx) = TenantHarness::create("test_ingest_real_wal") .unwrap() .load() @@ -2104,26 +2122,16 @@ mod tests { .await .unwrap(); - // Get test data. Steps to reconstruct it, if needed: - // 1. Run the pgbench python test - // 2. Take the first wal segment file from safekeeper - // 3. Compress it using `gzip -c input_file > output_file.gz` - // 4. Grep sk logs for "restart decoder" to get startpoint - // 5. Run just the decoder from this test to get the endpoint. - // It's the last LSN the decoder will output. - let path = "test_data/sk_wal_segment_from_pgbench.gz"; - let startpoint = Lsn::from_hex("14AEC08").unwrap(); - let endpoint = Lsn::from_hex("1FFFF98").unwrap(); - // We fully read and decompress this into memory before decoding // to get a more accurate perf profile of the decoder. let bytes = { - use std::io::Read; - let file_reader = std::fs::File::open(path).unwrap(); - let buffered_reader = std::io::BufReader::new(file_reader); - let mut decoder = flate2::read::GzDecoder::new(buffered_reader); - let mut buffer: Vec = vec![]; - decoder.read_to_end(&mut buffer).unwrap(); + use async_compression::tokio::bufread::ZstdDecoder; + let file = tokio::fs::File::open(wal_segment_path).await.unwrap(); + let reader = tokio::io::BufReader::new(file); + let decoder = ZstdDecoder::new(reader); + let mut reader = tokio::io::BufReader::new(decoder); + let mut buffer = Vec::new(); + tokio::io::copy_buf(&mut reader, &mut buffer).await.unwrap(); buffer }; diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 5d8cc0e181..4e684dec2d 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -1182,7 +1182,7 @@ mod tests { #[tokio::test] async fn short_v14_redo() { - let expected = std::fs::read("fixtures/short_v14_redo.page").unwrap(); + let expected = std::fs::read("test_data/short_v14_redo.page").unwrap(); let h = RedoHarness::new().unwrap(); diff --git a/pageserver/fixtures/short_v14_redo.page b/pageserver/test_data/short_v14_redo.page similarity index 100% rename from pageserver/fixtures/short_v14_redo.page rename to pageserver/test_data/short_v14_redo.page diff --git a/pageserver/test_data/sk_wal_segment_from_pgbench/000000010000000000000001.zst b/pageserver/test_data/sk_wal_segment_from_pgbench/000000010000000000000001.zst new file mode 100644 index 0000000000..3c478e7827 Binary files /dev/null and b/pageserver/test_data/sk_wal_segment_from_pgbench/000000010000000000000001.zst differ diff --git a/pageserver/test_data/sk_wal_segment_from_pgbench/initdb.tar.zst b/pageserver/test_data/sk_wal_segment_from_pgbench/initdb.tar.zst new file mode 100644 index 0000000000..17e9c7ea08 Binary files /dev/null and b/pageserver/test_data/sk_wal_segment_from_pgbench/initdb.tar.zst differ diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index 84835843bc..7fc99523db 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -20,7 +20,7 @@ SHLIB_LINK_INTERNAL = $(libpq) SHLIB_LINK = -lcurl EXTENSION = neon -DATA = neon--1.0.sql +DATA = neon--1.0.sql neon--1.0--1.1.sql PGFILEDESC = "neon - cloud storage for PostgreSQL" EXTRA_CLEAN = \ diff --git a/pgxn/neon/README.md b/pgxn/neon/README.md new file mode 100644 index 0000000000..39311aba82 --- /dev/null +++ b/pgxn/neon/README.md @@ -0,0 +1,20 @@ +neon extension consists of several parts: + +### shared preload library `neon.so` + +- implements storage manager API and network communications with remote page server. + +- walproposer: implements broadcast protocol between postgres and WAL safekeepers. + +- control plane connector: Captures updates to roles/databases using ProcessUtility_hook and sends them to the control ProcessUtility_hook. + +- remote extension server: Request compute_ctl to download extension files. + +- file_cache: Local file cache is used to temporary store relations pages in local file system for better performance. + +- relsize_cache: Relation size cache for better neon performance. + +### SQL functions in `neon--*.sql` + +Utility functions to expose neon specific information to user and metrics collection. +This extension is created in all databases in the cluster by default. diff --git a/pgxn/neon/file_cache.c b/pgxn/neon/file_cache.c index 22f20a4c0b..e70f0163c0 100644 --- a/pgxn/neon/file_cache.c +++ b/pgxn/neon/file_cache.c @@ -32,11 +32,13 @@ #include "storage/latch.h" #include "storage/ipc.h" #include "storage/lwlock.h" +#include "utils/builtins.h" #include "utils/dynahash.h" #include "utils/guc.h" #include "storage/fd.h" #include "storage/pg_shmem.h" #include "storage/buf_internals.h" +#include "pgstat.h" /* * Local file cache is used to temporary store relations pages in local file system. @@ -65,6 +67,7 @@ typedef struct FileCacheEntry { BufferTag key; + uint32 hash; uint32 offset; uint32 access_count; uint32 bitmap[BLOCKS_PER_CHUNK/32]; @@ -76,6 +79,10 @@ typedef struct FileCacheControl uint64 generation; /* generation is needed to handle correct hash reenabling */ uint32 size; /* size of cache file in chunks */ uint32 used; /* number of used chunks */ + uint32 limit; /* shared copy of lfc_size_limit */ + uint64 hits; + uint64 misses; + uint64 writes; dlist_head lru; /* double linked list for LRU replacement algorithm */ } FileCacheControl; @@ -91,10 +98,12 @@ static shmem_startup_hook_type prev_shmem_startup_hook; static shmem_request_hook_type prev_shmem_request_hook; #endif -void FileCacheMonitorMain(Datum main_arg); +#define LFC_ENABLED() (lfc_ctl->limit != 0) + +void PGDLLEXPORT FileCacheMonitorMain(Datum main_arg); /* - * Local file cache is mandatory and Neon can work without it. + * Local file cache is optional and Neon can work without it. * In case of any any errors with this cache, we should disable it but to not throw error. * Also we should allow re-enable it if source of failure (lack of disk space, permissions,...) is fixed. * All cache content should be invalidated to avoid reading of stale or corrupted data @@ -102,49 +111,77 @@ void FileCacheMonitorMain(Datum main_arg); static void lfc_disable(char const* op) { - HASH_SEQ_STATUS status; - FileCacheEntry* entry; - + int fd; elog(WARNING, "Failed to %s local file cache at %s: %m, disabling local file cache", op, lfc_path); + /* Invalidate hash */ + LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + + if (LFC_ENABLED()) + { + HASH_SEQ_STATUS status; + FileCacheEntry* entry; + + hash_seq_init(&status, lfc_hash); + while ((entry = hash_seq_search(&status)) != NULL) + { + hash_search_with_hash_value(lfc_hash, &entry->key, entry->hash, HASH_REMOVE, NULL); + } + lfc_ctl->generation += 1; + lfc_ctl->size = 0; + lfc_ctl->used = 0; + lfc_ctl->limit = 0; + dlist_init(&lfc_ctl->lru); + + if (lfc_desc > 0) + { + /* If the reason of error is ENOSPC, then truncation of file may help to reclaim some space */ + int rc = ftruncate(lfc_desc, 0); + if (rc < 0) + elog(WARNING, "Failed to truncate local file cache %s: %m", lfc_path); + } + } + /* We need to use unlink to to avoid races in LFC write, because it is not protectedby */ + unlink(lfc_path); + + fd = BasicOpenFile(lfc_path, O_RDWR|O_CREAT|O_TRUNC); + if (fd < 0) + elog(WARNING, "Failed to recreate local file cache %s: %m", lfc_path); + else + close(fd); + + LWLockRelease(lfc_lock); + if (lfc_desc > 0) close(lfc_desc); lfc_desc = -1; - lfc_size_limit = 0; +} - /* Invalidate hash */ - LWLockAcquire(lfc_lock, LW_EXCLUSIVE); - - hash_seq_init(&status, lfc_hash); - while ((entry = hash_seq_search(&status)) != NULL) - { - hash_search(lfc_hash, &entry->key, HASH_REMOVE, NULL); - memset(entry->bitmap, 0, sizeof entry->bitmap); - } - hash_seq_term(&status); - lfc_ctl->generation += 1; - lfc_ctl->size = 0; - lfc_ctl->used = 0; - dlist_init(&lfc_ctl->lru); - - LWLockRelease(lfc_lock); +/* + * This check is done without obtaining lfc_lock, so it is unreliable + */ +static bool +lfc_maybe_disabled(void) +{ + return !lfc_ctl || !LFC_ENABLED(); } static bool lfc_ensure_opened(void) { + bool enabled = !lfc_maybe_disabled(); /* Open cache file if not done yet */ - if (lfc_desc <= 0) + if (lfc_desc <= 0 && enabled) { - lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT); + lfc_desc = BasicOpenFile(lfc_path, O_RDWR); if (lfc_desc < 0) { lfc_disable("open"); return false; } } - return true; + return enabled; } static void @@ -163,6 +200,7 @@ lfc_shmem_startup(void) lfc_ctl = (FileCacheControl*)ShmemInitStruct("lfc", sizeof(FileCacheControl), &found); if (!found) { + int fd; uint32 lfc_size = SIZE_MB_TO_CHUNKS(lfc_max_size); lfc_lock = (LWLockId)GetNamedLWLockTranche("lfc_lock"); info.keysize = sizeof(BufferTag); @@ -175,10 +213,23 @@ lfc_shmem_startup(void) lfc_ctl->generation = 0; lfc_ctl->size = 0; lfc_ctl->used = 0; + lfc_ctl->hits = 0; + lfc_ctl->misses = 0; + lfc_ctl->writes = 0; dlist_init(&lfc_ctl->lru); - /* Remove file cache on restart */ - (void)unlink(lfc_path); + /* Recreate file cache on restart */ + fd = BasicOpenFile(lfc_path, O_RDWR|O_CREAT|O_TRUNC); + if (fd < 0) + { + elog(WARNING, "Failed to create local file cache %s: %m", lfc_path); + lfc_ctl->limit = 0; + } + else + { + close(fd); + lfc_ctl->limit = SIZE_MB_TO_CHUNKS(lfc_size_limit); + } } LWLockRelease(AddinShmemInitLock); } @@ -195,6 +246,17 @@ lfc_shmem_request(void) RequestNamedLWLockTranche("lfc_lock", 1); } +static bool +is_normal_backend(void) +{ + /* + * Stats collector detach shared memory, so we should not try to access shared memory here. + * Parallel workers first assign default value (0), so not perform truncation in parallel workers. + * The Postmaster can handle SIGHUP and it has access to shared memory (UsedShmemSegAddr != NULL), but has no PGPROC. + */ + return lfc_ctl && MyProc && UsedShmemSegAddr && !IsParallelWorker(); +} + static bool lfc_check_limit_hook(int *newval, void **extra, GucSource source) { @@ -210,25 +272,15 @@ static void lfc_change_limit_hook(int newval, void *extra) { uint32 new_size = SIZE_MB_TO_CHUNKS(newval); - /* - * Stats collector detach shared memory, so we should not try to access shared memory here. - * Parallel workers first assign default value (0), so not perform truncation in parallel workers. - * The Postmaster can handle SIGHUP and it has access to shared memory (UsedShmemSegAddr != NULL), but has no PGPROC. - */ - if (!lfc_ctl || !MyProc || !UsedShmemSegAddr || IsParallelWorker()) + + if (!is_normal_backend()) + return; + + if (!lfc_ensure_opened()) return; - /* Open cache file if not done yet */ - if (lfc_desc <= 0) - { - lfc_desc = BasicOpenFile(lfc_path, O_RDWR|O_CREAT); - if (lfc_desc < 0) { - elog(WARNING, "Failed to open file cache %s: %m, disabling file cache", lfc_path); - lfc_size_limit = 0; /* disable file cache */ - return; - } - } LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + while (new_size < lfc_ctl->used && !dlist_is_empty(&lfc_ctl->lru)) { /* Shrink cache by throwing away least recently accessed chunks and returning their space to file system */ @@ -238,10 +290,12 @@ lfc_change_limit_hook(int newval, void *extra) if (fallocate(lfc_desc, FALLOC_FL_PUNCH_HOLE|FALLOC_FL_KEEP_SIZE, (off_t)victim->offset*BLOCKS_PER_CHUNK*BLCKSZ, BLOCKS_PER_CHUNK*BLCKSZ) < 0) elog(LOG, "Failed to punch hole in file: %m"); #endif - hash_search(lfc_hash, &victim->key, HASH_REMOVE, NULL); + hash_search_with_hash_value(lfc_hash, &victim->key, victim->hash, HASH_REMOVE, NULL); lfc_ctl->used -= 1; } + lfc_ctl->limit = new_size; elog(DEBUG1, "set local file cache limit to %d", new_size); + LWLockRelease(lfc_lock); } @@ -255,6 +309,7 @@ lfc_init(void) if (!process_shared_preload_libraries_in_progress) elog(ERROR, "Neon module should be loaded via shared_preload_libraries"); + DefineCustomIntVariable("neon.max_file_cache_size", "Maximal size of Neon local file cache", NULL, @@ -315,10 +370,10 @@ lfc_cache_contains(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno) BufferTag tag; FileCacheEntry* entry; int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1); - bool found; + bool found = false; uint32 hash; - if (lfc_size_limit == 0) /* fast exit if file cache is disabled */ + if (lfc_maybe_disabled()) /* fast exit if file cache is disabled */ return false; CopyNRelFileInfoToBufTag(tag, rinfo); @@ -327,8 +382,11 @@ lfc_cache_contains(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno) hash = get_hash_value(lfc_hash, &tag); LWLockAcquire(lfc_lock, LW_SHARED); - entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); - found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) != 0; + if (LFC_ENABLED()) + { + entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); + found = entry != NULL && (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) != 0; + } LWLockRelease(lfc_lock); return found; } @@ -345,7 +403,7 @@ lfc_evict(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno) int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1); uint32 hash; - if (lfc_size_limit == 0) /* fast exit if file cache is disabled */ + if (lfc_maybe_disabled()) /* fast exit if file cache is disabled */ return; CopyNRelFileInfoToBufTag(tag, rinfo); @@ -355,6 +413,13 @@ lfc_evict(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno) hash = get_hash_value(lfc_hash, &tag); LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + + if (!LFC_ENABLED()) + { + LWLockRelease(lfc_lock); + return; + } + entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, &found); if (!found) @@ -405,7 +470,7 @@ lfc_evict(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno) /* * Try to read page from local cache. * Returns true if page is found in local cache. - * In case of error lfc_size_limit is set to zero to disable any further opera-tins with cache. + * In case of error local file cache is disabled (lfc->limit is set to zero). */ bool lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, @@ -420,7 +485,7 @@ lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, uint64 generation; uint32 entry_offset; - if (lfc_size_limit == 0) /* fast exit if file cache is disabled */ + if (lfc_maybe_disabled()) /* fast exit if file cache is disabled */ return false; if (!lfc_ensure_opened()) @@ -432,10 +497,18 @@ lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, hash = get_hash_value(lfc_hash, &tag); LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + + if (!LFC_ENABLED()) + { + LWLockRelease(lfc_lock); + return false; + } + entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL); if (entry == NULL || (entry->bitmap[chunk_offs >> 5] & (1 << (chunk_offs & 31))) == 0) { /* Page is not cached */ + lfc_ctl->misses += 1; LWLockRelease(lfc_lock); return false; } @@ -456,8 +529,11 @@ lfc_read(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, /* Place entry to the head of LRU list */ LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + if (lfc_ctl->generation == generation) { + Assert(LFC_ENABLED()); + lfc_ctl->hits += 1; Assert(entry->access_count > 0); if (--entry->access_count == 0) dlist_push_tail(&lfc_ctl->lru, &entry->lru_node); @@ -488,8 +564,10 @@ lfc_write(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, bool found; int chunk_offs = blkno & (BLOCKS_PER_CHUNK-1); uint32 hash; + uint64 generation; + uint32 entry_offset; - if (lfc_size_limit == 0) /* fast exit if file cache is disabled */ + if (lfc_maybe_disabled()) /* fast exit if file cache is disabled */ return; if (!lfc_ensure_opened()) @@ -497,12 +575,17 @@ lfc_write(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, tag.forkNum = forkNum; tag.blockNum = blkno & ~(BLOCKS_PER_CHUNK-1); - CopyNRelFileInfoToBufTag(tag, rinfo); - hash = get_hash_value(lfc_hash, &tag); LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + + if (!LFC_ENABLED()) + { + LWLockRelease(lfc_lock); + return; + } + entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_ENTER, &found); if (found) @@ -521,13 +604,13 @@ lfc_write(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, * there are should be very large number of concurrent IO operations and them are limited by max_connections, * we prefer not to complicate code and use second approach. */ - if (lfc_ctl->used >= SIZE_MB_TO_CHUNKS(lfc_size_limit) && !dlist_is_empty(&lfc_ctl->lru)) + if (lfc_ctl->used >= lfc_ctl->limit && !dlist_is_empty(&lfc_ctl->lru)) { /* Cache overflow: evict least recently used chunk */ FileCacheEntry* victim = dlist_container(FileCacheEntry, lru_node, dlist_pop_head_node(&lfc_ctl->lru)); Assert(victim->access_count == 0); entry->offset = victim->offset; /* grab victim's chunk */ - hash_search(lfc_hash, &victim->key, HASH_REMOVE, NULL); + hash_search_with_hash_value(lfc_hash, &victim->key, victim->hash, HASH_REMOVE, NULL); elog(DEBUG2, "Swap file cache page"); } else @@ -536,27 +619,140 @@ lfc_write(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno, entry->offset = lfc_ctl->size++; /* allocate new chunk at end of file */ } entry->access_count = 1; + entry->hash = hash; memset(entry->bitmap, 0, sizeof entry->bitmap); } - rc = pwrite(lfc_desc, buffer, BLCKSZ, ((off_t)entry->offset*BLOCKS_PER_CHUNK + chunk_offs)*BLCKSZ); + generation = lfc_ctl->generation; + entry_offset = entry->offset; + lfc_ctl->writes += 1; + LWLockRelease(lfc_lock); + + rc = pwrite(lfc_desc, buffer, BLCKSZ, ((off_t)entry_offset*BLOCKS_PER_CHUNK + chunk_offs)*BLCKSZ); if (rc != BLCKSZ) { - LWLockRelease(lfc_lock); lfc_disable("write"); } else { - /* Place entry to the head of LRU list */ - Assert(entry->access_count > 0); - if (--entry->access_count == 0) - dlist_push_tail(&lfc_ctl->lru, &entry->lru_node); + LWLockAcquire(lfc_lock, LW_EXCLUSIVE); + + if (lfc_ctl->generation == generation) + { + Assert(LFC_ENABLED()); + /* Place entry to the head of LRU list */ + Assert(entry->access_count > 0); + if (--entry->access_count == 0) + dlist_push_tail(&lfc_ctl->lru, &entry->lru_node); + + entry->bitmap[chunk_offs >> 5] |= (1 << (chunk_offs & 31)); + } - entry->bitmap[chunk_offs >> 5] |= (1 << (chunk_offs & 31)); LWLockRelease(lfc_lock); } } +typedef struct +{ + TupleDesc tupdesc; +} NeonGetStatsCtx; + +#define NUM_NEON_GET_STATS_COLS 2 +#define NUM_NEON_GET_STATS_ROWS 3 + +PG_FUNCTION_INFO_V1(neon_get_lfc_stats); +Datum +neon_get_lfc_stats(PG_FUNCTION_ARGS) +{ + FuncCallContext *funcctx; + NeonGetStatsCtx* fctx; + MemoryContext oldcontext; + TupleDesc tupledesc; + Datum result; + HeapTuple tuple; + char const* key; + uint64 value; + Datum values[NUM_NEON_GET_STATS_COLS]; + bool nulls[NUM_NEON_GET_STATS_COLS]; + + if (SRF_IS_FIRSTCALL()) + { + funcctx = SRF_FIRSTCALL_INIT(); + + /* Switch context when allocating stuff to be used in later calls */ + oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); + + /* Create a user function context for cross-call persistence */ + fctx = (NeonGetStatsCtx*) palloc(sizeof(NeonGetStatsCtx)); + + /* Construct a tuple descriptor for the result rows. */ + tupledesc = CreateTemplateTupleDesc(NUM_NEON_GET_STATS_COLS); + + TupleDescInitEntry(tupledesc, (AttrNumber) 1, "lfc_key", + TEXTOID, -1, 0); + TupleDescInitEntry(tupledesc, (AttrNumber) 2, "lfc_value", + INT8OID, -1, 0); + + fctx->tupdesc = BlessTupleDesc(tupledesc); + funcctx->max_calls = NUM_NEON_GET_STATS_ROWS; + funcctx->user_fctx = fctx; + + /* Return to original context when allocating transient memory */ + MemoryContextSwitchTo(oldcontext); + } + + funcctx = SRF_PERCALL_SETUP(); + + /* Get the saved state */ + fctx = (NeonGetStatsCtx*) funcctx->user_fctx; + + switch (funcctx->call_cntr) + { + case 0: + key = "file_cache_misses"; + if (lfc_ctl) + value = lfc_ctl->misses; + break; + case 1: + key = "file_cache_hits"; + if (lfc_ctl) + value = lfc_ctl->hits; + break; + case 2: + key = "file_cache_used"; + if (lfc_ctl) + value = lfc_ctl->used; + break; + case 3: + key = "file_cache_writes"; + if (lfc_ctl) + value = lfc_ctl->writes; + break; + default: + SRF_RETURN_DONE(funcctx); + } + values[0] = PointerGetDatum(cstring_to_text(key)); + nulls[0] = false; + if (lfc_ctl) + { + nulls[1] = false; + values[1] = Int64GetDatum(value); + } + else + nulls[1] = true; + + tuple = heap_form_tuple(fctx->tupdesc, values, nulls); + result = HeapTupleGetDatum(tuple); + SRF_RETURN_NEXT(funcctx, result); +} + + +/* + * Function returning data from the local file cache + * relation node/tablespace/database/blocknum and access_counter + */ +PG_FUNCTION_INFO_V1(local_cache_pages); + /* * Record structure holding the to be exposed cache data. */ @@ -580,11 +776,6 @@ typedef struct LocalCachePagesRec *record; } LocalCachePagesContext; -/* - * Function returning data from the local file cache - * relation node/tablespace/database/blocknum and access_counter - */ -PG_FUNCTION_INFO_V1(local_cache_pages); #define NUM_LOCALCACHE_PAGES_ELEM 7 @@ -651,15 +842,20 @@ local_cache_pages(PG_FUNCTION_ARGS) fctx->tupdesc = BlessTupleDesc(tupledesc); - LWLockAcquire(lfc_lock, LW_SHARED); - - hash_seq_init(&status, lfc_hash); - while ((entry = hash_seq_search(&status)) != NULL) + if (lfc_ctl) { - for (int i = 0; i < BLOCKS_PER_CHUNK; i++) - n_pages += (entry->bitmap[i >> 5] & (1 << (i & 31))) != 0; + LWLockAcquire(lfc_lock, LW_SHARED); + + if (LFC_ENABLED()) + { + hash_seq_init(&status, lfc_hash); + while ((entry = hash_seq_search(&status)) != NULL) + { + for (int i = 0; i < BLOCKS_PER_CHUNK/32; i++) + n_pages += pg_popcount32(entry->bitmap[i]); + } + } } - hash_seq_term(&status); fctx->record = (LocalCachePagesRec *) MemoryContextAllocHuge(CurrentMemoryContext, sizeof(LocalCachePagesRec) * n_pages); @@ -671,36 +867,35 @@ local_cache_pages(PG_FUNCTION_ARGS) /* Return to original context when allocating transient memory */ MemoryContextSwitchTo(oldcontext); - /* - * Scan through all the buffers, saving the relevant fields in the - * fctx->record structure. - * - * We don't hold the partition locks, so we don't get a consistent - * snapshot across all buffers, but we do grab the buffer header - * locks, so the information of each buffer is self-consistent. - */ - n_pages = 0; - hash_seq_init(&status, lfc_hash); - while ((entry = hash_seq_search(&status)) != NULL) + if (n_pages != 0) { - for (int i = 0; i < BLOCKS_PER_CHUNK; i++) + /* + * Scan through all the cache entries, saving the relevant fields in the + * fctx->record structure. + */ + uint32 n = 0; + hash_seq_init(&status, lfc_hash); + while ((entry = hash_seq_search(&status)) != NULL) { - if (entry->bitmap[i >> 5] & (1 << (i & 31))) + for (int i = 0; i < BLOCKS_PER_CHUNK; i++) { - fctx->record[n_pages].pageoffs = entry->offset*BLOCKS_PER_CHUNK + i; - fctx->record[n_pages].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n_pages].reltablespace = NInfoGetSpcOid(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n_pages].reldatabase = NInfoGetDbOid(BufTagGetNRelFileInfo(entry->key)); - fctx->record[n_pages].forknum = entry->key.forkNum; - fctx->record[n_pages].blocknum = entry->key.blockNum + i; - fctx->record[n_pages].accesscount = entry->access_count; - n_pages += 1; + if (entry->bitmap[i >> 5] & (1 << (i & 31))) + { + fctx->record[n].pageoffs = entry->offset*BLOCKS_PER_CHUNK + i; + fctx->record[n].relfilenode = NInfoGetRelNumber(BufTagGetNRelFileInfo(entry->key)); + fctx->record[n].reltablespace = NInfoGetSpcOid(BufTagGetNRelFileInfo(entry->key)); + fctx->record[n].reldatabase = NInfoGetDbOid(BufTagGetNRelFileInfo(entry->key)); + fctx->record[n].forknum = entry->key.forkNum; + fctx->record[n].blocknum = entry->key.blockNum + i; + fctx->record[n].accesscount = entry->access_count; + n += 1; + } } } + Assert(n_pages == n); } - hash_seq_term(&status); - Assert(n_pages == funcctx->max_calls); - LWLockRelease(lfc_lock); + if (lfc_ctl) + LWLockRelease(lfc_lock); } funcctx = SRF_PERCALL_SETUP(); diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index cc09fb849d..8eb9ebb915 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -21,6 +21,7 @@ #include "storage/buf_internals.h" #include "storage/lwlock.h" #include "storage/ipc.h" +#include "storage/pg_shmem.h" #include "c.h" #include "postmaster/interrupt.h" @@ -87,6 +88,12 @@ bool (*old_redo_read_buffer_filter) (XLogReaderState *record, uint8 block_id) = static bool pageserver_flush(void); static void pageserver_disconnect(void); +static bool +PagestoreShmemIsValid() +{ + return pagestore_shared && UsedShmemSegAddr; +} + static bool CheckPageserverConnstring(char **newval, void **extra, GucSource source) { @@ -96,7 +103,7 @@ CheckPageserverConnstring(char **newval, void **extra, GucSource source) static void AssignPageserverConnstring(const char *newval, void *extra) { - if(!pagestore_shared) + if(!PagestoreShmemIsValid()) return; LWLockAcquire(pagestore_shared->lock, LW_EXCLUSIVE); strlcpy(pagestore_shared->pageserver_connstring, newval, MAX_PAGESERVER_CONNSTRING_SIZE); @@ -107,7 +114,7 @@ AssignPageserverConnstring(const char *newval, void *extra) static bool CheckConnstringUpdated() { - if(!pagestore_shared) + if(!PagestoreShmemIsValid()) return false; return pagestore_local_counter < pg_atomic_read_u64(&pagestore_shared->update_counter); } @@ -115,7 +122,7 @@ CheckConnstringUpdated() static void ReloadConnstring() { - if(!pagestore_shared) + if(!PagestoreShmemIsValid()) return; LWLockAcquire(pagestore_shared->lock, LW_SHARED); strlcpy(local_pageserver_connstring, pagestore_shared->pageserver_connstring, sizeof(local_pageserver_connstring)); diff --git a/pgxn/neon/neon--1.0--1.1.sql b/pgxn/neon/neon--1.0--1.1.sql new file mode 100644 index 0000000000..d1760f7299 --- /dev/null +++ b/pgxn/neon/neon--1.0--1.1.sql @@ -0,0 +1,10 @@ +\echo Use "ALTER EXTENSION neon UPDATE TO '1.1'" to load this file. \quit + +CREATE FUNCTION neon_get_lfc_stats() +RETURNS SETOF RECORD +AS 'MODULE_PATHNAME', 'neon_get_lfc_stats' +LANGUAGE C PARALLEL SAFE; + +-- Create a view for convenient access. +CREATE VIEW neon_lfc_stats AS + SELECT P.* FROM neon_get_lfc_stats() AS P (lfc_key text, lfc_value bigint); diff --git a/pgxn/neon/neon.control b/pgxn/neon/neon.control index 84f79881c1..4e4cb9f372 100644 --- a/pgxn/neon/neon.control +++ b/pgxn/neon/neon.control @@ -1,4 +1,5 @@ # neon extension comment = 'cloud storage for PostgreSQL' -default_version = '1.0' +default_version = '1.1' module_pathname = '$libdir/neon' +relocatable = true diff --git a/poetry.lock b/poetry.lock index 58ab4e70f2..f8838b92df 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2,111 +2,99 @@ [[package]] name = "aiohttp" -version = "3.8.6" +version = "3.9.0" description = "Async http client/server framework (asyncio)" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" files = [ - {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:41d55fc043954cddbbd82503d9cc3f4814a40bcef30b3569bc7b5e34130718c1"}, - {file = "aiohttp-3.8.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1d84166673694841d8953f0a8d0c90e1087739d24632fe86b1a08819168b4566"}, - {file = "aiohttp-3.8.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:253bf92b744b3170eb4c4ca2fa58f9c4b87aeb1df42f71d4e78815e6e8b73c9e"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3fd194939b1f764d6bb05490987bfe104287bbf51b8d862261ccf66f48fb4096"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c5f938d199a6fdbdc10bbb9447496561c3a9a565b43be564648d81e1102ac22"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2817b2f66ca82ee699acd90e05c95e79bbf1dc986abb62b61ec8aaf851e81c93"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fa375b3d34e71ccccf172cab401cd94a72de7a8cc01847a7b3386204093bb47"}, - {file = "aiohttp-3.8.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9de50a199b7710fa2904be5a4a9b51af587ab24c8e540a7243ab737b45844543"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:e1d8cb0b56b3587c5c01de3bf2f600f186da7e7b5f7353d1bf26a8ddca57f965"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:8e31e9db1bee8b4f407b77fd2507337a0a80665ad7b6c749d08df595d88f1cf5"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:7bc88fc494b1f0311d67f29fee6fd636606f4697e8cc793a2d912ac5b19aa38d"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:ec00c3305788e04bf6d29d42e504560e159ccaf0be30c09203b468a6c1ccd3b2"}, - {file = "aiohttp-3.8.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:ad1407db8f2f49329729564f71685557157bfa42b48f4b93e53721a16eb813ed"}, - {file = "aiohttp-3.8.6-cp310-cp310-win32.whl", hash = "sha256:ccc360e87341ad47c777f5723f68adbb52b37ab450c8bc3ca9ca1f3e849e5fe2"}, - {file = "aiohttp-3.8.6-cp310-cp310-win_amd64.whl", hash = "sha256:93c15c8e48e5e7b89d5cb4613479d144fda8344e2d886cf694fd36db4cc86865"}, - {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:6e2f9cc8e5328f829f6e1fb74a0a3a939b14e67e80832975e01929e320386b34"}, - {file = "aiohttp-3.8.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:e6a00ffcc173e765e200ceefb06399ba09c06db97f401f920513a10c803604ca"}, - {file = "aiohttp-3.8.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:41bdc2ba359032e36c0e9de5a3bd00d6fb7ea558a6ce6b70acedf0da86458321"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:14cd52ccf40006c7a6cd34a0f8663734e5363fd981807173faf3a017e202fec9"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2d5b785c792802e7b275c420d84f3397668e9d49ab1cb52bd916b3b3ffcf09ad"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1bed815f3dc3d915c5c1e556c397c8667826fbc1b935d95b0ad680787896a358"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:96603a562b546632441926cd1293cfcb5b69f0b4159e6077f7c7dbdfb686af4d"}, - {file = "aiohttp-3.8.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d76e8b13161a202d14c9584590c4df4d068c9567c99506497bdd67eaedf36403"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e3f1e3f1a1751bb62b4a1b7f4e435afcdade6c17a4fd9b9d43607cebd242924a"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:76b36b3124f0223903609944a3c8bf28a599b2cc0ce0be60b45211c8e9be97f8"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:a2ece4af1f3c967a4390c284797ab595a9f1bc1130ef8b01828915a05a6ae684"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:16d330b3b9db87c3883e565340d292638a878236418b23cc8b9b11a054aaa887"}, - {file = "aiohttp-3.8.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:42c89579f82e49db436b69c938ab3e1559e5a4409eb8639eb4143989bc390f2f"}, - {file = "aiohttp-3.8.6-cp311-cp311-win32.whl", hash = "sha256:efd2fcf7e7b9d7ab16e6b7d54205beded0a9c8566cb30f09c1abe42b4e22bdcb"}, - {file = "aiohttp-3.8.6-cp311-cp311-win_amd64.whl", hash = "sha256:3b2ab182fc28e7a81f6c70bfbd829045d9480063f5ab06f6e601a3eddbbd49a0"}, - {file = "aiohttp-3.8.6-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:fdee8405931b0615220e5ddf8cd7edd8592c606a8e4ca2a00704883c396e4479"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d25036d161c4fe2225d1abff2bd52c34ed0b1099f02c208cd34d8c05729882f0"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5d791245a894be071d5ab04bbb4850534261a7d4fd363b094a7b9963e8cdbd31"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0cccd1de239afa866e4ce5c789b3032442f19c261c7d8a01183fd956b1935349"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f13f60d78224f0dace220d8ab4ef1dbc37115eeeab8c06804fec11bec2bbd07"}, - {file = "aiohttp-3.8.6-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a9b5a0606faca4f6cc0d338359d6fa137104c337f489cd135bb7fbdbccb1e39"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:13da35c9ceb847732bf5c6c5781dcf4780e14392e5d3b3c689f6d22f8e15ae31"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:4d4cbe4ffa9d05f46a28252efc5941e0462792930caa370a6efaf491f412bc66"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_ppc64le.whl", hash = "sha256:229852e147f44da0241954fc6cb910ba074e597f06789c867cb7fb0621e0ba7a"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_s390x.whl", hash = "sha256:713103a8bdde61d13490adf47171a1039fd880113981e55401a0f7b42c37d071"}, - {file = "aiohttp-3.8.6-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:45ad816b2c8e3b60b510f30dbd37fe74fd4a772248a52bb021f6fd65dff809b6"}, - {file = "aiohttp-3.8.6-cp36-cp36m-win32.whl", hash = "sha256:2b8d4e166e600dcfbff51919c7a3789ff6ca8b3ecce16e1d9c96d95dd569eb4c"}, - {file = "aiohttp-3.8.6-cp36-cp36m-win_amd64.whl", hash = "sha256:0912ed87fee967940aacc5306d3aa8ba3a459fcd12add0b407081fbefc931e53"}, - {file = "aiohttp-3.8.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:e2a988a0c673c2e12084f5e6ba3392d76c75ddb8ebc6c7e9ead68248101cd446"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebf3fd9f141700b510d4b190094db0ce37ac6361a6806c153c161dc6c041ccda"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3161ce82ab85acd267c8f4b14aa226047a6bee1e4e6adb74b798bd42c6ae1f80"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d95fc1bf33a9a81469aa760617b5971331cdd74370d1214f0b3109272c0e1e3c"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c43ecfef7deaf0617cee936836518e7424ee12cb709883f2c9a1adda63cc460"}, - {file = "aiohttp-3.8.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca80e1b90a05a4f476547f904992ae81eda5c2c85c66ee4195bb8f9c5fb47f28"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:90c72ebb7cb3a08a7f40061079817133f502a160561d0675b0a6adf231382c92"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:bb54c54510e47a8c7c8e63454a6acc817519337b2b78606c4e840871a3e15349"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:de6a1c9f6803b90e20869e6b99c2c18cef5cc691363954c93cb9adeb26d9f3ae"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:a3628b6c7b880b181a3ae0a0683698513874df63783fd89de99b7b7539e3e8a8"}, - {file = "aiohttp-3.8.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:fc37e9aef10a696a5a4474802930079ccfc14d9f9c10b4662169671ff034b7df"}, - {file = "aiohttp-3.8.6-cp37-cp37m-win32.whl", hash = "sha256:f8ef51e459eb2ad8e7a66c1d6440c808485840ad55ecc3cafefadea47d1b1ba2"}, - {file = "aiohttp-3.8.6-cp37-cp37m-win_amd64.whl", hash = "sha256:b2fe42e523be344124c6c8ef32a011444e869dc5f883c591ed87f84339de5976"}, - {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:9e2ee0ac5a1f5c7dd3197de309adfb99ac4617ff02b0603fd1e65b07dc772e4b"}, - {file = "aiohttp-3.8.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:01770d8c04bd8db568abb636c1fdd4f7140b284b8b3e0b4584f070180c1e5c62"}, - {file = "aiohttp-3.8.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c68330a59506254b556b99a91857428cab98b2f84061260a67865f7f52899f5"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89341b2c19fb5eac30c341133ae2cc3544d40d9b1892749cdd25892bbc6ac951"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:71783b0b6455ac8f34b5ec99d83e686892c50498d5d00b8e56d47f41b38fbe04"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f628dbf3c91e12f4d6c8b3f092069567d8eb17814aebba3d7d60c149391aee3a"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b04691bc6601ef47c88f0255043df6f570ada1a9ebef99c34bd0b72866c217ae"}, - {file = "aiohttp-3.8.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7ee912f7e78287516df155f69da575a0ba33b02dd7c1d6614dbc9463f43066e3"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:9c19b26acdd08dd239e0d3669a3dddafd600902e37881f13fbd8a53943079dbc"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:99c5ac4ad492b4a19fc132306cd57075c28446ec2ed970973bbf036bcda1bcc6"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:f0f03211fd14a6a0aed2997d4b1c013d49fb7b50eeb9ffdf5e51f23cfe2c77fa"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:8d399dade330c53b4106160f75f55407e9ae7505263ea86f2ccca6bfcbdb4921"}, - {file = "aiohttp-3.8.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:ec4fd86658c6a8964d75426517dc01cbf840bbf32d055ce64a9e63a40fd7b771"}, - {file = "aiohttp-3.8.6-cp38-cp38-win32.whl", hash = "sha256:33164093be11fcef3ce2571a0dccd9041c9a93fa3bde86569d7b03120d276c6f"}, - {file = "aiohttp-3.8.6-cp38-cp38-win_amd64.whl", hash = "sha256:bdf70bfe5a1414ba9afb9d49f0c912dc524cf60141102f3a11143ba3d291870f"}, - {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:d52d5dc7c6682b720280f9d9db41d36ebe4791622c842e258c9206232251ab2b"}, - {file = "aiohttp-3.8.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4ac39027011414dbd3d87f7edb31680e1f430834c8cef029f11c66dad0670aa5"}, - {file = "aiohttp-3.8.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3f5c7ce535a1d2429a634310e308fb7d718905487257060e5d4598e29dc17f0b"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b30e963f9e0d52c28f284d554a9469af073030030cef8693106d918b2ca92f54"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:918810ef188f84152af6b938254911055a72e0f935b5fbc4c1a4ed0b0584aed1"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:002f23e6ea8d3dd8d149e569fd580c999232b5fbc601c48d55398fbc2e582e8c"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4fcf3eabd3fd1a5e6092d1242295fa37d0354b2eb2077e6eb670accad78e40e1"}, - {file = "aiohttp-3.8.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:255ba9d6d5ff1a382bb9a578cd563605aa69bec845680e21c44afc2670607a95"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d67f8baed00870aa390ea2590798766256f31dc5ed3ecc737debb6e97e2ede78"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:86f20cee0f0a317c76573b627b954c412ea766d6ada1a9fcf1b805763ae7feeb"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:39a312d0e991690ccc1a61f1e9e42daa519dcc34ad03eb6f826d94c1190190dd"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:e827d48cf802de06d9c935088c2924e3c7e7533377d66b6f31ed175c1620e05e"}, - {file = "aiohttp-3.8.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:bd111d7fc5591ddf377a408ed9067045259ff2770f37e2d94e6478d0f3fc0c17"}, - {file = "aiohttp-3.8.6-cp39-cp39-win32.whl", hash = "sha256:caf486ac1e689dda3502567eb89ffe02876546599bbf915ec94b1fa424eeffd4"}, - {file = "aiohttp-3.8.6-cp39-cp39-win_amd64.whl", hash = "sha256:3f0e27e5b733803333bb2371249f41cf42bae8884863e8e8965ec69bebe53132"}, - {file = "aiohttp-3.8.6.tar.gz", hash = "sha256:b0cf2a4501bff9330a8a5248b4ce951851e415bdcce9dc158e76cfd55e15085c"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6896b8416be9ada4d22cd359d7cb98955576ce863eadad5596b7cdfbf3e17c6c"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1736d87dad8ef46a8ec9cddd349fa9f7bd3a064c47dd6469c0d6763d3d49a4fc"}, + {file = "aiohttp-3.9.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8c9e5f4d7208cda1a2bb600e29069eecf857e6980d0ccc922ccf9d1372c16f4b"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8488519aa05e636c5997719fe543c8daf19f538f4fa044f3ce94bee608817cff"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5ab16c254e2312efeb799bc3c06897f65a133b38b69682bf75d1f1ee1a9c43a9"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7a94bde005a8f926d0fa38b88092a03dea4b4875a61fbcd9ac6f4351df1b57cd"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4b777c9286b6c6a94f50ddb3a6e730deec327e9e2256cb08b5530db0f7d40fd8"}, + {file = "aiohttp-3.9.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:571760ad7736b34d05597a1fd38cbc7d47f7b65deb722cb8e86fd827404d1f6b"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:deac0a32aec29608eb25d730f4bc5a261a65b6c48ded1ed861d2a1852577c932"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:4ee1b4152bc3190cc40ddd6a14715e3004944263ea208229ab4c297712aa3075"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:3607375053df58ed6f23903aa10cf3112b1240e8c799d243bbad0f7be0666986"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:65b0a70a25456d329a5e1426702dde67be0fb7a4ead718005ba2ca582d023a94"}, + {file = "aiohttp-3.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5a2eb5311a37fe105aa35f62f75a078537e1a9e4e1d78c86ec9893a3c97d7a30"}, + {file = "aiohttp-3.9.0-cp310-cp310-win32.whl", hash = "sha256:2cbc14a13fb6b42d344e4f27746a4b03a2cb0c1c3c5b932b0d6ad8881aa390e3"}, + {file = "aiohttp-3.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ac9669990e2016d644ba8ae4758688534aabde8dbbc81f9af129c3f5f01ca9cd"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f8e05f5163528962ce1d1806fce763ab893b1c5b7ace0a3538cd81a90622f844"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4afa8f71dba3a5a2e1e1282a51cba7341ae76585345c43d8f0e624882b622218"}, + {file = "aiohttp-3.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f929f4c9b9a00f3e6cc0587abb95ab9c05681f8b14e0fe1daecfa83ea90f8318"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28185e36a78d247c55e9fbea2332d16aefa14c5276a582ce7a896231c6b1c208"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a486ddf57ab98b6d19ad36458b9f09e6022de0381674fe00228ca7b741aacb2f"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:70e851f596c00f40a2f00a46126c95c2e04e146015af05a9da3e4867cfc55911"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c5b7bf8fe4d39886adc34311a233a2e01bc10eb4e842220235ed1de57541a896"}, + {file = "aiohttp-3.9.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c67a51ea415192c2e53e4e048c78bab82d21955b4281d297f517707dc836bf3d"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:694df243f394629bcae2d8ed94c589a181e8ba8604159e6e45e7b22e58291113"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:3dd8119752dd30dd7bca7d4bc2a92a59be6a003e4e5c2cf7e248b89751b8f4b7"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:eb6dfd52063186ac97b4caa25764cdbcdb4b10d97f5c5f66b0fa95052e744eb7"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:d97c3e286d0ac9af6223bc132dc4bad6540b37c8d6c0a15fe1e70fb34f9ec411"}, + {file = "aiohttp-3.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:816f4db40555026e4cdda604a1088577c1fb957d02f3f1292e0221353403f192"}, + {file = "aiohttp-3.9.0-cp311-cp311-win32.whl", hash = "sha256:3abf0551874fecf95f93b58f25ef4fc9a250669a2257753f38f8f592db85ddea"}, + {file = "aiohttp-3.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:e18d92c3e9e22553a73e33784fcb0ed484c9874e9a3e96c16a8d6a1e74a0217b"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:99ae01fb13a618b9942376df77a1f50c20a281390dad3c56a6ec2942e266220d"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:05857848da443c8c12110d99285d499b4e84d59918a21132e45c3f0804876994"}, + {file = "aiohttp-3.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:317719d7f824eba55857fe0729363af58e27c066c731bc62cd97bc9c3d9c7ea4"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1e3b3c107ccb0e537f309f719994a55621acd2c8fdf6d5ce5152aed788fb940"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45820ddbb276113ead8d4907a7802adb77548087ff5465d5c554f9aa3928ae7d"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:05a183f1978802588711aed0dea31e697d760ce9055292db9dc1604daa9a8ded"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:51a4cd44788ea0b5e6bb8fa704597af3a30be75503a7ed1098bc5b8ffdf6c982"}, + {file = "aiohttp-3.9.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:673343fbc0c1ac44d0d2640addc56e97a052504beacd7ade0dc5e76d3a4c16e8"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7e8a3b79b6d186a9c99761fd4a5e8dd575a48d96021f220ac5b5fa856e5dd029"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:6777a390e41e78e7c45dab43a4a0196c55c3b8c30eebe017b152939372a83253"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:7ae5f99a32c53731c93ac3075abd3e1e5cfbe72fc3eaac4c27c9dd64ba3b19fe"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:f1e4f254e9c35d8965d377e065c4a8a55d396fe87c8e7e8429bcfdeeb229bfb3"}, + {file = "aiohttp-3.9.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:11ca808f9a6b63485059f5f6e164ef7ec826483c1212a44f268b3653c91237d8"}, + {file = "aiohttp-3.9.0-cp312-cp312-win32.whl", hash = "sha256:de3cc86f4ea8b4c34a6e43a7306c40c1275e52bfa9748d869c6b7d54aa6dad80"}, + {file = "aiohttp-3.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:ca4fddf84ac7d8a7d0866664936f93318ff01ee33e32381a115b19fb5a4d1202"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:f09960b5bb1017d16c0f9e9f7fc42160a5a49fa1e87a175fd4a2b1a1833ea0af"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:8303531e2c17b1a494ffaeba48f2da655fe932c4e9a2626c8718403c83e5dd2b"}, + {file = "aiohttp-3.9.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:4790e44f46a4aa07b64504089def5744d3b6780468c4ec3a1a36eb7f2cae9814"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1d7edf74a36de0e5ca50787e83a77cf352f5504eb0ffa3f07000a911ba353fb"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:94697c7293199c2a2551e3e3e18438b4cba293e79c6bc2319f5fd652fccb7456"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a1b66dbb8a7d5f50e9e2ea3804b01e766308331d0cac76eb30c563ac89c95985"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9623cfd9e85b76b83ef88519d98326d4731f8d71869867e47a0b979ffec61c73"}, + {file = "aiohttp-3.9.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f32c86dc967ab8c719fd229ce71917caad13cc1e8356ee997bf02c5b368799bf"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f50b4663c3e0262c3a361faf440761fbef60ccdde5fe8545689a4b3a3c149fb4"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:dcf71c55ec853826cd70eadb2b6ac62ec577416442ca1e0a97ad875a1b3a0305"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:42fe4fd9f0dfcc7be4248c162d8056f1d51a04c60e53366b0098d1267c4c9da8"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:76a86a9989ebf82ee61e06e2bab408aec4ea367dc6da35145c3352b60a112d11"}, + {file = "aiohttp-3.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f9e09a1c83521d770d170b3801eea19b89f41ccaa61d53026ed111cb6f088887"}, + {file = "aiohttp-3.9.0-cp38-cp38-win32.whl", hash = "sha256:a00ce44c21612d185c5275c5cba4bab8d7c1590f248638b667ed8a782fa8cd6f"}, + {file = "aiohttp-3.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:d5b9345ab92ebe6003ae11d8092ce822a0242146e6fa270889b9ba965457ca40"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:98d21092bf2637c5fa724a428a69e8f5955f2182bff61f8036827cf6ce1157bf"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:35a68cd63ca6aaef5707888f17a70c36efe62b099a4e853d33dc2e9872125be8"}, + {file = "aiohttp-3.9.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3d7f6235c7475658acfc1769d968e07ab585c79f6ca438ddfecaa9a08006aee2"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db04d1de548f7a62d1dd7e7cdf7c22893ee168e22701895067a28a8ed51b3735"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:536b01513d67d10baf6f71c72decdf492fb7433c5f2f133e9a9087379d4b6f31"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c8b0a6487e8109427ccf638580865b54e2e3db4a6e0e11c02639231b41fc0f"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7276fe0017664414fdc3618fca411630405f1aaf0cc3be69def650eb50441787"}, + {file = "aiohttp-3.9.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:23170247ef89ffa842a02bbfdc425028574d9e010611659abeb24d890bc53bb8"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b1a2ea8252cacc7fd51df5a56d7a2bb1986ed39be9397b51a08015727dfb69bd"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2d71abc15ff7047412ef26bf812dfc8d0d1020d664617f4913df2df469f26b76"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:2d820162c8c2bdbe97d328cd4f417c955ca370027dce593345e437b2e9ffdc4d"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:2779f5e7c70f7b421915fd47db332c81de365678180a9f3ab404088f87ba5ff9"}, + {file = "aiohttp-3.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:366bc870d7ac61726f32a489fbe3d1d8876e87506870be66b01aeb84389e967e"}, + {file = "aiohttp-3.9.0-cp39-cp39-win32.whl", hash = "sha256:1df43596b826022b14998f0460926ce261544fedefe0d2f653e1b20f49e96454"}, + {file = "aiohttp-3.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:9c196b30f1b1aa3363a69dd69079ae9bec96c2965c4707eaa6914ba099fb7d4f"}, + {file = "aiohttp-3.9.0.tar.gz", hash = "sha256:09f23292d29135025e19e8ff4f0a68df078fe4ee013bca0105b2e803989de92d"}, ] [package.dependencies] aiosignal = ">=1.1.2" -async-timeout = ">=4.0.0a3,<5.0" +async-timeout = {version = ">=4.0,<5.0", markers = "python_version < \"3.11\""} attrs = ">=17.3.0" -charset-normalizer = ">=2.0,<4.0" frozenlist = ">=1.1.1" multidict = ">=4.5,<7.0" yarl = ">=1.0,<2.0" [package.extras] -speedups = ["Brotli", "aiodns", "cchardet"] +speedups = ["Brotli", "aiodns", "brotlicffi"] [[package]] name = "aiopg" @@ -2719,4 +2707,4 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "0834e5cb69e5457741d4f476c3e49a4dc83598b5730685c8755da651b96ad3ec" +content-hash = "25ffa9ed98d890a3b85e6036792296a60bb705e8f9eaa1f07336501116a58756" diff --git a/proxy/Cargo.toml b/proxy/Cargo.toml index 0ec7efd316..39a9c3ddb0 100644 --- a/proxy/Cargo.toml +++ b/proxy/Cargo.toml @@ -76,3 +76,4 @@ tokio-util.workspace = true rcgen.workspace = true rstest.workspace = true tokio-postgres-rustls.workspace = true +postgres-protocol.workspace = true diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index 9cf45c0eec..f0197cc31b 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -6,6 +6,7 @@ pub use link::LinkAuthError; use tokio_postgres::config::AuthKeys; use crate::proxy::{handle_try_wake, retry_after, LatencyTimer}; +use crate::stream::Stream; use crate::{ auth::{self, ClientCredentials}, config::AuthenticationConfig, @@ -131,7 +132,7 @@ async fn auth_quirks_creds( api: &impl console::Api, extra: &ConsoleReqExtra<'_>, creds: &mut ClientCredentials<'_>, - client: &mut stream::PqStream, + client: &mut stream::PqStream>, allow_cleartext: bool, config: &'static AuthenticationConfig, latency_timer: &mut LatencyTimer, @@ -165,7 +166,7 @@ async fn auth_quirks( api: &impl console::Api, extra: &ConsoleReqExtra<'_>, creds: &mut ClientCredentials<'_>, - client: &mut stream::PqStream, + client: &mut stream::PqStream>, allow_cleartext: bool, config: &'static AuthenticationConfig, latency_timer: &mut LatencyTimer, @@ -241,7 +242,7 @@ impl BackendType<'_, ClientCredentials<'_>> { pub async fn authenticate( &mut self, extra: &ConsoleReqExtra<'_>, - client: &mut stream::PqStream, + client: &mut stream::PqStream>, allow_cleartext: bool, config: &'static AuthenticationConfig, latency_timer: &mut LatencyTimer, diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs index aee0057606..ac0d490db1 100644 --- a/proxy/src/auth/backend/classic.rs +++ b/proxy/src/auth/backend/classic.rs @@ -6,7 +6,7 @@ use crate::{ console::{self, AuthInfo, ConsoleReqExtra}, proxy::LatencyTimer, sasl, scram, - stream::PqStream, + stream::{PqStream, Stream}, }; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{info, warn}; @@ -15,7 +15,7 @@ pub(super) async fn authenticate( api: &impl console::Api, extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials<'_>, - client: &mut PqStream, + client: &mut PqStream>, config: &'static AuthenticationConfig, latency_timer: &mut LatencyTimer, ) -> auth::Result> { diff --git a/proxy/src/auth/backend/hacks.rs b/proxy/src/auth/backend/hacks.rs index 895683af1b..4448dbc56a 100644 --- a/proxy/src/auth/backend/hacks.rs +++ b/proxy/src/auth/backend/hacks.rs @@ -2,7 +2,7 @@ use super::{AuthSuccess, ComputeCredentials}; use crate::{ auth::{self, AuthFlow, ClientCredentials}, proxy::LatencyTimer, - stream, + stream::{self, Stream}, }; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::{info, warn}; @@ -12,7 +12,7 @@ use tracing::{info, warn}; /// These properties are benefical for serverless JS workers, so we /// use this mechanism for websocket connections. pub async fn cleartext_hack( - client: &mut stream::PqStream, + client: &mut stream::PqStream>, latency_timer: &mut LatencyTimer, ) -> auth::Result> { warn!("cleartext auth flow override is enabled, proceeding"); @@ -37,7 +37,7 @@ pub async fn cleartext_hack( /// Very similar to [`cleartext_hack`], but there's a specific password format. pub async fn password_hack( creds: &mut ClientCredentials<'_>, - client: &mut stream::PqStream, + client: &mut stream::PqStream>, latency_timer: &mut LatencyTimer, ) -> auth::Result> { warn!("project not specified, resorting to the password hack auth flow"); diff --git a/proxy/src/auth/flow.rs b/proxy/src/auth/flow.rs index 190abc9b2e..efb90733d6 100644 --- a/proxy/src/auth/flow.rs +++ b/proxy/src/auth/flow.rs @@ -1,16 +1,21 @@ //! Main authentication flow. use super::{AuthErrorImpl, PasswordHackPayload}; -use crate::{sasl, scram, stream::PqStream}; +use crate::{ + config::TlsServerEndPoint, + sasl, scram, + stream::{PqStream, Stream}, +}; use pq_proto::{BeAuthenticationSaslMessage, BeMessage, BeMessage as Be}; use std::io; use tokio::io::{AsyncRead, AsyncWrite}; +use tracing::info; /// Every authentication selector is supposed to implement this trait. pub trait AuthMethod { /// Any authentication selector should provide initial backend message /// containing auth method name and parameters, e.g. md5 salt. - fn first_message(&self) -> BeMessage<'_>; + fn first_message(&self, channel_binding: bool) -> BeMessage<'_>; } /// Initial state of [`AuthFlow`]. @@ -21,8 +26,14 @@ pub struct Scram<'a>(pub &'a scram::ServerSecret); impl AuthMethod for Scram<'_> { #[inline(always)] - fn first_message(&self) -> BeMessage<'_> { - Be::AuthenticationSasl(BeAuthenticationSaslMessage::Methods(scram::METHODS)) + fn first_message(&self, channel_binding: bool) -> BeMessage<'_> { + if channel_binding { + Be::AuthenticationSasl(BeAuthenticationSaslMessage::Methods(scram::METHODS)) + } else { + Be::AuthenticationSasl(BeAuthenticationSaslMessage::Methods( + scram::METHODS_WITHOUT_PLUS, + )) + } } } @@ -32,7 +43,7 @@ pub struct PasswordHack; impl AuthMethod for PasswordHack { #[inline(always)] - fn first_message(&self) -> BeMessage<'_> { + fn first_message(&self, _channel_binding: bool) -> BeMessage<'_> { Be::AuthenticationCleartextPassword } } @@ -43,37 +54,44 @@ pub struct CleartextPassword; impl AuthMethod for CleartextPassword { #[inline(always)] - fn first_message(&self) -> BeMessage<'_> { + fn first_message(&self, _channel_binding: bool) -> BeMessage<'_> { Be::AuthenticationCleartextPassword } } /// This wrapper for [`PqStream`] performs client authentication. #[must_use] -pub struct AuthFlow<'a, Stream, State> { +pub struct AuthFlow<'a, S, State> { /// The underlying stream which implements libpq's protocol. - stream: &'a mut PqStream, + stream: &'a mut PqStream>, /// State might contain ancillary data (see [`Self::begin`]). state: State, + tls_server_end_point: TlsServerEndPoint, } /// Initial state of the stream wrapper. -impl<'a, S: AsyncWrite + Unpin> AuthFlow<'a, S, Begin> { +impl<'a, S: AsyncRead + AsyncWrite + Unpin> AuthFlow<'a, S, Begin> { /// Create a new wrapper for client authentication. - pub fn new(stream: &'a mut PqStream) -> Self { + pub fn new(stream: &'a mut PqStream>) -> Self { + let tls_server_end_point = stream.get_ref().tls_server_end_point(); + Self { stream, state: Begin, + tls_server_end_point, } } /// Move to the next step by sending auth method's name & params to client. pub async fn begin(self, method: M) -> io::Result> { - self.stream.write_message(&method.first_message()).await?; + self.stream + .write_message(&method.first_message(self.tls_server_end_point.supported())) + .await?; Ok(AuthFlow { stream: self.stream, state: method, + tls_server_end_point: self.tls_server_end_point, }) } } @@ -123,9 +141,15 @@ impl AuthFlow<'_, S, Scram<'_>> { return Err(super::AuthError::bad_auth_method(sasl.method)); } + info!("client chooses {}", sasl.method); + let secret = self.state.0; let outcome = sasl::SaslStream::new(self.stream, sasl.message) - .authenticate(scram::Exchange::new(secret, rand::random, None)) + .authenticate(scram::Exchange::new( + secret, + rand::random, + self.tls_server_end_point, + )) .await?; Ok(outcome) diff --git a/proxy/src/bin/pg_sni_router.rs b/proxy/src/bin/pg_sni_router.rs index 6574500c97..2b859fc2db 100644 --- a/proxy/src/bin/pg_sni_router.rs +++ b/proxy/src/bin/pg_sni_router.rs @@ -6,6 +6,8 @@ use std::{net::SocketAddr, sync::Arc}; use futures::future::Either; +use itertools::Itertools; +use proxy::config::TlsServerEndPoint; use tokio::net::TcpListener; use anyhow::{anyhow, bail, ensure, Context}; @@ -65,7 +67,7 @@ async fn main() -> anyhow::Result<()> { let destination: String = args.get_one::("dest").unwrap().parse()?; // Configure TLS - let tls_config: Arc = match ( + let (tls_config, tls_server_end_point): (Arc, TlsServerEndPoint) = match ( args.get_one::("tls-key"), args.get_one::("tls-cert"), ) { @@ -89,16 +91,22 @@ async fn main() -> anyhow::Result<()> { ))? .into_iter() .map(rustls::Certificate) - .collect() + .collect_vec() }; - rustls::ServerConfig::builder() + // needed for channel bindings + let first_cert = cert_chain.first().context("missing certificate")?; + let tls_server_end_point = TlsServerEndPoint::new(first_cert)?; + + let tls_config = rustls::ServerConfig::builder() .with_safe_default_cipher_suites() .with_safe_default_kx_groups() .with_protocol_versions(&[&rustls::version::TLS13, &rustls::version::TLS12])? .with_no_client_auth() .with_single_cert(cert_chain, key)? - .into() + .into(); + + (tls_config, tls_server_end_point) } _ => bail!("tls-key and tls-cert must be specified"), }; @@ -113,6 +121,7 @@ async fn main() -> anyhow::Result<()> { let main = tokio::spawn(task_main( Arc::new(destination), tls_config, + tls_server_end_point, proxy_listener, cancellation_token.clone(), )); @@ -134,6 +143,7 @@ async fn main() -> anyhow::Result<()> { async fn task_main( dest_suffix: Arc, tls_config: Arc, + tls_server_end_point: TlsServerEndPoint, listener: tokio::net::TcpListener, cancellation_token: CancellationToken, ) -> anyhow::Result<()> { @@ -159,7 +169,7 @@ async fn task_main( .context("failed to set socket option")?; info!(%peer_addr, "serving"); - handle_client(dest_suffix, tls_config, socket).await + handle_client(dest_suffix, tls_config, tls_server_end_point, socket).await } .unwrap_or_else(|e| { // Acknowledge that the task has finished with an error. @@ -168,9 +178,18 @@ async fn task_main( .instrument(tracing::info_span!("handle_client", ?session_id)) ); } - Some(Err(e)) = connections.join_next(), if !connections.is_empty() => { - if !e.is_panic() && !e.is_cancelled() { - warn!("unexpected error from joined connection task: {e:?}"); + // Don't modify this unless you read https://docs.rs/tokio/latest/tokio/macro.select.html carefully. + // If this future completes and the pattern doesn't match, this branch is disabled for this call to `select!`. + // This only counts for this loop and it will be enabled again on next `select!`. + // + // Prior code had this as `Some(Err(e))` which _looks_ equivalent to the current setup, but it's not. + // When `connections.join_next()` returned `Some(Ok(()))` (which we expect), it would disable the join_next and it would + // not get called again, even if there are more connections to remove. + Some(res) = connections.join_next() => { + if let Err(e) = res { + if !e.is_panic() && !e.is_cancelled() { + warn!("unexpected error from joined connection task: {e:?}"); + } } } _ = cancellation_token.cancelled() => { @@ -198,6 +217,7 @@ const ERR_INSECURE_CONNECTION: &str = "connection is insecure (try using `sslmod async fn ssl_handshake( raw_stream: S, tls_config: Arc, + tls_server_end_point: TlsServerEndPoint, ) -> anyhow::Result> { let mut stream = PqStream::new(Stream::from_raw(raw_stream)); @@ -222,7 +242,11 @@ async fn ssl_handshake( if !read_buf.is_empty() { bail!("data is sent before server replied with EncryptionResponse"); } - Ok(raw.upgrade(tls_config).await?) + + Ok(Stream::Tls { + tls: Box::new(raw.upgrade(tls_config).await?), + tls_server_end_point, + }) } unexpected => { info!( @@ -237,9 +261,10 @@ async fn ssl_handshake( async fn handle_client( dest_suffix: Arc, tls_config: Arc, + tls_server_end_point: TlsServerEndPoint, stream: impl AsyncRead + AsyncWrite + Unpin, ) -> anyhow::Result<()> { - let tls_stream = ssl_handshake(stream, tls_config).await?; + let tls_stream = ssl_handshake(stream, tls_config, tls_server_end_point).await?; // Cut off first part of the SNI domain // We receive required destination details in the format of diff --git a/proxy/src/config.rs b/proxy/src/config.rs index bd00123905..0c094ff4aa 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -1,12 +1,15 @@ use crate::auth; use anyhow::{bail, ensure, Context, Ok}; -use rustls::sign; +use rustls::{sign, Certificate, PrivateKey}; +use sha2::{Digest, Sha256}; use std::{ collections::{HashMap, HashSet}, str::FromStr, sync::Arc, time::Duration, }; +use tracing::{error, info}; +use x509_parser::oid_registry; pub struct ProxyConfig { pub tls_config: Option, @@ -27,6 +30,7 @@ pub struct MetricCollectionConfig { pub struct TlsConfig { pub config: Arc, pub common_names: Option>, + pub cert_resolver: Arc, } pub struct HttpConfig { @@ -52,7 +56,7 @@ pub fn configure_tls( let mut cert_resolver = CertResolver::new(); // add default certificate - cert_resolver.add_cert(key_path, cert_path, true)?; + cert_resolver.add_cert_path(key_path, cert_path, true)?; // add extra certificates if let Some(certs_dir) = certs_dir { @@ -64,7 +68,7 @@ pub fn configure_tls( let key_path = path.join("tls.key"); let cert_path = path.join("tls.crt"); if key_path.exists() && cert_path.exists() { - cert_resolver.add_cert( + cert_resolver.add_cert_path( &key_path.to_string_lossy(), &cert_path.to_string_lossy(), false, @@ -76,35 +80,97 @@ pub fn configure_tls( let common_names = cert_resolver.get_common_names(); + let cert_resolver = Arc::new(cert_resolver); + let config = rustls::ServerConfig::builder() .with_safe_default_cipher_suites() .with_safe_default_kx_groups() // allow TLS 1.2 to be compatible with older client libraries .with_protocol_versions(&[&rustls::version::TLS13, &rustls::version::TLS12])? .with_no_client_auth() - .with_cert_resolver(Arc::new(cert_resolver)) + .with_cert_resolver(cert_resolver.clone()) .into(); Ok(TlsConfig { config, common_names: Some(common_names), + cert_resolver, }) } -struct CertResolver { - certs: HashMap>, - default: Option>, +/// Channel binding parameter +/// +/// +/// Description: The hash of the TLS server's certificate as it +/// appears, octet for octet, in the server's Certificate message. Note +/// that the Certificate message contains a certificate_list, in which +/// the first element is the server's certificate. +/// +/// The hash function is to be selected as follows: +/// +/// * if the certificate's signatureAlgorithm uses a single hash +/// function, and that hash function is either MD5 or SHA-1, then use SHA-256; +/// +/// * if the certificate's signatureAlgorithm uses a single hash +/// function and that hash function neither MD5 nor SHA-1, then use +/// the hash function associated with the certificate's +/// signatureAlgorithm; +/// +/// * if the certificate's signatureAlgorithm uses no hash functions or +/// uses multiple hash functions, then this channel binding type's +/// channel bindings are undefined at this time (updates to is channel +/// binding type may occur to address this issue if it ever arises). +#[derive(Debug, Clone, Copy)] +pub enum TlsServerEndPoint { + Sha256([u8; 32]), + Undefined, } -impl CertResolver { - fn new() -> Self { - Self { - certs: HashMap::new(), - default: None, +impl TlsServerEndPoint { + pub fn new(cert: &Certificate) -> anyhow::Result { + let sha256_oids = [ + // I'm explicitly not adding MD5 or SHA1 here... They're bad. + oid_registry::OID_SIG_ECDSA_WITH_SHA256, + oid_registry::OID_PKCS1_SHA256WITHRSA, + ]; + + let pem = x509_parser::parse_x509_certificate(&cert.0) + .context("Failed to parse PEM object from cerficiate")? + .1; + + info!(subject = %pem.subject, "parsing TLS certificate"); + + let reg = oid_registry::OidRegistry::default().with_all_crypto(); + let oid = pem.signature_algorithm.oid(); + let alg = reg.get(oid); + if sha256_oids.contains(oid) { + let tls_server_end_point: [u8; 32] = + Sha256::new().chain_update(&cert.0).finalize().into(); + info!(subject = %pem.subject, signature_algorithm = alg.map(|a| a.description()), tls_server_end_point = %base64::encode(tls_server_end_point), "determined channel binding"); + Ok(Self::Sha256(tls_server_end_point)) + } else { + error!(subject = %pem.subject, signature_algorithm = alg.map(|a| a.description()), "unknown channel binding"); + Ok(Self::Undefined) } } - fn add_cert( + pub fn supported(&self) -> bool { + !matches!(self, TlsServerEndPoint::Undefined) + } +} + +#[derive(Default)] +pub struct CertResolver { + certs: HashMap, TlsServerEndPoint)>, + default: Option<(Arc, TlsServerEndPoint)>, +} + +impl CertResolver { + pub fn new() -> Self { + Self::default() + } + + fn add_cert_path( &mut self, key_path: &str, cert_path: &str, @@ -120,57 +186,65 @@ impl CertResolver { keys.pop().map(rustls::PrivateKey).unwrap() }; - let key = sign::any_supported_type(&priv_key).context("invalid private key")?; - let cert_chain_bytes = std::fs::read(cert_path) .context(format!("Failed to read TLS cert file at '{cert_path}.'"))?; let cert_chain = { rustls_pemfile::certs(&mut &cert_chain_bytes[..]) - .context(format!( + .with_context(|| { + format!( "Failed to read TLS certificate chain from bytes from file at '{cert_path}'." - ))? + ) + })? .into_iter() .map(rustls::Certificate) .collect() }; - let common_name = { - let pem = x509_parser::pem::parse_x509_pem(&cert_chain_bytes) - .context(format!( - "Failed to parse PEM object from bytes from file at '{cert_path}'." - ))? - .1; - let common_name = pem.parse_x509()?.subject().to_string(); + self.add_cert(priv_key, cert_chain, is_default) + } - // We only use non-wildcard certificates in link proxy so it seems okay to treat them the same as - // wildcard ones as we don't use SNI there. That treatment only affects certificate selection, so - // verify-full will still check wildcard match. Old coding here just ignored non-wildcard common names - // and passed None instead, which blows up number of cases downstream code should handle. Proper coding - // here should better avoid Option for common_names, and do wildcard-based certificate selection instead - // of cutting off '*.' parts. - if common_name.starts_with("CN=*.") { - common_name.strip_prefix("CN=*.").map(|s| s.to_string()) - } else { - common_name.strip_prefix("CN=").map(|s| s.to_string()) - } + pub fn add_cert( + &mut self, + priv_key: PrivateKey, + cert_chain: Vec, + is_default: bool, + ) -> anyhow::Result<()> { + let key = sign::any_supported_type(&priv_key).context("invalid private key")?; + + let first_cert = &cert_chain[0]; + let tls_server_end_point = TlsServerEndPoint::new(first_cert)?; + let pem = x509_parser::parse_x509_certificate(&first_cert.0) + .context("Failed to parse PEM object from cerficiate")? + .1; + + let common_name = pem.subject().to_string(); + + // We only use non-wildcard certificates in link proxy so it seems okay to treat them the same as + // wildcard ones as we don't use SNI there. That treatment only affects certificate selection, so + // verify-full will still check wildcard match. Old coding here just ignored non-wildcard common names + // and passed None instead, which blows up number of cases downstream code should handle. Proper coding + // here should better avoid Option for common_names, and do wildcard-based certificate selection instead + // of cutting off '*.' parts. + let common_name = if common_name.starts_with("CN=*.") { + common_name.strip_prefix("CN=*.").map(|s| s.to_string()) + } else { + common_name.strip_prefix("CN=").map(|s| s.to_string()) } - .context(format!( - "Failed to parse common name from certificate at '{cert_path}'." - ))?; + .context("Failed to parse common name from certificate")?; let cert = Arc::new(rustls::sign::CertifiedKey::new(cert_chain, key)); if is_default { - self.default = Some(cert.clone()); + self.default = Some((cert.clone(), tls_server_end_point)); } - self.certs.insert(common_name, cert); + self.certs.insert(common_name, (cert, tls_server_end_point)); Ok(()) } - fn get_common_names(&self) -> HashSet { + pub fn get_common_names(&self) -> HashSet { self.certs.keys().map(|s| s.to_string()).collect() } } @@ -178,15 +252,24 @@ impl CertResolver { impl rustls::server::ResolvesServerCert for CertResolver { fn resolve( &self, - _client_hello: rustls::server::ClientHello, + client_hello: rustls::server::ClientHello, ) -> Option> { + self.resolve(client_hello.server_name()).map(|x| x.0) + } +} + +impl CertResolver { + pub fn resolve( + &self, + server_name: Option<&str>, + ) -> Option<(Arc, TlsServerEndPoint)> { // loop here and cut off more and more subdomains until we find // a match to get a proper wildcard support. OTOH, we now do not // use nested domains, so keep this simple for now. // // With the current coding foo.com will match *.foo.com and that // repeats behavior of the old code. - if let Some(mut sni_name) = _client_hello.server_name() { + if let Some(mut sni_name) = server_name { loop { if let Some(cert) = self.certs.get(sni_name) { return Some(cert.clone()); diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index b05b902303..9560c8546a 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -294,9 +294,18 @@ pub async fn task_main( }), ); } - Some(Err(e)) = connections.join_next(), if !connections.is_empty() => { - if !e.is_panic() && !e.is_cancelled() { - warn!("unexpected error from joined connection task: {e:?}"); + // Don't modify this unless you read https://docs.rs/tokio/latest/tokio/macro.select.html carefully. + // If this future completes and the pattern doesn't match, this branch is disabled for this call to `select!`. + // This only counts for this loop and it will be enabled again on next `select!`. + // + // Prior code had this as `Some(Err(e))` which _looks_ equivalent to the current setup, but it's not. + // When `connections.join_next()` returned `Some(Ok(()))` (which we expect), it would disable the join_next and it would + // not get called again, even if there are more connections to remove. + Some(res) = connections.join_next() => { + if let Err(e) = res { + if !e.is_panic() && !e.is_cancelled() { + warn!("unexpected error from joined connection task: {e:?}"); + } } } _ = cancellation_token.cancelled() => { @@ -461,7 +470,17 @@ async fn handshake( if !read_buf.is_empty() { bail!("data is sent before server replied with EncryptionResponse"); } - stream = PqStream::new(raw.upgrade(tls.to_server_config()).await?); + let tls_stream = raw.upgrade(tls.to_server_config()).await?; + + let (_, tls_server_end_point) = tls + .cert_resolver + .resolve(tls_stream.get_ref().1.server_name()) + .context("missing certificate")?; + + stream = PqStream::new(Stream::Tls { + tls: Box::new(tls_stream), + tls_server_end_point, + }); } } _ => bail!(ERR_PROTO_VIOLATION), @@ -866,7 +885,7 @@ pub async fn proxy_pass( /// Thin connection context. struct Client<'a, S> { /// The underlying libpq protocol stream. - stream: PqStream, + stream: PqStream>, /// Client credentials that we care about. creds: auth::BackendType<'a, auth::ClientCredentials<'a>>, /// KV-dictionary with PostgreSQL connection params. @@ -880,7 +899,7 @@ struct Client<'a, S> { impl<'a, S> Client<'a, S> { /// Construct a new connection context. fn new( - stream: PqStream, + stream: PqStream>, creds: auth::BackendType<'a, auth::ClientCredentials<'a>>, params: &'a StartupMessageParams, session_id: uuid::Uuid, diff --git a/proxy/src/proxy/tests.rs b/proxy/src/proxy/tests.rs index 3ae4df46ef..de9cc0800b 100644 --- a/proxy/src/proxy/tests.rs +++ b/proxy/src/proxy/tests.rs @@ -1,19 +1,23 @@ //! A group of high-level tests for connection establishing logic and auth. -//! + +mod mitm; + use super::*; use crate::auth::backend::TestBackend; use crate::auth::ClientCredentials; +use crate::config::CertResolver; use crate::console::{CachedNodeInfo, NodeInfo}; use crate::{auth, http, sasl, scram}; use async_trait::async_trait; use rstest::rstest; use tokio_postgres::config::SslMode; use tokio_postgres::tls::{MakeTlsConnect, NoTls}; -use tokio_postgres_rustls::MakeRustlsConnect; +use tokio_postgres_rustls::{MakeRustlsConnect, RustlsStream}; /// Generate a set of TLS certificates: CA + server. fn generate_certs( hostname: &str, + common_name: &str, ) -> anyhow::Result<(rustls::Certificate, rustls::Certificate, rustls::PrivateKey)> { let ca = rcgen::Certificate::from_params({ let mut params = rcgen::CertificateParams::default(); @@ -21,7 +25,15 @@ fn generate_certs( params })?; - let cert = rcgen::generate_simple_self_signed(vec![hostname.into()])?; + let cert = rcgen::Certificate::from_params({ + let mut params = rcgen::CertificateParams::new(vec![hostname.into()]); + params.distinguished_name = rcgen::DistinguishedName::new(); + params + .distinguished_name + .push(rcgen::DnType::CommonName, common_name); + params + })?; + Ok(( rustls::Certificate(ca.serialize_der()?), rustls::Certificate(cert.serialize_der_with_signer(&ca)?), @@ -37,7 +49,14 @@ struct ClientConfig<'a> { impl ClientConfig<'_> { fn make_tls_connect( self, - ) -> anyhow::Result> { + ) -> anyhow::Result< + impl tokio_postgres::tls::TlsConnect< + S, + Error = impl std::fmt::Debug, + Future = impl Send, + Stream = RustlsStream, + >, + > { let mut mk = MakeRustlsConnect::new(self.config); let tls = MakeTlsConnect::::make_tls_connect(&mut mk, self.hostname)?; Ok(tls) @@ -49,20 +68,24 @@ fn generate_tls_config<'a>( hostname: &'a str, common_name: &'a str, ) -> anyhow::Result<(ClientConfig<'a>, TlsConfig)> { - let (ca, cert, key) = generate_certs(hostname)?; + let (ca, cert, key) = generate_certs(hostname, common_name)?; let tls_config = { let config = rustls::ServerConfig::builder() .with_safe_defaults() .with_no_client_auth() - .with_single_cert(vec![cert], key)? + .with_single_cert(vec![cert.clone()], key.clone())? .into(); - let common_names = Some([common_name.to_owned()].iter().cloned().collect()); + let mut cert_resolver = CertResolver::new(); + cert_resolver.add_cert(key, vec![cert], true)?; + + let common_names = Some(cert_resolver.get_common_names()); TlsConfig { config, common_names, + cert_resolver: Arc::new(cert_resolver), } }; @@ -253,6 +276,7 @@ async fn scram_auth_good(#[case] password: &str) -> anyhow::Result<()> { )); let (_client, _conn) = tokio_postgres::Config::new() + .channel_binding(tokio_postgres::config::ChannelBinding::Require) .user("user") .dbname("db") .password(password) @@ -263,6 +287,30 @@ async fn scram_auth_good(#[case] password: &str) -> anyhow::Result<()> { proxy.await? } +#[tokio::test] +async fn scram_auth_disable_channel_binding() -> anyhow::Result<()> { + let (client, server) = tokio::io::duplex(1024); + + let (client_config, server_config) = + generate_tls_config("generic-project-name.localhost", "localhost")?; + let proxy = tokio::spawn(dummy_proxy( + client, + Some(server_config), + Scram::new("password")?, + )); + + let (_client, _conn) = tokio_postgres::Config::new() + .channel_binding(tokio_postgres::config::ChannelBinding::Disable) + .user("user") + .dbname("db") + .password("password") + .ssl_mode(SslMode::Require) + .connect_raw(server, client_config.make_tls_connect()?) + .await?; + + proxy.await? +} + #[tokio::test] async fn scram_auth_mock() -> anyhow::Result<()> { let (client, server) = tokio::io::duplex(1024); diff --git a/proxy/src/proxy/tests/mitm.rs b/proxy/src/proxy/tests/mitm.rs new file mode 100644 index 0000000000..50b3034936 --- /dev/null +++ b/proxy/src/proxy/tests/mitm.rs @@ -0,0 +1,257 @@ +//! Man-in-the-middle tests +//! +//! Channel binding should prevent a proxy server +//! - that has access to create valid certificates - +//! from controlling the TLS connection. + +use std::fmt::Debug; + +use super::*; +use bytes::{Bytes, BytesMut}; +use futures::{SinkExt, StreamExt}; +use postgres_protocol::message::frontend; +use tokio::io::{AsyncReadExt, DuplexStream}; +use tokio_postgres::config::SslMode; +use tokio_postgres::tls::TlsConnect; +use tokio_util::codec::{Decoder, Encoder}; + +enum Intercept { + None, + Methods, + SASLResponse, +} + +async fn proxy_mitm( + intercept: Intercept, +) -> (DuplexStream, DuplexStream, ClientConfig<'static>, TlsConfig) { + let (end_server1, client1) = tokio::io::duplex(1024); + let (server2, end_client2) = tokio::io::duplex(1024); + + let (client_config1, server_config1) = + generate_tls_config("generic-project-name.localhost", "localhost").unwrap(); + let (client_config2, server_config2) = + generate_tls_config("generic-project-name.localhost", "localhost").unwrap(); + + tokio::spawn(async move { + // begin handshake with end_server + let end_server = connect_tls(server2, client_config2.make_tls_connect().unwrap()).await; + // process handshake with end_client + let (end_client, startup) = + handshake(client1, Some(&server_config1), &CancelMap::default()) + .await + .unwrap() + .unwrap(); + + let mut end_server = tokio_util::codec::Framed::new(end_server, PgFrame); + let (end_client, buf) = end_client.framed.into_inner(); + assert!(buf.is_empty()); + let mut end_client = tokio_util::codec::Framed::new(end_client, PgFrame); + + // give the end_server the startup parameters + let mut buf = BytesMut::new(); + frontend::startup_message(startup.iter(), &mut buf).unwrap(); + end_server.send(buf.freeze()).await.unwrap(); + + // proxy messages between end_client and end_server + loop { + tokio::select! { + message = end_server.next() => { + match message { + Some(Ok(message)) => { + // intercept SASL and return only SCRAM-SHA-256 ;) + if matches!(intercept, Intercept::Methods) && message.starts_with(b"R") && message[5..].starts_with(&[0,0,0,10]) { + end_client.send(Bytes::from_static(b"R\0\0\0\x17\0\0\0\x0aSCRAM-SHA-256\0\0")).await.unwrap(); + continue; + } + end_client.send(message).await.unwrap() + } + _ => break, + } + } + message = end_client.next() => { + match message { + Some(Ok(message)) => { + // intercept SASL response and return SCRAM-SHA-256 with no channel binding ;) + if matches!(intercept, Intercept::SASLResponse) && message.starts_with(b"p") && message[5..].starts_with(b"SCRAM-SHA-256-PLUS\0") { + let sasl_message = &message[1+4+19+4..]; + let mut new_message = b"n,,".to_vec(); + new_message.extend_from_slice(sasl_message.strip_prefix(b"p=tls-server-end-point,,").unwrap()); + + let mut buf = BytesMut::new(); + frontend::sasl_initial_response("SCRAM-SHA-256", &new_message, &mut buf).unwrap(); + + end_server.send(buf.freeze()).await.unwrap(); + continue; + } + end_server.send(message).await.unwrap() + } + _ => break, + } + } + else => { break } + } + } + }); + + (end_server1, end_client2, client_config1, server_config2) +} + +/// taken from tokio-postgres +pub async fn connect_tls(mut stream: S, tls: T) -> T::Stream +where + S: AsyncRead + AsyncWrite + Unpin, + T: TlsConnect, + T::Error: Debug, +{ + let mut buf = BytesMut::new(); + frontend::ssl_request(&mut buf); + stream.write_all(&buf).await.unwrap(); + + let mut buf = [0]; + stream.read_exact(&mut buf).await.unwrap(); + + if buf[0] != b'S' { + panic!("ssl not supported by server"); + } + + tls.connect(stream).await.unwrap() +} + +struct PgFrame; +impl Decoder for PgFrame { + type Item = Bytes; + type Error = io::Error; + + fn decode(&mut self, src: &mut BytesMut) -> Result, Self::Error> { + if src.len() < 5 { + src.reserve(5 - src.len()); + return Ok(None); + } + let len = u32::from_be_bytes(src[1..5].try_into().unwrap()) as usize + 1; + if src.len() < len { + src.reserve(len - src.len()); + return Ok(None); + } + Ok(Some(src.split_to(len).freeze())) + } +} +impl Encoder for PgFrame { + type Error = io::Error; + + fn encode(&mut self, item: Bytes, dst: &mut BytesMut) -> Result<(), Self::Error> { + dst.extend_from_slice(&item); + Ok(()) + } +} + +/// If the client doesn't support channel bindings, it can be exploited. +#[tokio::test] +async fn scram_auth_disable_channel_binding() -> anyhow::Result<()> { + let (server, client, client_config, server_config) = proxy_mitm(Intercept::None).await; + let proxy = tokio::spawn(dummy_proxy( + client, + Some(server_config), + Scram::new("password")?, + )); + + let _client_err = tokio_postgres::Config::new() + .channel_binding(tokio_postgres::config::ChannelBinding::Disable) + .user("user") + .dbname("db") + .password("password") + .ssl_mode(SslMode::Require) + .connect_raw(server, client_config.make_tls_connect()?) + .await?; + + proxy.await? +} + +/// If the client chooses SCRAM-PLUS, it will fail +#[tokio::test] +async fn scram_auth_prefer_channel_binding() -> anyhow::Result<()> { + connect_failure( + Intercept::None, + tokio_postgres::config::ChannelBinding::Prefer, + ) + .await +} + +/// If the MITM pretends like SCRAM-PLUS isn't available, but the client supports it, it will fail +#[tokio::test] +async fn scram_auth_prefer_channel_binding_intercept() -> anyhow::Result<()> { + connect_failure( + Intercept::Methods, + tokio_postgres::config::ChannelBinding::Prefer, + ) + .await +} + +/// If the MITM pretends like the client doesn't support channel bindings, it will fail +#[tokio::test] +async fn scram_auth_prefer_channel_binding_intercept_response() -> anyhow::Result<()> { + connect_failure( + Intercept::SASLResponse, + tokio_postgres::config::ChannelBinding::Prefer, + ) + .await +} + +/// If the client chooses SCRAM-PLUS, it will fail +#[tokio::test] +async fn scram_auth_require_channel_binding() -> anyhow::Result<()> { + connect_failure( + Intercept::None, + tokio_postgres::config::ChannelBinding::Require, + ) + .await +} + +/// If the client requires SCRAM-PLUS, and it is spoofed to remove SCRAM-PLUS, it will fail +#[tokio::test] +async fn scram_auth_require_channel_binding_intercept() -> anyhow::Result<()> { + connect_failure( + Intercept::Methods, + tokio_postgres::config::ChannelBinding::Require, + ) + .await +} + +/// If the client requires SCRAM-PLUS, and it is spoofed to remove SCRAM-PLUS, it will fail +#[tokio::test] +async fn scram_auth_require_channel_binding_intercept_response() -> anyhow::Result<()> { + connect_failure( + Intercept::SASLResponse, + tokio_postgres::config::ChannelBinding::Require, + ) + .await +} + +async fn connect_failure( + intercept: Intercept, + channel_binding: tokio_postgres::config::ChannelBinding, +) -> anyhow::Result<()> { + let (server, client, client_config, server_config) = proxy_mitm(intercept).await; + let proxy = tokio::spawn(dummy_proxy( + client, + Some(server_config), + Scram::new("password")?, + )); + + let _client_err = tokio_postgres::Config::new() + .channel_binding(channel_binding) + .user("user") + .dbname("db") + .password("password") + .ssl_mode(SslMode::Require) + .connect_raw(server, client_config.make_tls_connect()?) + .await + .err() + .context("client shouldn't be able to connect")?; + + let _server_err = proxy + .await? + .err() + .context("server shouldn't accept client")?; + + Ok(()) +} diff --git a/proxy/src/sasl/channel_binding.rs b/proxy/src/sasl/channel_binding.rs index 776adabe55..13d681de6d 100644 --- a/proxy/src/sasl/channel_binding.rs +++ b/proxy/src/sasl/channel_binding.rs @@ -36,9 +36,9 @@ impl<'a> ChannelBinding<&'a str> { impl ChannelBinding { /// Encode channel binding data as base64 for subsequent checks. - pub fn encode( + pub fn encode<'a, E>( &self, - get_cbind_data: impl FnOnce(&T) -> Result, + get_cbind_data: impl FnOnce(&T) -> Result<&'a [u8], E>, ) -> Result, E> { use ChannelBinding::*; Ok(match self { @@ -51,12 +51,11 @@ impl ChannelBinding { "eSws".into() } Required(mode) => { - let msg = format!( - "p={mode},,{data}", - mode = mode, - data = get_cbind_data(mode)? - ); - base64::encode(msg).into() + use std::io::Write; + let mut cbind_input = vec![]; + write!(&mut cbind_input, "p={mode},,",).unwrap(); + cbind_input.extend_from_slice(get_cbind_data(mode)?); + base64::encode(&cbind_input).into() } }) } @@ -77,7 +76,7 @@ mod tests { ]; for (cb, input) in cases { - assert_eq!(cb.encode(|_| anyhow::Ok("bar".to_owned()))?, input); + assert_eq!(cb.encode(|_| anyhow::Ok(b"bar"))?, input); } Ok(()) diff --git a/proxy/src/scram.rs b/proxy/src/scram.rs index 2de26af96b..63271309e1 100644 --- a/proxy/src/scram.rs +++ b/proxy/src/scram.rs @@ -22,9 +22,12 @@ pub use secret::ServerSecret; use hmac::{Hmac, Mac}; use sha2::{Digest, Sha256}; -// TODO: add SCRAM-SHA-256-PLUS +const SCRAM_SHA_256: &str = "SCRAM-SHA-256"; +const SCRAM_SHA_256_PLUS: &str = "SCRAM-SHA-256-PLUS"; + /// A list of supported SCRAM methods. -pub const METHODS: &[&str] = &["SCRAM-SHA-256"]; +pub const METHODS: &[&str] = &[SCRAM_SHA_256_PLUS, SCRAM_SHA_256]; +pub const METHODS_WITHOUT_PLUS: &[&str] = &[SCRAM_SHA_256]; /// Decode base64 into array without any heap allocations fn base64_decode_array(input: impl AsRef<[u8]>) -> Option<[u8; N]> { @@ -80,7 +83,11 @@ mod tests { const NONCE: [u8; 18] = [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, ]; - let mut exchange = Exchange::new(&secret, || NONCE, None); + let mut exchange = Exchange::new( + &secret, + || NONCE, + crate::config::TlsServerEndPoint::Undefined, + ); let client_first = "n,,n=user,r=rOprNGfwEbeRWgbNEkqO"; let client_final = "c=biws,r=rOprNGfwEbeRWgbNEkqOAQIDBAUGBwgJCgsMDQ4PEBES,p=rw1r5Kph5ThxmaUBC2GAQ6MfXbPnNkFiTIvdb/Rear0="; diff --git a/proxy/src/scram/exchange.rs b/proxy/src/scram/exchange.rs index 882769a70d..319d9b1014 100644 --- a/proxy/src/scram/exchange.rs +++ b/proxy/src/scram/exchange.rs @@ -5,9 +5,11 @@ use super::messages::{ }; use super::secret::ServerSecret; use super::signature::SignatureBuilder; +use crate::config; use crate::sasl::{self, ChannelBinding, Error as SaslError}; /// The only channel binding mode we currently support. +#[derive(Debug)] struct TlsServerEndPoint; impl std::fmt::Display for TlsServerEndPoint { @@ -43,20 +45,20 @@ pub struct Exchange<'a> { state: ExchangeState, secret: &'a ServerSecret, nonce: fn() -> [u8; SCRAM_RAW_NONCE_LEN], - cert_digest: Option<&'a [u8]>, + tls_server_end_point: config::TlsServerEndPoint, } impl<'a> Exchange<'a> { pub fn new( secret: &'a ServerSecret, nonce: fn() -> [u8; SCRAM_RAW_NONCE_LEN], - cert_digest: Option<&'a [u8]>, + tls_server_end_point: config::TlsServerEndPoint, ) -> Self { Self { state: ExchangeState::Initial, secret, nonce, - cert_digest, + tls_server_end_point, } } } @@ -71,6 +73,14 @@ impl sasl::Mechanism for Exchange<'_> { let client_first_message = ClientFirstMessage::parse(input) .ok_or(SaslError::BadClientMessage("invalid client-first-message"))?; + // If the flag is set to "y" and the server supports channel + // binding, the server MUST fail authentication + if client_first_message.cbind_flag == ChannelBinding::NotSupportedServer + && self.tls_server_end_point.supported() + { + return Err(SaslError::ChannelBindingFailed("SCRAM-PLUS not used")); + } + let server_first_message = client_first_message.build_server_first_message( &(self.nonce)(), &self.secret.salt_base64, @@ -94,10 +104,11 @@ impl sasl::Mechanism for Exchange<'_> { let client_final_message = ClientFinalMessage::parse(input) .ok_or(SaslError::BadClientMessage("invalid client-final-message"))?; - let channel_binding = cbind_flag.encode(|_| { - self.cert_digest - .map(base64::encode) - .ok_or(SaslError::ChannelBindingFailed("no cert digest provided")) + let channel_binding = cbind_flag.encode(|_| match &self.tls_server_end_point { + config::TlsServerEndPoint::Sha256(x) => Ok(x), + config::TlsServerEndPoint::Undefined => { + Err(SaslError::ChannelBindingFailed("no cert digest provided")) + } })?; // This might've been caused by a MITM attack diff --git a/proxy/src/stream.rs b/proxy/src/stream.rs index 6210601a80..f48b3fe39f 100644 --- a/proxy/src/stream.rs +++ b/proxy/src/stream.rs @@ -1,7 +1,8 @@ +use crate::config::TlsServerEndPoint; use crate::error::UserFacingError; use anyhow::bail; use bytes::BytesMut; -use pin_project_lite::pin_project; + use pq_proto::framed::{ConnectionError, Framed}; use pq_proto::{BeMessage, FeMessage, FeStartupPacket, ProtocolError}; use rustls::ServerConfig; @@ -17,7 +18,7 @@ use tokio_rustls::server::TlsStream; /// or [`AsyncWrite`] to prevent subtle errors (e.g. trying /// to pass random malformed bytes through the connection). pub struct PqStream { - framed: Framed, + pub(crate) framed: Framed, } impl PqStream { @@ -118,19 +119,21 @@ impl PqStream { } } -pin_project! { - /// Wrapper for upgrading raw streams into secure streams. - /// NOTE: it should be possible to decompose this object as necessary. - #[project = StreamProj] - pub enum Stream { - /// We always begin with a raw stream, - /// which may then be upgraded into a secure stream. - Raw { #[pin] raw: S }, +/// Wrapper for upgrading raw streams into secure streams. +pub enum Stream { + /// We always begin with a raw stream, + /// which may then be upgraded into a secure stream. + Raw { raw: S }, + Tls { /// We box [`TlsStream`] since it can be quite large. - Tls { #[pin] tls: Box> }, - } + tls: Box>, + /// Channel binding parameter + tls_server_end_point: TlsServerEndPoint, + }, } +impl Unpin for Stream {} + impl Stream { /// Construct a new instance from a raw stream. pub fn from_raw(raw: S) -> Self { @@ -141,7 +144,17 @@ impl Stream { pub fn sni_hostname(&self) -> Option<&str> { match self { Stream::Raw { .. } => None, - Stream::Tls { tls } => tls.get_ref().1.server_name(), + Stream::Tls { tls, .. } => tls.get_ref().1.server_name(), + } + } + + pub fn tls_server_end_point(&self) -> TlsServerEndPoint { + match self { + Stream::Raw { .. } => TlsServerEndPoint::Undefined, + Stream::Tls { + tls_server_end_point, + .. + } => *tls_server_end_point, } } } @@ -158,12 +171,9 @@ pub enum StreamUpgradeError { impl Stream { /// If possible, upgrade raw stream into a secure TLS-based stream. - pub async fn upgrade(self, cfg: Arc) -> Result { + pub async fn upgrade(self, cfg: Arc) -> Result, StreamUpgradeError> { match self { - Stream::Raw { raw } => { - let tls = Box::new(tokio_rustls::TlsAcceptor::from(cfg).accept(raw).await?); - Ok(Stream::Tls { tls }) - } + Stream::Raw { raw } => Ok(tokio_rustls::TlsAcceptor::from(cfg).accept(raw).await?), Stream::Tls { .. } => Err(StreamUpgradeError::AlreadyTls), } } @@ -171,50 +181,46 @@ impl Stream { impl AsyncRead for Stream { fn poll_read( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, context: &mut task::Context<'_>, buf: &mut ReadBuf<'_>, ) -> task::Poll> { - use StreamProj::*; - match self.project() { - Raw { raw } => raw.poll_read(context, buf), - Tls { tls } => tls.poll_read(context, buf), + match &mut *self { + Self::Raw { raw } => Pin::new(raw).poll_read(context, buf), + Self::Tls { tls, .. } => Pin::new(tls).poll_read(context, buf), } } } impl AsyncWrite for Stream { fn poll_write( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, context: &mut task::Context<'_>, buf: &[u8], ) -> task::Poll> { - use StreamProj::*; - match self.project() { - Raw { raw } => raw.poll_write(context, buf), - Tls { tls } => tls.poll_write(context, buf), + match &mut *self { + Self::Raw { raw } => Pin::new(raw).poll_write(context, buf), + Self::Tls { tls, .. } => Pin::new(tls).poll_write(context, buf), } } fn poll_flush( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, context: &mut task::Context<'_>, ) -> task::Poll> { - use StreamProj::*; - match self.project() { - Raw { raw } => raw.poll_flush(context), - Tls { tls } => tls.poll_flush(context), + match &mut *self { + Self::Raw { raw } => Pin::new(raw).poll_flush(context), + Self::Tls { tls, .. } => Pin::new(tls).poll_flush(context), } } fn poll_shutdown( - self: Pin<&mut Self>, + mut self: Pin<&mut Self>, context: &mut task::Context<'_>, ) -> task::Poll> { - use StreamProj::*; - match self.project() { - Raw { raw } => raw.poll_shutdown(context), - Tls { tls } => tls.poll_shutdown(context), + match &mut *self { + Self::Raw { raw } => Pin::new(raw).poll_shutdown(context), + Self::Tls { tls, .. } => Pin::new(tls).poll_shutdown(context), } } } diff --git a/pyproject.toml b/pyproject.toml index 396edabe10..60cf0e5c96 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ psutil = "^5.9.4" types-psutil = "^5.9.5.12" types-toml = "^0.10.8.6" pytest-httpserver = "^1.0.8" -aiohttp = "3.8.6" +aiohttp = "3.9.0" pytest-rerunfailures = "^11.1.2" types-pytest-lazy-fixture = "^0.6.3.3" pytest-split = "^0.8.1" diff --git a/s3_scrubber/src/checks.rs b/s3_scrubber/src/checks.rs index 64702fca3d..0686225d1b 100644 --- a/s3_scrubber/src/checks.rs +++ b/s3_scrubber/src/checks.rs @@ -94,11 +94,10 @@ pub(crate) async fn branch_cleanup_and_check_errors( != index_part.get_disk_consistent_lsn() { result.errors.push(format!( - "Mismatching disk_consistent_lsn in TimelineMetadata ({}) and in the index_part ({})", - index_part.metadata.disk_consistent_lsn(), - index_part.get_disk_consistent_lsn(), - - )) + "Mismatching disk_consistent_lsn in TimelineMetadata ({}) and in the index_part ({})", + index_part.metadata.disk_consistent_lsn(), + index_part.get_disk_consistent_lsn(), + )) } if index_part.layer_metadata.is_empty() { @@ -109,8 +108,8 @@ pub(crate) async fn branch_cleanup_and_check_errors( for (layer, metadata) in index_part.layer_metadata { if metadata.file_size == 0 { result.errors.push(format!( - "index_part.json contains a layer {} that has 0 size in its layer metadata", layer.file_name(), - )) + "index_part.json contains a layer {} that has 0 size in its layer metadata", layer.file_name(), + )) } let layer_map_key = (layer, metadata.generation); @@ -136,7 +135,7 @@ pub(crate) async fn branch_cleanup_and_check_errors( // a new generation that didn't upload an index yet. // // Even so, a layer that is not referenced by the index could just - // be something enqueued for deletion, so while this check is valid + // be something enqueued for deletion, so while this check is valid // for indicating that a layer is garbage, it is not an indicator // of a problem. gen < &index_part_generation) diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 2e54380471..0b5bb22c8b 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -202,6 +202,7 @@ async fn main() -> anyhow::Result<()> { logging::init( LogFormat::from_config(&args.log_format)?, logging::TracingErrorLayerEnablement::Disabled, + logging::Output::Stdout, )?; logging::replace_panic_hook_with_tracing_panic_hook().forget(); info!("version: {GIT_VERSION}"); diff --git a/storage_broker/src/bin/storage_broker.rs b/storage_broker/src/bin/storage_broker.rs index 5ce24a6a42..9f81ac6cac 100644 --- a/storage_broker/src/bin/storage_broker.rs +++ b/storage_broker/src/bin/storage_broker.rs @@ -434,6 +434,7 @@ async fn main() -> Result<(), Box> { logging::init( LogFormat::from_config(&args.log_format)?, logging::TracingErrorLayerEnablement::Disabled, + logging::Output::Stdout, )?; logging::replace_panic_hook_with_tracing_panic_hook().forget(); // initialize sentry if SENTRY_DSN is provided diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 4776fbc511..f060a63344 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -41,7 +41,12 @@ from urllib3.util.retry import Retry from fixtures.broker import NeonBroker from fixtures.log_helper import log +from fixtures.pageserver.allowed_errors import ( + DEFAULT_PAGESERVER_ALLOWED_ERRORS, + scan_pageserver_log_for_errors, +) from fixtures.pageserver.http import PageserverHttpClient +from fixtures.pageserver.types import IndexPartDump from fixtures.pageserver.utils import wait_for_last_record_lsn, wait_for_upload from fixtures.pg_version import PgVersion from fixtures.port_distributor import PortDistributor @@ -429,8 +434,6 @@ class NeonEnvBuilder: # Pageserver remote storage self.pageserver_remote_storage = pageserver_remote_storage - # Extensions remote storage - self.ext_remote_storage: Optional[S3Storage] = None # Safekeepers remote storage self.sk_remote_storage: Optional[RemoteStorage] = None @@ -529,24 +532,6 @@ class NeonEnvBuilder: ) self.pageserver_remote_storage = ret - def enable_extensions_remote_storage(self, kind: RemoteStorageKind): - assert self.ext_remote_storage is None, "already configured extensions remote storage" - - # there is an assumption that REAL_S3 for extensions is never - # cleaned up these are also special in that they have a hardcoded - # bucket and region, which is most likely the same as our normal - ext = self._configure_and_create_remote_storage( - kind, - RemoteStorageUser.EXTENSIONS, - bucket_name="neon-dev-extensions-eu-central-1", - bucket_region="eu-central-1", - ) - assert isinstance( - ext, S3Storage - ), "unsure why, but only MOCK_S3 and REAL_S3 are currently supported for extensions" - ext.cleanup = False - self.ext_remote_storage = ext - def enable_safekeeper_remote_storage(self, kind: RemoteStorageKind): assert self.sk_remote_storage is None, "sk_remote_storage already configured" @@ -603,8 +588,7 @@ class NeonEnvBuilder: directory_to_clean.rmdir() def cleanup_remote_storage(self): - # extensions are currently not cleaned up, disabled when creating - for x in [self.pageserver_remote_storage, self.ext_remote_storage, self.sk_remote_storage]: + for x in [self.pageserver_remote_storage, self.sk_remote_storage]: if isinstance(x, S3Storage): x.do_cleanup() @@ -702,12 +686,12 @@ class NeonEnv: self.port_distributor = config.port_distributor self.s3_mock_server = config.mock_s3_server self.neon_cli = NeonCli(env=self) + self.pagectl = Pagectl(env=self) self.endpoints = EndpointFactory(self) self.safekeepers: List[Safekeeper] = [] self.pageservers: List[NeonPageserver] = [] self.broker = config.broker self.pageserver_remote_storage = config.pageserver_remote_storage - self.ext_remote_storage = config.ext_remote_storage self.safekeepers_remote_storage = config.sk_remote_storage self.pg_version = config.pg_version # Binary path for pageserver, safekeeper, etc @@ -1222,6 +1206,7 @@ class NeonCli(AbstractNeonCli): self, new_branch_name: str, tenant_id: Optional[TenantId] = None, + timeline_id: Optional[TimelineId] = None, ) -> TimelineId: cmd = [ "timeline", @@ -1234,6 +1219,9 @@ class NeonCli(AbstractNeonCli): self.env.pg_version, ] + if timeline_id is not None: + cmd.extend(["--timeline-id", str(timeline_id)]) + res = self.raw_cli(cmd) res.check_returncode() @@ -1459,12 +1447,7 @@ class NeonCli(AbstractNeonCli): if pageserver_id is not None: args.extend(["--pageserver-id", str(pageserver_id)]) - storage = self.env.ext_remote_storage - s3_env_vars = None - if isinstance(storage, S3Storage): - s3_env_vars = storage.access_env_vars() - - res = self.raw_cli(args, extra_env_vars=s3_env_vars) + res = self.raw_cli(args) res.check_returncode() return res @@ -1558,6 +1541,20 @@ class ComputeCtl(AbstractNeonCli): COMMAND = "compute_ctl" +class Pagectl(AbstractNeonCli): + """ + A typed wrapper around the `pagectl` utility CLI tool. + """ + + COMMAND = "pagectl" + + def dump_index_part(self, path: Path) -> IndexPartDump: + res = self.raw_cli(["index-part", "dump", str(path)]) + res.check_returncode() + parsed = json.loads(res.stdout) + return IndexPartDump.from_json(parsed) + + class NeonAttachmentService: def __init__(self, env: NeonEnv): self.env = env @@ -1575,7 +1572,7 @@ class NeonAttachmentService: self.running = False return self - def attach_hook(self, tenant_id: TenantId, pageserver_id: int) -> int: + def attach_hook_issue(self, tenant_id: TenantId, pageserver_id: int) -> int: response = requests.post( f"{self.env.control_plane_api}/attach-hook", json={"tenant_id": str(tenant_id), "node_id": pageserver_id}, @@ -1585,6 +1582,13 @@ class NeonAttachmentService: assert isinstance(gen, int) return gen + def attach_hook_drop(self, tenant_id: TenantId): + response = requests.post( + f"{self.env.control_plane_api}/attach-hook", + json={"tenant_id": str(tenant_id), "node_id": None}, + ) + response.raise_for_status() + def __enter__(self) -> "NeonAttachmentService": return self @@ -1622,57 +1626,7 @@ class NeonPageserver(PgProtocol): # env.pageserver.allowed_errors.append(".*could not open garage door.*") # # The entries in the list are regular experessions. - self.allowed_errors = [ - # All tests print these, when starting up or shutting down - ".*wal receiver task finished with an error: walreceiver connection handling failure.*", - ".*Shutdown task error: walreceiver connection handling failure.*", - ".*wal_connection_manager.*tcp connect error: Connection refused.*", - ".*query handler for .* failed: Socket IO error: Connection reset by peer.*", - ".*serving compute connection task.*exited with error: Postgres connection error.*", - ".*serving compute connection task.*exited with error: Connection reset by peer.*", - ".*serving compute connection task.*exited with error: Postgres query error.*", - ".*Connection aborted: error communicating with the server: Transport endpoint is not connected.*", - # FIXME: replication patch for tokio_postgres regards any but CopyDone/CopyData message in CopyBoth stream as unexpected - ".*Connection aborted: unexpected message from server*", - ".*kill_and_wait_impl.*: wait successful.*", - ".*query handler for 'pagestream.*failed: Broken pipe.*", # pageserver notices compute shut down - ".*query handler for 'pagestream.*failed: Connection reset by peer.*", # pageserver notices compute shut down - # safekeeper connection can fail with this, in the window between timeline creation - # and streaming start - ".*Failed to process query for timeline .*: state uninitialized, no data to read.*", - # Tests related to authentication and authorization print these - ".*Error processing HTTP request: Forbidden", - # intentional failpoints - ".*failpoint ", - # FIXME: These need investigation - ".*manual_gc.*is_shutdown_requested\\(\\) called in an unexpected task or thread.*", - ".*tenant_list: timeline is not found in remote index while it is present in the tenants registry.*", - ".*Removing intermediate uninit mark file.*", - # Tenant::delete_timeline() can cause any of the four following errors. - # FIXME: we shouldn't be considering it an error: https://github.com/neondatabase/neon/issues/2946 - ".*could not flush frozen layer.*queue is in state Stopped", # when schedule layer upload fails because queued got closed before compaction got killed - ".*wait for layer upload ops to complete.*", # .*Caused by:.*wait_completion aborted because upload queue was stopped - ".*gc_loop.*Gc failed, retrying in.*timeline is Stopping", # When gc checks timeline state after acquiring layer_removal_cs - ".*gc_loop.*Gc failed, retrying in.*: Cannot run GC iteration on inactive tenant", # Tenant::gc precondition - ".*compaction_loop.*Compaction failed.*, retrying in.*timeline or pageserver is shutting down", # When compaction checks timeline state after acquiring layer_removal_cs - ".*query handler for 'pagestream.*failed: Timeline .* was not found", # postgres reconnects while timeline_delete doesn't hold the tenant's timelines.lock() - ".*query handler for 'pagestream.*failed: Timeline .* is not active", # timeline delete in progress - ".*task iteration took longer than the configured period.*", - # this is until #3501 - ".*Compaction failed.*, retrying in [^:]+: Cannot run compaction iteration on inactive tenant", - # these can happen anytime we do compactions from background task and shutdown pageserver - r".*ERROR.*ancestor timeline \S+ is being stopped", - # this is expected given our collaborative shutdown approach for the UploadQueue - ".*Compaction failed.*, retrying in .*: Other\\(queue is in state Stopped.*", - # Pageserver timeline deletion should be polled until it gets 404, so ignore it globally - ".*Error processing HTTP request: NotFound: Timeline .* was not found", - ".*took more than expected to complete.*", - # these can happen during shutdown, but it should not be a reason to fail a test - ".*completed, took longer than expected.*", - # AWS S3 may emit 500 errors for keys in a DeleteObjects response: we retry these - # and it is not a failure of our code when it happens. - ".*DeleteObjects.*We encountered an internal error. Please try again.*", - ] + self.allowed_errors: List[str] = list(DEFAULT_PAGESERVER_ALLOWED_ERRORS) def timeline_dir(self, tenant_id: TenantId, timeline_id: Optional[TimelineId] = None) -> Path: """Get a timeline directory's path based on the repo directory of the test environment""" @@ -1782,27 +1736,9 @@ class NeonPageserver(PgProtocol): def assert_no_errors(self): logfile = open(os.path.join(self.workdir, "pageserver.log"), "r") - error_or_warn = re.compile(r"\s(ERROR|WARN)") - errors = [] - while True: - line = logfile.readline() - if not line: - break + errors = scan_pageserver_log_for_errors(logfile, self.allowed_errors) - if error_or_warn.search(line): - # Is this a torn log line? This happens when force-killing a process and restarting - # Example: "2023-10-25T09:38:31.752314Z WARN deletion executo2023-10-25T09:38:31.875947Z INFO version: git-env:0f9452f76e8ccdfc88291bccb3f53e3016f40192" - if re.match("\\d{4}-\\d{2}-\\d{2}T.+\\d{4}-\\d{2}-\\d{2}T.+INFO version.+", line): - continue - - # It's an ERROR or WARN. Is it in the allow-list? - for a in self.allowed_errors: - if re.match(a, line): - break - else: - errors.append(line) - - for error in errors: + for _lineno, error in errors: log.info(f"not allowed error: {error.strip()}") assert not errors @@ -1852,13 +1788,20 @@ class NeonPageserver(PgProtocol): to call into the pageserver HTTP client. """ if self.env.attachment_service is not None: - generation = self.env.attachment_service.attach_hook(tenant_id, self.id) + generation = self.env.attachment_service.attach_hook_issue(tenant_id, self.id) else: generation = None client = self.http_client() return client.tenant_attach(tenant_id, config, config_null, generation=generation) + def tenant_detach(self, tenant_id: TenantId): + if self.env.attachment_service is not None: + self.env.attachment_service.attach_hook_drop(tenant_id) + + client = self.http_client() + return client.tenant_detach(tenant_id) + def append_pageserver_param_overrides( params_to_update: List[str], @@ -2626,6 +2569,17 @@ class Endpoint(PgProtocol): with open(config_path, "w") as file: json.dump(dict(data_dict, **kwargs), file, indent=4) + # Mock the extension part of spec passed from control plane for local testing + # endpooint.rs adds content of this file as a part of the spec.json + def create_remote_extension_spec(self, spec: dict[str, Any]): + """Create a remote extension spec file for the endpoint.""" + remote_extensions_spec_path = os.path.join( + self.endpoint_path(), "remote_extensions_spec.json" + ) + + with open(remote_extensions_spec_path, "w") as file: + json.dump(spec, file, indent=4) + def stop(self) -> "Endpoint": """ Stop the Postgres instance if it's running. diff --git a/test_runner/fixtures/pageserver/allowed_errors.py b/test_runner/fixtures/pageserver/allowed_errors.py new file mode 100755 index 0000000000..5c3ae3ce4b --- /dev/null +++ b/test_runner/fixtures/pageserver/allowed_errors.py @@ -0,0 +1,116 @@ +#! /usr/bin/env python3 + +import argparse +import re +import sys +from typing import Iterable, List, Tuple + + +def scan_pageserver_log_for_errors( + input: Iterable[str], allowed_errors: List[str] +) -> List[Tuple[int, str]]: + error_or_warn = re.compile(r"\s(ERROR|WARN)") + errors = [] + for lineno, line in enumerate(input, start=1): + if len(line) == 0: + continue + + if error_or_warn.search(line): + # Is this a torn log line? This happens when force-killing a process and restarting + # Example: "2023-10-25T09:38:31.752314Z WARN deletion executo2023-10-25T09:38:31.875947Z INFO version: git-env:0f9452f76e8ccdfc88291bccb3f53e3016f40192" + if re.match("\\d{4}-\\d{2}-\\d{2}T.+\\d{4}-\\d{2}-\\d{2}T.+INFO version.+", line): + continue + + # It's an ERROR or WARN. Is it in the allow-list? + for a in allowed_errors: + if re.match(a, line): + break + else: + errors.append((lineno, line)) + return errors + + +DEFAULT_PAGESERVER_ALLOWED_ERRORS = ( + # All tests print these, when starting up or shutting down + ".*wal receiver task finished with an error: walreceiver connection handling failure.*", + ".*Shutdown task error: walreceiver connection handling failure.*", + ".*wal_connection_manager.*tcp connect error: Connection refused.*", + ".*query handler for .* failed: Socket IO error: Connection reset by peer.*", + ".*serving compute connection task.*exited with error: Postgres connection error.*", + ".*serving compute connection task.*exited with error: Connection reset by peer.*", + ".*serving compute connection task.*exited with error: Postgres query error.*", + ".*Connection aborted: error communicating with the server: Transport endpoint is not connected.*", + # FIXME: replication patch for tokio_postgres regards any but CopyDone/CopyData message in CopyBoth stream as unexpected + ".*Connection aborted: unexpected message from server*", + ".*kill_and_wait_impl.*: wait successful.*", + ".*query handler for 'pagestream.*failed: Broken pipe.*", # pageserver notices compute shut down + ".*query handler for 'pagestream.*failed: Connection reset by peer.*", # pageserver notices compute shut down + # safekeeper connection can fail with this, in the window between timeline creation + # and streaming start + ".*Failed to process query for timeline .*: state uninitialized, no data to read.*", + # Tests related to authentication and authorization print these + ".*Error processing HTTP request: Forbidden", + # intentional failpoints + ".*failpoint ", + # FIXME: These need investigation + ".*manual_gc.*is_shutdown_requested\\(\\) called in an unexpected task or thread.*", + ".*tenant_list: timeline is not found in remote index while it is present in the tenants registry.*", + ".*Removing intermediate uninit mark file.*", + # Tenant::delete_timeline() can cause any of the four following errors. + # FIXME: we shouldn't be considering it an error: https://github.com/neondatabase/neon/issues/2946 + ".*could not flush frozen layer.*queue is in state Stopped", # when schedule layer upload fails because queued got closed before compaction got killed + ".*wait for layer upload ops to complete.*", # .*Caused by:.*wait_completion aborted because upload queue was stopped + ".*gc_loop.*Gc failed, retrying in.*timeline is Stopping", # When gc checks timeline state after acquiring layer_removal_cs + ".*gc_loop.*Gc failed, retrying in.*: Cannot run GC iteration on inactive tenant", # Tenant::gc precondition + ".*compaction_loop.*Compaction failed.*, retrying in.*timeline or pageserver is shutting down", # When compaction checks timeline state after acquiring layer_removal_cs + ".*query handler for 'pagestream.*failed: Timeline .* was not found", # postgres reconnects while timeline_delete doesn't hold the tenant's timelines.lock() + ".*query handler for 'pagestream.*failed: Timeline .* is not active", # timeline delete in progress + ".*task iteration took longer than the configured period.*", + # these can happen anytime we do compactions from background task and shutdown pageserver + r".*ERROR.*ancestor timeline \S+ is being stopped", + # this is expected given our collaborative shutdown approach for the UploadQueue + ".*Compaction failed.*, retrying in .*: Other\\(queue is in state Stopped.*", + ".*Compaction failed.*, retrying in .*: ShuttingDown", + # Pageserver timeline deletion should be polled until it gets 404, so ignore it globally + ".*Error processing HTTP request: NotFound: Timeline .* was not found", + ".*took more than expected to complete.*", + # these can happen during shutdown, but it should not be a reason to fail a test + ".*completed, took longer than expected.*", + # AWS S3 may emit 500 errors for keys in a DeleteObjects response: we retry these + # and it is not a failure of our code when it happens. + ".*DeleteObjects.*We encountered an internal error. Please try again.*", +) + + +def _check_allowed_errors(input): + allowed_errors: List[str] = list(DEFAULT_PAGESERVER_ALLOWED_ERRORS) + + # add any test specifics here; cli parsing is not provided for the + # difficulty of copypasting regexes as arguments without any quoting + # errors. + + errors = scan_pageserver_log_for_errors(input, allowed_errors) + + for lineno, error in errors: + print(f"-:{lineno}: {error.strip()}", file=sys.stderr) + + print(f"\n{len(errors)} not allowed errors", file=sys.stderr) + + return errors + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="check input against pageserver global allowed_errors" + ) + parser.add_argument( + "-i", + "--input", + type=argparse.FileType("r"), + default=sys.stdin, + help="Pageserver logs file. Reads from stdin if no file is provided.", + ) + args = parser.parse_args() + errors = _check_allowed_errors(args.input) + + sys.exit(len(errors) > 0) diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index aff7959aa7..76edf45496 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -4,7 +4,7 @@ import json import time from collections import defaultdict from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional, Set, Tuple import requests from requests.adapters import HTTPAdapter @@ -100,6 +100,15 @@ class LayerMapInfo: counts[hist_layer.kind] += 1 return counts + def delta_layers(self) -> List[HistoricLayerInfo]: + return [x for x in self.historic_layers if x.kind == "Delta"] + + def image_layers(self) -> List[HistoricLayerInfo]: + return [x for x in self.historic_layers if x.kind == "Image"] + + def historic_by_name(self) -> Set[str]: + return set(x.layer_file_name for x in self.historic_layers) + @dataclass class TenantConfig: @@ -416,6 +425,10 @@ class PageserverHttpClient(requests.Session): def timeline_gc( self, tenant_id: TenantId, timeline_id: TimelineId, gc_horizon: Optional[int] ) -> dict[str, Any]: + """ + Unlike most handlers, this will wait for the layers to be actually + complete registering themselves to the deletion queue. + """ self.is_testing_enabled_or_skip() log.info( @@ -432,12 +445,18 @@ class PageserverHttpClient(requests.Session): assert isinstance(res_json, dict) return res_json - def timeline_compact(self, tenant_id: TenantId, timeline_id: TimelineId): + def timeline_compact( + self, tenant_id: TenantId, timeline_id: TimelineId, force_repartition=False + ): self.is_testing_enabled_or_skip() + query = {} + if force_repartition: + query["force_repartition"] = "true" log.info(f"Requesting compact: tenant {tenant_id}, timeline {timeline_id}") res = self.put( - f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/compact" + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/compact", + params=query, ) log.info(f"Got compact request response code: {res.status_code}") self.verbose_error(res) @@ -466,12 +485,18 @@ class PageserverHttpClient(requests.Session): res_json = res.json() return res_json - def timeline_checkpoint(self, tenant_id: TenantId, timeline_id: TimelineId): + def timeline_checkpoint( + self, tenant_id: TenantId, timeline_id: TimelineId, force_repartition=False + ): self.is_testing_enabled_or_skip() + query = {} + if force_repartition: + query["force_repartition"] = "true" log.info(f"Requesting checkpoint: tenant {tenant_id}, timeline {timeline_id}") res = self.put( - f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/checkpoint" + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/checkpoint", + params=query, ) log.info(f"Got checkpoint request response code: {res.status_code}") self.verbose_error(res) diff --git a/test_runner/fixtures/pageserver/types.py b/test_runner/fixtures/pageserver/types.py new file mode 100644 index 0000000000..30e3f527bf --- /dev/null +++ b/test_runner/fixtures/pageserver/types.py @@ -0,0 +1,146 @@ +from dataclasses import dataclass +from typing import Any, Dict, Tuple, Union + +from fixtures.types import KEY_MAX, KEY_MIN, Key, Lsn + + +@dataclass +class IndexLayerMetadata: + @classmethod + def from_json(cls, d: Dict[str, Any]): + return {} + + +@dataclass(frozen=True) +class ImageLayerFileName: + lsn: Lsn + key_start: Key + key_end: Key + + def to_str(self): + ret = ( + f"{self.key_start.as_int():036X}-{self.key_end.as_int():036X}__{self.lsn.as_int():016X}" + ) + assert self == parse_layer_file_name(ret) + return ret + + +@dataclass(frozen=True) +class DeltaLayerFileName: + lsn_start: Lsn + lsn_end: Lsn + key_start: Key + key_end: Key + + def is_l0(self): + return self.key_start == KEY_MIN and self.key_end == KEY_MAX + + def to_str(self): + ret = f"{self.key_start.as_int():036X}-{self.key_end.as_int():036X}__{self.lsn_start.as_int():016X}-{self.lsn_end.as_int():016X}" + assert self == parse_layer_file_name(ret) + return ret + + +LayerFileName = Union[ImageLayerFileName, DeltaLayerFileName] + + +class InvalidFileName(Exception): + pass + + +def parse_image_layer(f_name: str) -> Tuple[int, int, int]: + """Parse an image layer file name. Return key start, key end, and snapshot lsn""" + parts = f_name.split("__") + if len(parts) != 2: + raise InvalidFileName(f"expecting two parts separated by '__', got: {parts}") + key_parts = parts[0].split("-") + if len(key_parts) != 2: + raise InvalidFileName( + f"expecting two key parts separated by '--' in parts[0], got: {key_parts}" + ) + try: + return int(key_parts[0], 16), int(key_parts[1], 16), int(parts[1], 16) + except ValueError as e: + raise InvalidFileName(f"conversion error: {f_name}") from e + + +def parse_delta_layer(f_name: str) -> Tuple[int, int, int, int]: + """Parse a delta layer file name. Return key start, key end, lsn start, and lsn end""" + parts = f_name.split("__") + if len(parts) != 2: + raise InvalidFileName(f"expecting two parts separated by '__', got: {parts}") + key_parts = parts[0].split("-") + if len(key_parts) != 2: + raise InvalidFileName( + f"expecting two key parts separated by '--' in parts[0], got: {key_parts}" + ) + lsn_parts = parts[1].split("-") + if len(lsn_parts) != 2: + raise InvalidFileName( + f"expecting two lsn parts separated by '--' in parts[1], got: {lsn_parts}" + ) + try: + return ( + int(key_parts[0], 16), + int(key_parts[1], 16), + int(lsn_parts[0], 16), + int(lsn_parts[1], 16), + ) + except ValueError as e: + raise InvalidFileName(f"conversion error: {f_name}") from e + + +def parse_layer_file_name(file_name: str) -> LayerFileName: + try: + key_start, key_end, lsn = parse_image_layer(file_name) + return ImageLayerFileName(lsn=Lsn(lsn), key_start=Key(key_start), key_end=Key(key_end)) + except InvalidFileName: + pass + + try: + key_start, key_end, lsn_start, lsn_end = parse_delta_layer(file_name) + return DeltaLayerFileName( + lsn_start=Lsn(lsn_start), + lsn_end=Lsn(lsn_end), + key_start=Key(key_start), + key_end=Key(key_end), + ) + except InvalidFileName: + pass + + raise ValueError() + + +def is_future_layer(layer_file_name: LayerFileName, disk_consistent_lsn: Lsn): + """ + Determines if this layer file is considered to be in future meaning we will discard these + layers during timeline initialization from the given disk_consistent_lsn. + """ + if ( + isinstance(layer_file_name, ImageLayerFileName) + and layer_file_name.lsn > disk_consistent_lsn + ): + return True + elif ( + isinstance(layer_file_name, DeltaLayerFileName) + and layer_file_name.lsn_end > disk_consistent_lsn + 1 + ): + return True + else: + return False + + +@dataclass +class IndexPartDump: + layer_metadata: Dict[LayerFileName, IndexLayerMetadata] + disk_consistent_lsn: Lsn + + @classmethod + def from_json(cls, d: Dict[str, Any]) -> "IndexPartDump": + return IndexPartDump( + layer_metadata={ + parse_layer_file_name(n): IndexLayerMetadata.from_json(v) + for n, v in d["layer_metadata"].items() + }, + disk_consistent_lsn=Lsn(d["disk_consistent_lsn"]), + ) diff --git a/test_runner/fixtures/remote_storage.py b/test_runner/fixtures/remote_storage.py index 535f8c2fe7..954c3142a3 100644 --- a/test_runner/fixtures/remote_storage.py +++ b/test_runner/fixtures/remote_storage.py @@ -12,6 +12,7 @@ import boto3 from mypy_boto3_s3 import S3Client from fixtures.log_helper import log +from fixtures.pageserver.types import LayerFileName from fixtures.types import TenantId, TimelineId TIMELINE_INDEX_PART_FILE_NAME = "index_part.json" @@ -87,6 +88,11 @@ class LocalFsStorage: def timeline_path(self, tenant_id: TenantId, timeline_id: TimelineId) -> Path: return self.tenant_path(tenant_id) / "timelines" / str(timeline_id) + def layer_path( + self, tenant_id: TenantId, timeline_id: TimelineId, layer_file_name: LayerFileName + ): + return self.timeline_path(tenant_id, timeline_id) / layer_file_name.to_str() + def index_path(self, tenant_id: TenantId, timeline_id: TimelineId) -> Path: return self.timeline_path(tenant_id, timeline_id) / TIMELINE_INDEX_PART_FILE_NAME diff --git a/test_runner/fixtures/types.py b/test_runner/fixtures/types.py index ef88e09de4..d95368f990 100644 --- a/test_runner/fixtures/types.py +++ b/test_runner/fixtures/types.py @@ -1,4 +1,5 @@ import random +from dataclasses import dataclass from functools import total_ordering from typing import Any, Type, TypeVar, Union @@ -36,6 +37,11 @@ class Lsn: return NotImplemented return self.lsn_int < other.lsn_int + def __gt__(self, other: Any) -> bool: + if not isinstance(other, Lsn): + raise NotImplementedError + return self.lsn_int > other.lsn_int + def __eq__(self, other: Any) -> bool: if not isinstance(other, Lsn): return NotImplemented @@ -47,9 +53,32 @@ class Lsn: return NotImplemented return self.lsn_int - other.lsn_int + def __add__(self, other: Union[int, "Lsn"]) -> "Lsn": + if isinstance(other, int): + return Lsn(self.lsn_int + other) + elif isinstance(other, Lsn): + return Lsn(self.lsn_int + other.lsn_int) + else: + raise NotImplementedError + def __hash__(self) -> int: return hash(self.lsn_int) + def as_int(self) -> int: + return self.lsn_int + + +@dataclass(frozen=True) +class Key: + key_int: int + + def as_int(self) -> int: + return self.key_int + + +KEY_MAX = Key(0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF) +KEY_MIN = Key(0) + @total_ordering class Id: diff --git a/test_runner/fixtures/utils.py b/test_runner/fixtures/utils.py index ba8d70d5a9..6e857766e5 100644 --- a/test_runner/fixtures/utils.py +++ b/test_runner/fixtures/utils.py @@ -6,7 +6,16 @@ import subprocess import threading import time from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional, Tuple, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Dict, + List, + Optional, + Tuple, + TypeVar, +) from urllib.parse import urlencode import allure @@ -14,6 +23,10 @@ import zstandard from psycopg2.extensions import cursor from fixtures.log_helper import log +from fixtures.pageserver.types import ( + parse_delta_layer, + parse_image_layer, +) if TYPE_CHECKING: from fixtures.neon_fixtures import PgBin @@ -193,26 +206,6 @@ def get_timeline_dir_size(path: Path) -> int: return sz -def parse_image_layer(f_name: str) -> Tuple[int, int, int]: - """Parse an image layer file name. Return key start, key end, and snapshot lsn""" - parts = f_name.split("__") - key_parts = parts[0].split("-") - return int(key_parts[0], 16), int(key_parts[1], 16), int(parts[1], 16) - - -def parse_delta_layer(f_name: str) -> Tuple[int, int, int, int]: - """Parse a delta layer file name. Return key start, key end, lsn start, and lsn end""" - parts = f_name.split("__") - key_parts = parts[0].split("-") - lsn_parts = parts[1].split("-") - return ( - int(key_parts[0], 16), - int(key_parts[1], 16), - int(lsn_parts[0], 16), - int(lsn_parts[1], 16), - ) - - def get_scale_for_db(size_mb: int) -> int: """Returns pgbench scale factor for given target db size in MB. diff --git a/test_runner/performance/README.md b/test_runner/performance/README.md index d113e9e637..7ad65821d4 100644 --- a/test_runner/performance/README.md +++ b/test_runner/performance/README.md @@ -14,7 +14,7 @@ Some handy pytest flags for local development: - `-s` shows test output - `-k` selects a test to run - `--timeout=0` disables our default timeout of 300s (see `setup.cfg`) -- `--cleanup-test-ouput` cleans up after each test +- `--preserve-database-files` to skip cleanup # What performance tests do we have and how we run them diff --git a/test_runner/regress/test_backpressure.py b/test_runner/regress/test_backpressure.py index b14974279e..bc3faf9271 100644 --- a/test_runner/regress/test_backpressure.py +++ b/test_runner/regress/test_backpressure.py @@ -24,8 +24,6 @@ def check_backpressure(endpoint: Endpoint, stop_event: threading.Event, polling_ log.info("checks started") with pg_cur(endpoint) as cur: - cur.execute("CREATE EXTENSION neon") # TODO move it to neon_fixtures? - cur.execute("select pg_size_bytes(current_setting('max_replication_write_lag'))") res = cur.fetchone() max_replication_write_lag_bytes = res[0] @@ -102,9 +100,13 @@ def test_backpressure_received_lsn_lag(neon_env_builder: NeonEnvBuilder): # Create a branch for us env.neon_cli.create_branch("test_backpressure") - endpoint = env.endpoints.create_start( + endpoint = env.endpoints.create( "test_backpressure", config_lines=["max_replication_write_lag=30MB"] ) + # don't skip pg_catalog updates - it runs CREATE EXTENSION neon + # which is needed for backpressure_lsns() to work + endpoint.respec(skip_pg_catalog_updates=False) + endpoint.start() log.info("postgres is running on 'test_backpressure' branch") # setup check thread diff --git a/test_runner/regress/test_broken_timeline.py b/test_runner/regress/test_broken_timeline.py index 23839a4dd1..4911fc09d6 100644 --- a/test_runner/regress/test_broken_timeline.py +++ b/test_runner/regress/test_broken_timeline.py @@ -114,6 +114,7 @@ def test_timeline_init_break_before_checkpoint(neon_env_builder: NeonEnvBuilder) [ ".*Failed to process timeline dir contents.*Timeline has no ancestor and no layer files.*", ".*Timeline got dropped without initializing, cleaning its files.*", + ".*Failed to load index_part from remote storage, failed creation?.*", ] ) @@ -143,6 +144,58 @@ def test_timeline_init_break_before_checkpoint(neon_env_builder: NeonEnvBuilder) ), "pageserver should clean its temp timeline files on timeline creation failure" +def test_timeline_init_break_before_checkpoint_recreate(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + pageserver_http = env.pageserver.http_client() + + env.pageserver.allowed_errors.extend( + [ + ".*Failed to process timeline dir contents.*Timeline has no ancestor and no layer files.*", + ".*Timeline got dropped without initializing, cleaning its files.*", + ".*Failed to load index_part from remote storage, failed creation?.*", + ] + ) + + tenant_id = env.initial_tenant + + timelines_dir = env.pageserver.timeline_dir(tenant_id) + old_tenant_timelines = env.neon_cli.list_timelines(tenant_id) + initial_timeline_dirs = [d for d in timelines_dir.iterdir()] + + # Some fixed timeline ID (like control plane does) + timeline_id = TimelineId("1080243c1f76fe3c5147266663c9860b") + + # Introduce failpoint during timeline init (some intermediate files are on disk), before it's checkpointed. + pageserver_http.configure_failpoints(("before-checkpoint-new-timeline", "return")) + with pytest.raises(Exception, match="before-checkpoint-new-timeline"): + _ = env.neon_cli.create_timeline( + "test_timeline_init_break_before_checkpoint", tenant_id, timeline_id + ) + + # Restart the page server + env.pageserver.restart(immediate=True) + + # Creating the timeline didn't finish. The other timelines on tenant should still be present and work normally. + new_tenant_timelines = env.neon_cli.list_timelines(tenant_id) + assert ( + new_tenant_timelines == old_tenant_timelines + ), f"Pageserver after restart should ignore non-initialized timelines for tenant {tenant_id}" + + timeline_dirs = [d for d in timelines_dir.iterdir()] + assert ( + timeline_dirs == initial_timeline_dirs + ), "pageserver should clean its temp timeline files on timeline creation failure" + + # Disable the failpoint again + pageserver_http.configure_failpoints(("before-checkpoint-new-timeline", "off")) + # creating the branch should have worked now + new_timeline_id = env.neon_cli.create_timeline( + "test_timeline_init_break_before_checkpoint", tenant_id, timeline_id + ) + + assert timeline_id == new_timeline_id + + def test_timeline_create_break_after_uninit_mark(neon_env_builder: NeonEnvBuilder): env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() diff --git a/test_runner/regress/test_compatibility.py b/test_runner/regress/test_compatibility.py index 98f6677c00..9a2980280c 100644 --- a/test_runner/regress/test_compatibility.py +++ b/test_runner/regress/test_compatibility.py @@ -411,7 +411,6 @@ def check_neon_works( config.initial_tenant = snapshot_config["default_tenant_id"] config.pg_distrib_dir = pg_distrib_dir config.remote_storage = None - config.ext_remote_storage = None config.sk_remote_storage = None # Use the "target" binaries to launch the storage nodes diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py index 775ad10241..27eb05ac09 100644 --- a/test_runner/regress/test_download_extensions.py +++ b/test_runner/regress/test_download_extensions.py @@ -1,316 +1,137 @@ import os import shutil -import threading from contextlib import closing from pathlib import Path +from typing import Any, Dict import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, ) -from fixtures.pg_version import PgVersion, skip_on_postgres -from fixtures.remote_storage import ( - RemoteStorageKind, - S3Storage, - available_s3_storages, -) +from fixtures.pg_version import PgVersion +from pytest_httpserver import HTTPServer +from werkzeug.wrappers.request import Request +from werkzeug.wrappers.response import Response -# Cleaning up downloaded files is important for local tests -# or else one test could reuse the files from another test or another test run -def cleanup(pg_version): - PGDIR = Path(f"pg_install/v{pg_version}") +# use neon_env_builder_local fixture to override the default neon_env_builder fixture +# and use a test-specific pg_install instead of shared one +@pytest.fixture(scope="function") +def neon_env_builder_local( + neon_env_builder: NeonEnvBuilder, + test_output_dir: Path, + pg_distrib_dir: Path, + pg_version: PgVersion, +) -> NeonEnvBuilder: + test_local_pginstall = test_output_dir / "pg_install" + log.info(f"copy {pg_distrib_dir} to {test_local_pginstall}") + shutil.copytree( + pg_distrib_dir / pg_version.v_prefixed, test_local_pginstall / pg_version.v_prefixed + ) - LIB_DIR = PGDIR / Path("lib/postgresql") - cleanup_lib_globs = ["anon*", "postgis*", "pg_buffercache*"] - cleanup_lib_glob_paths = [LIB_DIR.glob(x) for x in cleanup_lib_globs] + neon_env_builder.pg_distrib_dir = test_local_pginstall + log.info(f"local neon_env_builder.pg_distrib_dir: {neon_env_builder.pg_distrib_dir}") - SHARE_DIR = PGDIR / Path("share/postgresql/extension") - cleanup_ext_globs = [ - "anon*", - "address_standardizer*", - "postgis*", - "pageinspect*", - "pg_buffercache*", - "pgrouting*", - ] - cleanup_ext_glob_paths = [SHARE_DIR.glob(x) for x in cleanup_ext_globs] - - all_glob_paths = cleanup_lib_glob_paths + cleanup_ext_glob_paths - all_cleanup_files = [] - for file_glob in all_glob_paths: - for file in file_glob: - all_cleanup_files.append(file) - - for file in all_cleanup_files: - try: - os.remove(file) - log.info(f"removed file {file}") - except Exception as err: - log.info( - f"skipping remove of file {file} because it doesn't exist.\ - this may be expected or unexpected depending on the test {err}" - ) - - cleanup_folders = [SHARE_DIR / Path("anon"), PGDIR / Path("download_extensions")] - for folder in cleanup_folders: - try: - shutil.rmtree(folder) - log.info(f"removed folder {folder}") - except Exception as err: - log.info( - f"skipping remove of folder {folder} because it doesn't exist.\ - this may be expected or unexpected depending on the test {err}" - ) + return neon_env_builder -def upload_files(env): - log.info("Uploading test files to mock bucket") - os.chdir("test_runner/regress/data/extension_test") - for path in os.walk("."): - prefix, _, files = path - for file in files: - # the [2:] is to remove the leading "./" - full_path = os.path.join(prefix, file)[2:] - - with open(full_path, "rb") as f: - log.info(f"UPLOAD {full_path} to ext/{full_path}") - assert isinstance(env.pageserver_remote_storage, S3Storage) - env.pageserver_remote_storage.client.upload_fileobj( - f, - env.ext_remote_storage.bucket_name, - f"ext/{full_path}", - ) - os.chdir("../../../..") - - -# Test downloading remote extension. -@skip_on_postgres(PgVersion.V16, reason="TODO: PG16 extension building") -@pytest.mark.parametrize("remote_storage_kind", available_s3_storages()) -@pytest.mark.skip(reason="https://github.com/neondatabase/neon/issues/4949") def test_remote_extensions( - neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, - pg_version: PgVersion, + httpserver: HTTPServer, + neon_env_builder_local: NeonEnvBuilder, + httpserver_listen_address, + pg_version, ): - neon_env_builder.enable_extensions_remote_storage(remote_storage_kind) - env = neon_env_builder.init_start() - tenant_id, _ = env.neon_cli.create_tenant() - env.neon_cli.create_timeline("test_remote_extensions", tenant_id=tenant_id) + if pg_version == PgVersion.V16: + pytest.skip("TODO: PG16 extension building") - assert env.ext_remote_storage is not None # satisfy mypy + # setup mock http server + # that expects request for anon.tar.zst + # and returns the requested file + (host, port) = httpserver_listen_address + extensions_endpoint = f"http://{host}:{port}/pg-ext-s3-gateway" - # For MOCK_S3 we upload test files. - # For REAL_S3 we use the files already in the bucket - if remote_storage_kind == RemoteStorageKind.MOCK_S3: - upload_files(env) + build_tag = os.environ.get("BUILD_TAG", "latest") + archive_path = f"{build_tag}/v{pg_version}/extensions/anon.tar.zst" - # Start a compute node and check that it can download the extensions - # and use them to CREATE EXTENSION and LOAD - endpoint = env.endpoints.create_start( + def endpoint_handler_build_tag(request: Request) -> Response: + log.info(f"request: {request}") + + file_name = "anon.tar.zst" + file_path = f"test_runner/regress/data/extension_test/5670669815/v{pg_version}/extensions/anon.tar.zst" + file_size = os.path.getsize(file_path) + fh = open(file_path, "rb") + + return Response( + fh, + mimetype="application/octet-stream", + headers=[ + ("Content-Length", str(file_size)), + ("Content-Disposition", 'attachment; filename="%s"' % file_name), + ], + direct_passthrough=True, + ) + + httpserver.expect_request( + f"/pg-ext-s3-gateway/{archive_path}", method="GET" + ).respond_with_handler(endpoint_handler_build_tag) + + # Start a compute node with remote_extension spec + # and check that it can download the extensions and use them to CREATE EXTENSION. + env = neon_env_builder_local.init_start() + env.neon_cli.create_branch("test_remote_extensions") + endpoint = env.endpoints.create( "test_remote_extensions", - tenant_id=tenant_id, - remote_ext_config=env.ext_remote_storage.to_string(), - # config_lines=["log_min_messages=debug3"], + config_lines=["log_min_messages=debug3"], ) + + # mock remote_extensions spec + spec: Dict[str, Any] = { + "library_index": { + "anon": "anon", + }, + "extension_data": { + "anon": { + "archive_path": "", + "control_data": { + "anon.control": "# PostgreSQL Anonymizer (anon) extension\ncomment = 'Data anonymization tools'\ndefault_version = '1.1.0'\ndirectory='extension/anon'\nrelocatable = false\nrequires = 'pgcrypto'\nsuperuser = false\nmodule_pathname = '$libdir/anon'\ntrusted = true\n" + }, + }, + }, + } + spec["extension_data"]["anon"]["archive_path"] = archive_path + + endpoint.create_remote_extension_spec(spec) + + endpoint.start( + remote_ext_config=extensions_endpoint, + ) + + # this is expected to fail if there's no pgcrypto extension, that's ok + # we just want to check that the extension was downloaded try: with closing(endpoint.connect()) as conn: with conn.cursor() as cur: - # Check that appropriate control files were downloaded - cur.execute("SELECT * FROM pg_available_extensions") - all_extensions = [x[0] for x in cur.fetchall()] - log.info(all_extensions) - assert "anon" in all_extensions + # Check that appropriate files were downloaded + cur.execute("CREATE EXTENSION anon") + res = [x[0] for x in cur.fetchall()] + log.info(res) + except Exception as err: + assert "pgcrypto" in str(err), f"unexpected error creating anon extension {err}" - # postgis is on real s3 but not mock s3. - # it's kind of a big file, would rather not upload to github - if remote_storage_kind == RemoteStorageKind.REAL_S3: - assert "postgis" in all_extensions - # this may fail locally if dependency is missing - # we don't really care about the error, - # we just want to make sure it downloaded - try: - cur.execute("CREATE EXTENSION postgis") - except Exception as err: - log.info(f"(expected) error creating postgis extension: {err}") - # we do not check the error, so this is basically a NO-OP - # however checking the log you can make sure that it worked - # and also get valuable information about how long loading the extension took - - # this is expected to fail on my computer because I don't have the pgcrypto extension - try: - cur.execute("CREATE EXTENSION anon") - except Exception as err: - log.info("error creating anon extension") - assert "pgcrypto" in str(err), "unexpected error creating anon extension" - finally: - cleanup(pg_version) + httpserver.check() -# Test downloading remote library. -@skip_on_postgres(PgVersion.V16, reason="TODO: PG16 extension building") -@pytest.mark.parametrize("remote_storage_kind", available_s3_storages()) -@pytest.mark.skip(reason="https://github.com/neondatabase/neon/issues/4949") -def test_remote_library( - neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, - pg_version: PgVersion, -): - neon_env_builder.enable_extensions_remote_storage(remote_storage_kind) - env = neon_env_builder.init_start() - tenant_id, _ = env.neon_cli.create_tenant() - env.neon_cli.create_timeline("test_remote_library", tenant_id=tenant_id) - - assert env.ext_remote_storage is not None # satisfy mypy - - # For MOCK_S3 we upload test files. - # For REAL_S3 we use the files already in the bucket - if remote_storage_kind == RemoteStorageKind.MOCK_S3: - upload_files(env) - - # and use them to run LOAD library - endpoint = env.endpoints.create_start( - "test_remote_library", - tenant_id=tenant_id, - remote_ext_config=env.ext_remote_storage.to_string(), - # config_lines=["log_min_messages=debug3"], - ) - try: - with closing(endpoint.connect()) as conn: - with conn.cursor() as cur: - # try to load library - try: - cur.execute("LOAD 'anon'") - except Exception as err: - log.info(f"error loading anon library: {err}") - raise AssertionError("unexpected error loading anon library") from err - - # test library which name is different from extension name - # this may fail locally if dependency is missing - # however, it does successfully download the postgis archive - if remote_storage_kind == RemoteStorageKind.REAL_S3: - try: - cur.execute("LOAD 'postgis_topology-3'") - except Exception as err: - log.info("error loading postgis_topology-3") - assert "No such file or directory" in str( - err - ), "unexpected error loading postgis_topology-3" - finally: - cleanup(pg_version) - - -# Here we test a complex extension -# which has multiple extensions in one archive -# using postgis as an example -# @pytest.mark.skipif( -# RemoteStorageKind.REAL_S3 not in available_s3_storages(), -# reason="skipping test because real s3 not enabled", -# ) -@skip_on_postgres(PgVersion.V16, reason="TODO: PG16 extension building") -@pytest.mark.skip(reason="https://github.com/neondatabase/neon/issues/4949") -def test_multiple_extensions_one_archive( - neon_env_builder: NeonEnvBuilder, - pg_version: PgVersion, -): - neon_env_builder.enable_extensions_remote_storage(RemoteStorageKind.REAL_S3) - env = neon_env_builder.init_start() - tenant_id, _ = env.neon_cli.create_tenant() - env.neon_cli.create_timeline("test_multiple_extensions_one_archive", tenant_id=tenant_id) - - assert env.ext_remote_storage is not None # satisfy mypy - - endpoint = env.endpoints.create_start( - "test_multiple_extensions_one_archive", - tenant_id=tenant_id, - remote_ext_config=env.ext_remote_storage.to_string(), - ) - with closing(endpoint.connect()) as conn: - with conn.cursor() as cur: - cur.execute("CREATE EXTENSION address_standardizer;") - cur.execute("CREATE EXTENSION address_standardizer_data_us;") - # execute query to ensure that it works - cur.execute( - "SELECT house_num, name, suftype, city, country, state, unit \ - FROM standardize_address('us_lex', 'us_gaz', 'us_rules', \ - 'One Rust Place, Boston, MA 02109');" - ) - res = cur.fetchall() - log.info(res) - assert len(res) > 0 - - cleanup(pg_version) - - -# Test that extension is downloaded after endpoint restart, -# when the library is used in the query. +# TODO +# 1. Test downloading remote library. # +# 2. Test a complex extension, which has multiple extensions in one archive +# using postgis as an example +# +# 3.Test that extension is downloaded after endpoint restart, +# when the library is used in the query. # Run the test with mutliple simultaneous connections to an endpoint. # to ensure that the extension is downloaded only once. # -@pytest.mark.skip(reason="https://github.com/neondatabase/neon/issues/4949") -def test_extension_download_after_restart( - neon_env_builder: NeonEnvBuilder, - pg_version: PgVersion, -): - # TODO: PG15 + PG16 extension building - if "v14" not in pg_version: # test set only has extension built for v14 - return None - - neon_env_builder.enable_extensions_remote_storage(RemoteStorageKind.MOCK_S3) - env = neon_env_builder.init_start() - tenant_id, _ = env.neon_cli.create_tenant() - env.neon_cli.create_timeline("test_extension_download_after_restart", tenant_id=tenant_id) - - assert env.ext_remote_storage is not None # satisfy mypy - - # For MOCK_S3 we upload test files. - upload_files(env) - - endpoint = env.endpoints.create_start( - "test_extension_download_after_restart", - tenant_id=tenant_id, - remote_ext_config=env.ext_remote_storage.to_string(), - config_lines=["log_min_messages=debug3"], - ) - with closing(endpoint.connect()) as conn: - with conn.cursor() as cur: - cur.execute("CREATE extension pg_buffercache;") - cur.execute("SELECT * from pg_buffercache;") - res = cur.fetchall() - assert len(res) > 0 - log.info(res) - - # shutdown compute node - endpoint.stop() - # remove extension files locally - cleanup(pg_version) - - # spin up compute node again (there are no extension files available, because compute is stateless) - endpoint = env.endpoints.create_start( - "test_extension_download_after_restart", - tenant_id=tenant_id, - remote_ext_config=env.ext_remote_storage.to_string(), - config_lines=["log_min_messages=debug3"], - ) - - # connect to compute node and run the query - # that will trigger the download of the extension - def run_query(endpoint, thread_id: int): - log.info("thread_id {%d} starting", thread_id) - with closing(endpoint.connect()) as conn: - with conn.cursor() as cur: - cur.execute("SELECT * from pg_buffercache;") - res = cur.fetchall() - assert len(res) > 0 - log.info("thread_id {%d}, res = %s", thread_id, res) - - threads = [threading.Thread(target=run_query, args=(endpoint, i)) for i in range(2)] - - for thread in threads: - thread.start() - for thread in threads: - thread.join() - - cleanup(pg_version) +# 4. Test that private extensions are only downloaded when they are present in the spec. +# diff --git a/test_runner/regress/test_layers_from_future.py b/test_runner/regress/test_layers_from_future.py new file mode 100644 index 0000000000..b36c927628 --- /dev/null +++ b/test_runner/regress/test_layers_from_future.py @@ -0,0 +1,222 @@ +import time + +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnvBuilder +from fixtures.pageserver.types import ( + DeltaLayerFileName, + ImageLayerFileName, + is_future_layer, +) +from fixtures.pageserver.utils import ( + wait_for_last_record_lsn, + wait_for_upload_queue_empty, + wait_until_tenant_active, +) +from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind +from fixtures.types import Lsn +from fixtures.utils import query_scalar, wait_until + + +def test_issue_5878(neon_env_builder: NeonEnvBuilder): + """ + Regression test for issue https://github.com/neondatabase/neon/issues/5878 . + + Create a situation where IndexPart contains an image layer from a future + (i.e., image layer > IndexPart::disk_consistent_lsn). + Detach. + Attach. + Wait for tenant to finish load_layer_map (by waiting for it to become active). + Wait for any remote timeline client ops to finish that the attach started. + Integrity-check the index part. + + Before fixing the issue, load_layer_map would schedule removal of the future + image layer. A compaction run could later re-create the image layer with + the same file name, scheduling a PUT. + Due to lack of an upload queue barrier, the PUT and DELETE could be re-ordered. + The result was IndexPart referencing a non-existent object. + """ + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) + + env = neon_env_builder.init_start() + + ps_http = env.pageserver.http_client() + + l0_l1_threshold = 3 + image_creation_threshold = 1 + + tenant_config = { + "gc_period": "0s", # disable GC (shouldn't matter for this test but still) + "compaction_period": "0s", # we want to control when compaction runs + "checkpoint_timeout": "24h", # something we won't reach + "checkpoint_distance": f"{50 * (1024**2)}", # something we won't reach, we checkpoint manually + "image_creation_threshold": f"{image_creation_threshold}", + "compaction_threshold": f"{l0_l1_threshold}", + "compaction_target_size": f"{128 * (1024**3)}", # make it so that we only have 1 partition => image coverage for delta layers => enables gc of delta layers + } + + tenant_id, timeline_id = env.neon_cli.create_tenant(conf=tenant_config) + + endpoint = env.endpoints.create_start("main", tenant_id=tenant_id) + + def get_index_part(): + assert isinstance(env.pageserver_remote_storage, LocalFsStorage) + ip_path = env.pageserver_remote_storage.index_path(tenant_id, timeline_id) + return env.pagectl.dump_index_part(ip_path) + + def get_future_layers(): + ip = get_index_part() + future_layers = [ + layer_file_name + for layer_file_name in ip.layer_metadata.keys() + if is_future_layer(layer_file_name, ip.disk_consistent_lsn) + ] + return future_layers + + assert len(get_future_layers()) == 0 + + current = get_index_part() + assert len(set(current.layer_metadata.keys())) == 1 + layer_file_name = list(current.layer_metadata.keys())[0] + assert isinstance(layer_file_name, DeltaLayerFileName) + assert layer_file_name.is_l0(), f"{layer_file_name}" + + log.info("force image layer creation in the future by writing some data into in-memory layer") + + # Create a number of layers in the tenant + with endpoint.cursor() as cur: + cur.execute("CREATE TABLE foo (t text)") + iters = l0_l1_threshold * image_creation_threshold + for i in range(0, iters): + cur.execute( + f""" + INSERT INTO foo + SELECT '{i}' || g + FROM generate_series(1, 10000) g + """ + ) + last_record_lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()")) + wait_for_last_record_lsn(ps_http, tenant_id, timeline_id, last_record_lsn) + # 0..iters-1: create a stack of delta layers + # iters: leave a non-empty in-memory layer which we'll use for image layer generation + if i < iters - 1: + ps_http.timeline_checkpoint(tenant_id, timeline_id, force_repartition=True) + assert ( + len( + [ + layer + for layer in ps_http.layer_map_info( + tenant_id, timeline_id + ).historic_layers + if layer.kind == "Image" + ] + ) + == 0 + ) + + endpoint.stop() + + wait_for_upload_queue_empty(ps_http, tenant_id, timeline_id) + + ip = get_index_part() + assert len(ip.layer_metadata.keys()) + assert ( + ip.disk_consistent_lsn < last_record_lsn + ), "sanity check for what above loop is supposed to do" + + # create the image layer from the future + ps_http.timeline_compact(tenant_id, timeline_id, force_repartition=True) + assert ( + len( + [ + layer + for layer in ps_http.layer_map_info(tenant_id, timeline_id).historic_layers + if layer.kind == "Image" + ] + ) + == 1 + ) + wait_for_upload_queue_empty(ps_http, tenant_id, timeline_id) + future_layers = get_future_layers() + assert len(future_layers) == 1 + future_layer = future_layers[0] + assert isinstance(future_layer, ImageLayerFileName) + assert future_layer.lsn == last_record_lsn + log.info( + f"got layer from the future: lsn={future_layer.lsn} disk_consistent_lsn={ip.disk_consistent_lsn} last_record_lsn={last_record_lsn}" + ) + assert isinstance(env.pageserver_remote_storage, LocalFsStorage) + future_layer_path = env.pageserver_remote_storage.layer_path( + tenant_id, timeline_id, future_layer + ) + log.info(f"future layer path: {future_layer_path}") + pre_stat = future_layer_path.stat() + time.sleep(1.1) # so that we can use change in pre_stat.st_mtime to detect overwrites + + # force removal of layers from the future + tenant_conf = ps_http.tenant_config(tenant_id) + ps_http.tenant_detach(tenant_id) + failpoint_name = "before-delete-layer-pausable" + ps_http.configure_failpoints((failpoint_name, "pause")) + ps_http.tenant_attach(tenant_id, tenant_conf.tenant_specific_overrides) + wait_until_tenant_active(ps_http, tenant_id) + + # Ensure the IndexPart upload that unlinks the layer file finishes, i.e., doesn't clog the queue. + def future_layer_is_gone_from_index_part(): + future_layers = set(get_future_layers()) + assert future_layer not in future_layers + + wait_until(10, 0.5, future_layer_is_gone_from_index_part) + + # NB: the layer file is unlinked index part now, but, because we made the delete + # operation stuck, the layer file itself is still in the remote_storage + def delete_at_pause_point(): + assert env.pageserver.log_contains(f".*{tenant_id}.*at failpoint.*{failpoint_name}") + + wait_until(10, 0.5, delete_at_pause_point) + assert future_layer_path.exists() + + # wait for re-ingestion of the WAL from safekeepers into the in-memory layer + # (this happens in parallel to the above) + wait_for_last_record_lsn(ps_http, tenant_id, timeline_id, last_record_lsn) + + # re-do image layer generation + # This will produce the same image layer and queue an upload. + # However, we still have the deletion for the layer queued, stuck on the failpoint. + # An incorrect implementation would let the PUT execute before the DELETE. + # The later code in this test asserts that this doesn't happen. + ps_http.timeline_compact(tenant_id, timeline_id, force_repartition=True) + + # Let things sit for some time; a good implementation makes no progress because + # we can't execute the PUT before the DELETE. A bad implementation would do that. + max_race_opportunity_window = 4 + start = time.monotonic() + while True: + post_stat = future_layer_path.stat() + assert ( + pre_stat.st_mtime == post_stat.st_mtime + ), "observed PUT overtake the stucked DELETE => bug isn't fixed yet" + if time.monotonic() - start > max_race_opportunity_window: + log.info( + "a correct implementation would never let the later PUT overtake the earlier DELETE" + ) + break + time.sleep(1) + + # Window has passed, unstuck the delete, let upload queue drain. + log.info("unstuck the DELETE") + ps_http.configure_failpoints(("before-delete-layer-pausable", "off")) + + wait_for_upload_queue_empty(ps_http, tenant_id, timeline_id) + + # Examine the resulting S3 state. + log.info("integrity-check the remote storage") + ip = get_index_part() + for layer_file_name in ip.layer_metadata.keys(): + layer_path = env.pageserver_remote_storage.layer_path( + tenant_id, timeline_id, layer_file_name + ) + assert layer_path.exists(), f"{layer_file_name.to_str()}" + + log.info("assert that the overwritten layer won") + final_stat = future_layer_path.stat() + assert final_stat.st_mtime != pre_stat.st_mtime diff --git a/test_runner/regress/test_local_file_cache.py b/test_runner/regress/test_local_file_cache.py new file mode 100644 index 0000000000..38f2034c18 --- /dev/null +++ b/test_runner/regress/test_local_file_cache.py @@ -0,0 +1,74 @@ +import os +import random +import threading +import time +from typing import List + +from fixtures.neon_fixtures import NeonEnv +from fixtures.utils import query_scalar + + +def test_local_file_cache_unlink(neon_simple_env: NeonEnv): + env = neon_simple_env + + cache_dir = os.path.join(env.repo_dir, "file_cache") + os.mkdir(cache_dir) + + env.neon_cli.create_branch("test_local_file_cache_unlink", "empty") + + endpoint = env.endpoints.create_start( + "test_local_file_cache_unlink", + config_lines=[ + "shared_buffers='1MB'", + f"neon.file_cache_path='{cache_dir}/file.cache'", + "neon.max_file_cache_size='64MB'", + "neon.file_cache_size_limit='10MB'", + ], + ) + + cur = endpoint.connect().cursor() + + n_rows = 100000 + n_threads = 20 + n_updates_per_thread = 10000 + n_updates_per_connection = 1000 + n_total_updates = n_threads * n_updates_per_thread + + cur.execute("CREATE TABLE lfctest (id int4 PRIMARY KEY, n int) WITH (fillfactor=10)") + cur.execute(f"INSERT INTO lfctest SELECT g, 1 FROM generate_series(1, {n_rows}) g") + + # Start threads that will perform random UPDATEs. Each UPDATE + # increments the counter on the row, so that we can check at the + # end that the sum of all the counters match the number of updates + # performed (plus the initial 1 on each row). + # + # Furthermore, each thread will reconnect between every 1000 updates. + def run_updates(): + n_updates_performed = 0 + conn = endpoint.connect() + cur = conn.cursor() + for _ in range(n_updates_per_thread): + id = random.randint(1, n_rows) + cur.execute(f"UPDATE lfctest SET n = n + 1 WHERE id = {id}") + n_updates_performed += 1 + if n_updates_performed % n_updates_per_connection == 0: + cur.close() + conn.close() + conn = endpoint.connect() + cur = conn.cursor() + + threads: List[threading.Thread] = [] + for _i in range(n_threads): + thread = threading.Thread(target=run_updates, args=(), daemon=True) + thread.start() + threads.append(thread) + + time.sleep(5) + + new_cache_dir = os.path.join(env.repo_dir, "file_cache_new") + os.rename(cache_dir, new_cache_dir) + + for thread in threads: + thread.join() + + assert query_scalar(cur, "SELECT SUM(n) FROM lfctest") == n_total_updates + n_rows diff --git a/test_runner/regress/test_neon_extension.py b/test_runner/regress/test_neon_extension.py new file mode 100644 index 0000000000..998f84f968 --- /dev/null +++ b/test_runner/regress/test_neon_extension.py @@ -0,0 +1,28 @@ +from contextlib import closing + +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnvBuilder + + +# Verify that the neon extension is installed and has the correct version. +def test_neon_extension(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + env.neon_cli.create_branch("test_create_extension_neon") + + endpoint_main = env.endpoints.create("test_create_extension_neon") + # don't skip pg_catalog updates - it runs CREATE EXTENSION neon + endpoint_main.respec(skip_pg_catalog_updates=False) + endpoint_main.start() + + log.info("postgres is running on 'test_create_extension_neon' branch") + + with closing(endpoint_main.connect()) as conn: + with conn.cursor() as cur: + cur.execute("SELECT extversion from pg_extension where extname='neon'") + # If this fails, it means the extension is either not installed + # or was updated and the version is different. + # + # IMPORTANT: + # If the version has changed, the test should be updated. + # Ensure that the default version is also updated in the neon.control file + assert cur.fetchone() == ("1.1",) diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index c3f4ad476f..66cc286aba 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -282,7 +282,7 @@ def test_deferred_deletion(neon_env_builder: NeonEnvBuilder): # Now advance the generation in the control plane: subsequent validations # from the running pageserver will fail. No more deletions should happen. - env.attachment_service.attach_hook(env.initial_tenant, some_other_pageserver) + env.attachment_service.attach_hook_issue(env.initial_tenant, some_other_pageserver) generate_uploads_and_deletions(env, init=False) assert_deletion_queue(ps_http, lambda n: n > 0) @@ -397,7 +397,7 @@ def test_deletion_queue_recovery( if keep_attachment == KeepAttachment.LOSE: some_other_pageserver = 101010 assert env.attachment_service is not None - env.attachment_service.attach_hook(env.initial_tenant, some_other_pageserver) + env.attachment_service.attach_hook_issue(env.initial_tenant, some_other_pageserver) env.pageserver.start() diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index 31bc97703e..4da8dd957d 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -763,9 +763,7 @@ def test_compaction_waits_for_upload( neon_env_builder: NeonEnvBuilder, ): """ - Compaction waits for outstanding uploads to complete, so that it avoids deleting layers - files that have not yet been uploaded. This test forces a race between upload and - compaction. + This test forces a race between upload and compaction. """ neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) @@ -784,6 +782,16 @@ def test_compaction_waits_for_upload( timeline_id = env.initial_timeline client = env.pageserver.http_client() + layers_at_creation = client.layer_map_info(tenant_id, timeline_id) + deltas_at_creation = len(layers_at_creation.delta_layers()) + assert ( + deltas_at_creation == 1 + ), "are you fixing #5863? make sure we end up with 2 deltas at the end of endpoint lifecycle" + + # Make new layer uploads get stuck. + # Note that timeline creation waits for the initial layers to reach remote storage. + # So at this point, the `layers_at_creation` are in remote storage. + client.configure_failpoints(("before-upload-layer-pausable", "pause")) with env.endpoints.create_start("main", tenant_id=tenant_id) as endpoint: # Build two tables with some data inside @@ -791,85 +799,71 @@ def test_compaction_waits_for_upload( wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) client.timeline_checkpoint(tenant_id, timeline_id) + deltas_at_first = len(client.layer_map_info(tenant_id, timeline_id).delta_layers()) + assert ( + deltas_at_first == 2 + ), "are you fixing #5863? just add one more checkpoint after 'CREATE TABLE bar ...' statement." endpoint.safe_psql("CREATE TABLE bar AS SELECT x FROM generate_series(1, 10000) g(x)") - wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) - - # Now make the flushing hang and update one small piece of data - client.configure_failpoints(("before-upload-layer-pausable", "pause")) - endpoint.safe_psql("UPDATE foo SET x = 0 WHERE x = 1") - wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) - checkpoint_result: queue.Queue[Optional[PageserverApiException]] = queue.Queue() - compact_result: queue.Queue[Optional[PageserverApiException]] = queue.Queue() - compact_barrier = threading.Barrier(2) + layers_before_last_checkpoint = client.layer_map_info(tenant_id, timeline_id).historic_by_name() + upload_stuck_layers = layers_before_last_checkpoint - layers_at_creation.historic_by_name() - def checkpoint_in_background(): - try: - log.info("Checkpoint starting") - client.timeline_checkpoint(tenant_id, timeline_id) - log.info("Checkpoint complete") - checkpoint_result.put(None) - except PageserverApiException as e: - log.info("Checkpoint errored: {e}") - checkpoint_result.put(e) + assert len(upload_stuck_layers) > 0 - def compact_in_background(): - compact_barrier.wait() - try: - log.info("Compaction starting") - client.timeline_compact(tenant_id, timeline_id) - log.info("Compaction complete") - compact_result.put(None) - except PageserverApiException as e: - log.info("Compaction errored: {e}") - compact_result.put(e) + for name in upload_stuck_layers: + path = env.pageserver.timeline_dir(tenant_id, timeline_id) / name + assert path.exists(), "while uploads are stuck the layers should be present on disk" - checkpoint_thread = threading.Thread(target=checkpoint_in_background) - checkpoint_thread.start() + # now this will do the L0 => L1 compaction and want to remove + # upload_stuck_layers and the original initdb L0 + client.timeline_checkpoint(tenant_id, timeline_id) - compact_thread = threading.Thread(target=compact_in_background) - compact_thread.start() + # as uploads are paused, the the upload_stuck_layers should still be with us + for name in upload_stuck_layers: + path = env.pageserver.timeline_dir(tenant_id, timeline_id) / name + assert path.exists(), "uploads are stuck still over compaction" - try: - # Start the checkpoint, see that it blocks - log.info("Waiting to see checkpoint hang...") - time.sleep(5) - assert checkpoint_result.empty() + compacted_layers = client.layer_map_info(tenant_id, timeline_id).historic_by_name() + overlap = compacted_layers.intersection(upload_stuck_layers) + assert len(overlap) == 0, "none of the L0's should remain after L0 => L1 compaction" + assert ( + len(compacted_layers) == 1 + ), "there should be one L1 after L0 => L1 compaction (without #5863 being fixed)" - # Start the compaction, see that it finds work to do but blocks - compact_barrier.wait() - log.info("Waiting to see compaction hang...") - time.sleep(5) - assert compact_result.empty() + def layer_deletes_completed(): + m = client.get_metric_value("pageserver_layer_gcs_count_total", {"state": "completed"}) + if m is None: + return 0 + return int(m) - # This is logged once compaction is started, but before we wait for operations to complete - assert env.pageserver.log_contains("compact_level0_phase1 stats available.") + # if initdb created an initial delta layer, it might already be gc'd + # because it was uploaded before the failpoint was enabled. however, the + # deletion is not guaranteed to be complete. + assert layer_deletes_completed() <= 1 - # Once we unblock uploads the compaction should complete successfully - log.info("Disabling failpoint") - client.configure_failpoints(("before-upload-layer-pausable", "off")) - log.info("Awaiting compaction result") - assert compact_result.get(timeout=10) is None - log.info("Awaiting checkpoint result") - assert checkpoint_result.get(timeout=10) is None - - except Exception: - # Log the actual failure's backtrace here, before we proceed to join threads - log.exception("Failure, cleaning up...") - raise - finally: - compact_barrier.abort() - - checkpoint_thread.join() - compact_thread.join() + client.configure_failpoints(("before-upload-layer-pausable", "off")) # Ensure that this actually terminates wait_upload_queue_empty(client, tenant_id, timeline_id) - # We should not have hit the error handling path in uploads where the remote file is gone + def until_layer_deletes_completed(): + deletes = layer_deletes_completed() + log.info(f"layer_deletes: {deletes}") + # ensure that initdb delta layer AND the previously stuck are now deleted + assert deletes >= len(upload_stuck_layers) + 1 + + wait_until(10, 1, until_layer_deletes_completed) + + for name in upload_stuck_layers: + path = env.pageserver.timeline_dir(tenant_id, timeline_id) / name + assert ( + not path.exists() + ), "l0 should now be removed because of L0 => L1 compaction and completed uploads" + + # We should not have hit the error handling path in uploads where a uploaded file is gone assert not env.pageserver.log_contains( "File to upload doesn't exist. Likely the file has been deleted and an upload is not required any more." ) diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index 2fdcfca671..fcc3243e81 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -336,10 +336,15 @@ def test_live_reconfig_get_evictions_low_residence_duration_metric_threshold( ): neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) - env = neon_env_builder.init_start() + env = neon_env_builder.init_start( + initial_tenant_conf={ + # disable compaction so that it will not download the layer for repartitioning + "compaction_period": "0s" + } + ) assert isinstance(env.pageserver_remote_storage, LocalFsStorage) - (tenant_id, timeline_id) = env.neon_cli.create_tenant() + (tenant_id, timeline_id) = env.initial_tenant, env.initial_timeline ps_http = env.pageserver.http_client() def get_metric(): diff --git a/test_runner/regress/test_tenant_delete.py b/test_runner/regress/test_tenant_delete.py index 6e35890922..0dd1f9a295 100644 --- a/test_runner/regress/test_tenant_delete.py +++ b/test_runner/regress/test_tenant_delete.py @@ -1,6 +1,7 @@ import enum import os import shutil +from threading import Thread import pytest from fixtures.log_helper import log @@ -27,7 +28,7 @@ from fixtures.remote_storage import ( available_s3_storages, ) from fixtures.types import TenantId -from fixtures.utils import run_pg_bench_small +from fixtures.utils import run_pg_bench_small, wait_until @pytest.mark.parametrize("remote_storage_kind", available_remote_storages()) @@ -399,4 +400,78 @@ def test_tenant_delete_is_resumed_on_attach( ) +def test_long_timeline_create_cancelled_by_tenant_delete(neon_env_builder: NeonEnvBuilder): + """Reproduction of 2023-11-23 stuck tenants investigation""" + + # do not use default tenant/timeline creation because it would output the failpoint log message too early + env = neon_env_builder.init_configs() + env.start() + pageserver_http = env.pageserver.http_client() + + # happens with the cancellation bailing flushing loop earlier, leaving disk_consistent_lsn at zero + env.pageserver.allowed_errors.append( + ".*Timeline got dropped without initializing, cleaning its files" + ) + # the response hit_pausable_failpoint_and_later_fail + env.pageserver.allowed_errors.append( + f".*Error processing HTTP request: InternalServerError\\(new timeline {env.initial_tenant}/{env.initial_timeline} has invalid disk_consistent_lsn" + ) + + pageserver_http.tenant_create(env.initial_tenant) + + failpoint = "flush-layer-cancel-after-writing-layer-out-pausable" + pageserver_http.configure_failpoints((failpoint, "pause")) + + def hit_pausable_failpoint_and_later_fail(): + with pytest.raises( + PageserverApiException, match="new timeline \\S+ has invalid disk_consistent_lsn" + ): + pageserver_http.timeline_create( + env.pg_version, env.initial_tenant, env.initial_timeline + ) + + def start_deletion(): + pageserver_http.tenant_delete(env.initial_tenant) + + def has_hit_failpoint(): + assert env.pageserver.log_contains(f"at failpoint {failpoint}") is not None + + def deletion_has_started_waiting_for_timelines(): + assert env.pageserver.log_contains("Waiting for timelines...") is not None + + def tenant_is_deleted(): + try: + pageserver_http.tenant_status(env.initial_tenant) + except PageserverApiException as e: + assert e.status_code == 404 + else: + raise RuntimeError("tenant was still accessible") + + creation = Thread(target=hit_pausable_failpoint_and_later_fail) + creation.start() + + deletion = None + + try: + wait_until(10, 1, has_hit_failpoint) + + # it should start ok, sync up with the stuck creation, then fail because disk_consistent_lsn was not updated + # then deletion should fail and set the tenant broken + deletion = Thread(target=start_deletion) + deletion.start() + + wait_until(10, 1, deletion_has_started_waiting_for_timelines) + + pageserver_http.configure_failpoints((failpoint, "off")) + + creation.join() + deletion.join() + + wait_until(10, 1, tenant_is_deleted) + finally: + creation.join() + if deletion is not None: + deletion.join() + + # TODO test concurrent deletions with "hang" failpoint diff --git a/test_runner/regress/test_timeline_size.py b/test_runner/regress/test_timeline_size.py index c2e93c48c7..eb98348823 100644 --- a/test_runner/regress/test_timeline_size.py +++ b/test_runner/regress/test_timeline_size.py @@ -134,10 +134,11 @@ def wait_for_pageserver_catchup(endpoint_main: Endpoint, polling_interval=1, tim res = endpoint_main.safe_psql( """ SELECT - pg_size_pretty(pg_cluster_size()), + pg_size_pretty(neon.pg_cluster_size()), pg_wal_lsn_diff(pg_current_wal_flush_lsn(), received_lsn) as received_lsn_lag - FROM backpressure_lsns(); - """ + FROM neon.backpressure_lsns(); + """, + dbname="postgres", )[0] log.info(f"pg_cluster_size = {res[0]}, received_lsn_lag = {res[1]}") received_lsn_lag = res[1] @@ -152,17 +153,20 @@ def test_timeline_size_quota(neon_env_builder: NeonEnvBuilder): wait_for_timeline_size_init(client, tenant=env.initial_tenant, timeline=new_timeline_id) - endpoint_main = env.endpoints.create_start( + endpoint_main = env.endpoints.create( "test_timeline_size_quota", # Set small limit for the test config_lines=["neon.max_cluster_size=30MB"], ) + # don't skip pg_catalog updates - it runs CREATE EXTENSION neon + # which is needed for pg_cluster_size() to work + endpoint_main.respec(skip_pg_catalog_updates=False) + endpoint_main.start() + log.info("postgres is running on 'test_timeline_size_quota' branch") with closing(endpoint_main.connect()) as conn: with conn.cursor() as cur: - cur.execute("CREATE EXTENSION neon") # TODO move it to neon_fixtures? - cur.execute("CREATE TABLE foo (t text)") wait_for_pageserver_catchup(endpoint_main) @@ -211,7 +215,7 @@ def test_timeline_size_quota(neon_env_builder: NeonEnvBuilder): wait_for_pageserver_catchup(endpoint_main) - cur.execute("SELECT * from pg_size_pretty(pg_cluster_size())") + cur.execute("SELECT * from pg_size_pretty(neon.pg_cluster_size())") pg_cluster_size = cur.fetchone() log.info(f"pg_cluster_size = {pg_cluster_size}") diff --git a/test_runner/regress/test_wal_restore.py b/test_runner/regress/test_wal_restore.py index a4f03be7a0..b039b36255 100644 --- a/test_runner/regress/test_wal_restore.py +++ b/test_runner/regress/test_wal_restore.py @@ -1,14 +1,19 @@ import sys +import tarfile +import tempfile from pathlib import Path import pytest +import zstandard +from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, PgBin, VanillaPostgres, ) from fixtures.port_distributor import PortDistributor -from fixtures.types import TenantId, TimelineId +from fixtures.remote_storage import LocalFsStorage +from fixtures.types import Lsn, TenantId, TimelineId @pytest.mark.skipif( @@ -53,3 +58,70 @@ def test_wal_restore( ) restored.start() assert restored.safe_psql("select count(*) from t", user="cloud_admin") == [(300000,)] + + +def decompress_zstd( + input_file_name: Path, + output_dir: Path, +): + log.info(f"decompressing zstd to: {output_dir}") + output_dir.mkdir(mode=0o750, parents=True, exist_ok=True) + with tempfile.TemporaryFile(suffix=".tar") as temp: + decompressor = zstandard.ZstdDecompressor() + with open(input_file_name, "rb") as input_file: + decompressor.copy_stream(input_file, temp) + temp.seek(0) + with tarfile.open(fileobj=temp) as tfile: + tfile.extractall(path=output_dir) + + +def test_wal_restore_initdb( + neon_env_builder: NeonEnvBuilder, + pg_bin: PgBin, + test_output_dir: Path, + port_distributor: PortDistributor, + base_dir: Path, + pg_distrib_dir: Path, +): + env = neon_env_builder.init_start() + endpoint = env.endpoints.create_start("main") + endpoint.safe_psql("create table t as select generate_series(1,300000)") + tenant_id = env.initial_tenant + timeline_id = env.initial_timeline + original_lsn = Lsn(endpoint.safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0]) + env.pageserver.stop() + port = port_distributor.get_port() + data_dir = test_output_dir / "pgsql.restored" + + assert isinstance(env.pageserver_remote_storage, LocalFsStorage) + + initdb_zst_path = ( + env.pageserver_remote_storage.timeline_path(tenant_id, timeline_id) / "initdb.tar.zst" + ) + + decompress_zstd(initdb_zst_path, data_dir) + with VanillaPostgres( + data_dir, PgBin(test_output_dir, env.pg_distrib_dir, env.pg_version), port, init=False + ) as restored: + pg_bin.run_capture( + [ + str(base_dir / "libs" / "utils" / "scripts" / "restore_from_wal_initdb.sh"), + str(pg_distrib_dir / f"v{env.pg_version}/bin"), + str( + test_output_dir + / "repo" + / "safekeepers" + / "sk1" + / str(tenant_id) + / str(timeline_id) + ), + str(data_dir), + str(port), + ] + ) + restored.start() + restored_lsn = Lsn( + restored.safe_psql("SELECT pg_current_wal_flush_lsn()", user="cloud_admin")[0][0] + ) + log.info(f"original lsn: {original_lsn}, restored lsn: {restored_lsn}") + assert restored.safe_psql("select count(*) from t", user="cloud_admin") == [(300000,)] diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 3535c2cf94..66828fb53d 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -68,6 +68,9 @@ tracing-core = { version = "0.1" } tungstenite = { version = "0.20" } url = { version = "2", features = ["serde"] } uuid = { version = "1", features = ["serde", "v4"] } +zstd = { version = "0.12" } +zstd-safe = { version = "6", default-features = false, features = ["arrays", "legacy", "std", "zdict_builder"] } +zstd-sys = { version = "2", default-features = false, features = ["legacy", "std", "zdict_builder"] } [build-dependencies] anyhow = { version = "1", features = ["backtrace"] }