diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index a41258c401..3f1c728c70 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -320,6 +320,9 @@ jobs: - name: Build neon extensions run: mold -run make neon-pg-ext -j$(nproc) + - name: Build walproposer-lib + run: mold -run make walproposer-lib -j$(nproc) + - name: Run cargo build run: | ${cov_prefix} mold -run cargo build $CARGO_FLAGS $CARGO_FEATURES --bins --tests @@ -834,7 +837,7 @@ jobs: run: shell: sh -eu {0} env: - VM_BUILDER_VERSION: v0.17.12 + VM_BUILDER_VERSION: v0.18.1 steps: - name: Checkout diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 8a1e4571fd..d7f5295c5b 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -32,7 +32,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 @@ -90,18 +90,21 @@ jobs: - name: Build postgres v14 if: steps.cache_pg_14.outputs.cache-hit != 'true' - run: make postgres-v14 -j$(nproc) + run: make postgres-v14 -j$(sysctl -n hw.ncpu) - name: Build postgres v15 if: steps.cache_pg_15.outputs.cache-hit != 'true' - run: make postgres-v15 -j$(nproc) + run: make postgres-v15 -j$(sysctl -n hw.ncpu) - name: Build postgres v16 if: steps.cache_pg_16.outputs.cache-hit != 'true' - run: make postgres-v16 -j$(nproc) + run: make postgres-v16 -j$(sysctl -n hw.ncpu) - name: Build neon extensions - run: make neon-pg-ext -j$(nproc) + run: make neon-pg-ext -j$(sysctl -n hw.ncpu) + + - name: Build walproposer-lib + run: make walproposer-lib -j$(sysctl -n hw.ncpu) - name: Run cargo build run: cargo build --all --release @@ -126,7 +129,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: submodules: true fetch-depth: 1 @@ -135,6 +138,9 @@ jobs: - name: Get postgres headers run: make postgres-headers -j$(nproc) + - name: Build walproposer-lib + run: make walproposer-lib -j$(nproc) + - name: Produce the build stats run: cargo build --all --release --timings diff --git a/Cargo.lock b/Cargo.lock index aacf4e53d7..f7598a79cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -285,7 +285,7 @@ dependencies = [ "log", "parking", "polling", - "rustix 0.37.19", + "rustix 0.37.25", "slab", "socket2 0.4.9", "waker-fn", @@ -853,6 +853,27 @@ dependencies = [ "uuid", ] +[[package]] +name = "azure_identity" +version = "0.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b67b337346da8739e91ea1e9400a6ebc9bc54e0b2af1d23c9bcd565950588f9" +dependencies = [ + "async-lock", + "async-trait", + "azure_core", + "futures", + "log", + "oauth2", + "pin-project", + "serde", + "serde_json", + "time 0.3.21", + "tz-rs", + "url", + "uuid", +] + [[package]] name = "azure_storage" version = "0.16.0" @@ -1131,9 +1152,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e3c5919066adf22df73762e50cffcde3a758f2a848b113b586d1f86728b673b" dependencies = [ "iana-time-zone", + "js-sys", "num-integer", "num-traits", "serde", + "wasm-bindgen", "winapi", ] @@ -2559,7 +2582,7 @@ checksum = "adcf93614601c8129ddf72e2d5633df827ba6551541c6d8c59520a371475be1f" dependencies = [ "hermit-abi", "io-lifetimes", - "rustix 0.37.19", + "rustix 0.37.25", "windows-sys 0.48.0", ] @@ -2949,6 +2972,34 @@ dependencies = [ "libc", ] +[[package]] +name = "num_threads" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +dependencies = [ + "libc", +] + +[[package]] +name = "oauth2" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c38841cdd844847e3e7c8d29cef9dcfed8877f8f56f9071f77843ecf3baf937f" +dependencies = [ + "base64 0.13.1", + "chrono", + "getrandom 0.2.9", + "http", + "rand 0.8.5", + "serde", + "serde_json", + "serde_path_to_error", + "sha2 0.10.6", + "thiserror", + "url", +] + [[package]] name = "object" version = "0.30.3" @@ -3510,7 +3561,7 @@ dependencies = [ [[package]] name = "postgres" version = "0.19.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "bytes", "fallible-iterator", @@ -3523,7 +3574,7 @@ dependencies = [ [[package]] name = "postgres-native-tls" version = "0.5.0" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "native-tls", "tokio", @@ -3534,7 +3585,7 @@ dependencies = [ [[package]] name = "postgres-protocol" version = "0.6.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "base64 0.20.0", "byteorder", @@ -3552,7 +3603,7 @@ dependencies = [ [[package]] name = "postgres-types" version = "0.2.4" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "bytes", "fallible-iterator", @@ -3683,7 +3734,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix 0.36.14", + "rustix 0.36.16", ] [[package]] @@ -4021,6 +4072,7 @@ dependencies = [ "aws-smithy-http", "aws-types", "azure_core", + "azure_identity", "azure_storage", "azure_storage_blobs", "bytes", @@ -4265,9 +4317,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.14" +version = "0.36.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" +checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab" dependencies = [ "bitflags", "errno", @@ -4279,9 +4331,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.37.19" +version = "0.37.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acf8729d8542766f1b2cf77eb034d52f40d375bb8b615d0b147089946e16613d" +checksum = "d4eb579851244c2c03e7c24f501c3432bed80b8f720af1d6e5b0e0f01555a035" dependencies = [ "bitflags", "errno", @@ -5122,7 +5174,7 @@ dependencies = [ "cfg-if", "fastrand 1.9.0", "redox_syscall 0.3.5", - "rustix 0.37.19", + "rustix 0.37.25", "windows-sys 0.45.0", ] @@ -5219,6 +5271,8 @@ checksum = "8f3403384eaacbca9923fa06940178ac13e4edb725486d70e8e15881d0c836cc" dependencies = [ "itoa", "js-sys", + "libc", + "num_threads", "serde", "time-core", "time-macros 0.2.9", @@ -5353,7 +5407,7 @@ dependencies = [ [[package]] name = "tokio-postgres" version = "0.7.7" -source = "git+https://github.com/neondatabase/rust-postgres.git?rev=9011f7110db12b5e15afaf98f8ac834501d50ddc#9011f7110db12b5e15afaf98f8ac834501d50ddc" +source = "git+https://github.com/neondatabase/rust-postgres.git?rev=7434d9388965a17a6d113e5dfc0e65666a03b4c2#7434d9388965a17a6d113e5dfc0e65666a03b4c2" dependencies = [ "async-trait", "byteorder", @@ -5368,7 +5422,7 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2 0.4.9", + "socket2 0.5.3", "tokio", "tokio-util", ] @@ -5770,6 +5824,15 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +[[package]] +name = "tz-rs" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33851b15c848fad2cf4b105c6bb66eb9512b6f6c44a4b13f57c53c73c707e2b4" +dependencies = [ + "const_fn", +] + [[package]] name = "uname" version = "0.1.1" @@ -6029,6 +6092,17 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "walproposer" +version = "0.1.0" +dependencies = [ + "anyhow", + "bindgen", + "postgres_ffi", + "utils", + "workspace_hack", +] + [[package]] name = "want" version = "0.3.0" @@ -6434,7 +6508,6 @@ dependencies = [ "serde", "serde_json", "smallvec", - "socket2 0.4.9", "standback", "syn 1.0.109", "syn 2.0.28", diff --git a/Cargo.toml b/Cargo.toml index 4827652229..a0be7bb9ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,6 +26,7 @@ members = [ "libs/tracing-utils", "libs/postgres_ffi/wal_craft", "libs/vm_monitor", + "libs/walproposer", ] [workspace.package] @@ -36,9 +37,10 @@ license = "Apache-2.0" [workspace.dependencies] anyhow = { version = "1.0", features = ["backtrace"] } async-compression = { version = "0.4.0", features = ["tokio", "gzip"] } -azure_core = "0.16.0" +azure_core = "0.16" +azure_identity = "0.16" azure_storage = "0.16" -azure_storage_blobs = "0.16.0" +azure_storage_blobs = "0.16" flate2 = "1.0.26" async-stream = "0.3" async-trait = "0.1" @@ -159,11 +161,11 @@ env_logger = "0.10" log = "0.4" ## Libraries from neondatabase/ git forks, ideally with changes to be upstreamed -postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } +postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres-native-tls = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } ## Other git libraries heapless = { default-features=false, features=[], git = "https://github.com/japaric/heapless.git", rev = "644653bf3b831c6bb4963be2de24804acf5e5001" } # upstream release pending @@ -184,6 +186,7 @@ tenant_size_model = { version = "0.1", path = "./libs/tenant_size_model/" } tracing-utils = { version = "0.1", path = "./libs/tracing-utils/" } utils = { version = "0.1", path = "./libs/utils/" } vm_monitor = { version = "0.1", path = "./libs/vm_monitor/" } +walproposer = { version = "0.1", path = "./libs/walproposer/" } ## Common library dependency workspace_hack = { version = "0.1", path = "./workspace_hack/" } @@ -199,7 +202,7 @@ tonic-build = "0.9" # This is only needed for proxy's tests. # TODO: we should probably fork `tokio-postgres-rustls` instead. -tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="9011f7110db12b5e15afaf98f8ac834501d50ddc" } +tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", rev="7434d9388965a17a6d113e5dfc0e65666a03b4c2" } ################# Binary contents sections diff --git a/Makefile b/Makefile index 33b5dcad99..64bbc1677c 100644 --- a/Makefile +++ b/Makefile @@ -62,7 +62,7 @@ all: neon postgres neon-pg-ext # # The 'postgres_ffi' depends on the Postgres headers. .PHONY: neon -neon: postgres-headers +neon: postgres-headers walproposer-lib +@echo "Compiling Neon" $(CARGO_CMD_PREFIX) cargo build $(CARGO_BUILD_FLAGS) @@ -168,6 +168,42 @@ neon-pg-ext-clean-%: -C $(POSTGRES_INSTALL_DIR)/build/neon-utils-$* \ -f $(ROOT_PROJECT_DIR)/pgxn/neon_utils/Makefile clean +# Build walproposer as a static library. walproposer source code is located +# in the pgxn/neon directory. +# +# We also need to include libpgport.a and libpgcommon.a, because walproposer +# uses some functions from those libraries. +# +# Some object files are removed from libpgport.a and libpgcommon.a because +# they depend on openssl and other libraries that are not included in our +# Rust build. +.PHONY: walproposer-lib +walproposer-lib: neon-pg-ext-v16 + +@echo "Compiling walproposer-lib" + mkdir -p $(POSTGRES_INSTALL_DIR)/build/walproposer-lib + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v16/bin/pg_config CFLAGS='$(PG_CFLAGS) $(COPT)' \ + -C $(POSTGRES_INSTALL_DIR)/build/walproposer-lib \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile walproposer-lib + cp $(POSTGRES_INSTALL_DIR)/v16/lib/libpgport.a $(POSTGRES_INSTALL_DIR)/build/walproposer-lib + cp $(POSTGRES_INSTALL_DIR)/v16/lib/libpgcommon.a $(POSTGRES_INSTALL_DIR)/build/walproposer-lib +ifeq ($(UNAME_S),Linux) + $(AR) d $(POSTGRES_INSTALL_DIR)/build/walproposer-lib/libpgport.a \ + pg_strong_random.o + $(AR) d $(POSTGRES_INSTALL_DIR)/build/walproposer-lib/libpgcommon.a \ + pg_crc32c.o \ + hmac_openssl.o \ + cryptohash_openssl.o \ + scram-common.o \ + md5_common.o \ + checksum_helper.o +endif + +.PHONY: walproposer-lib-clean +walproposer-lib-clean: + $(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/v16/bin/pg_config \ + -C $(POSTGRES_INSTALL_DIR)/build/walproposer-lib \ + -f $(ROOT_PROJECT_DIR)/pgxn/neon/Makefile clean + .PHONY: neon-pg-ext neon-pg-ext: \ neon-pg-ext-v14 \ diff --git a/NOTICE b/NOTICE index 4fbec9763b..c13dc2f0b3 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Neon Copyright 2022 Neon Inc. -The PostgreSQL submodules in vendor/postgres-v14 and vendor/postgres-v15 are licensed under the -PostgreSQL license. See vendor/postgres-v14/COPYRIGHT and vendor/postgres-v15/COPYRIGHT. +The PostgreSQL submodules in vendor/ are licensed under the PostgreSQL license. +See vendor/postgres-vX/COPYRIGHT for details. diff --git a/compute_tools/src/compute.rs b/compute_tools/src/compute.rs index e67430d061..9fd88e5818 100644 --- a/compute_tools/src/compute.rs +++ b/compute_tools/src/compute.rs @@ -252,7 +252,7 @@ fn create_neon_superuser(spec: &ComputeSpec, client: &mut Client) -> Result<()> IF NOT EXISTS ( SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser') THEN - CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN IN ROLE pg_read_all_data, pg_write_all_data; + CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN REPLICATION IN ROLE pg_read_all_data, pg_write_all_data; IF array_length(roles, 1) IS NOT NULL THEN EXECUTE format('GRANT neon_superuser TO %s', array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(roles) as x), ', ')); diff --git a/compute_tools/src/spec.rs b/compute_tools/src/spec.rs index 591cbe90c0..f7ca2eb33c 100644 --- a/compute_tools/src/spec.rs +++ b/compute_tools/src/spec.rs @@ -302,7 +302,7 @@ pub fn handle_roles(spec: &ComputeSpec, client: &mut Client) -> Result<()> { } RoleAction::Create => { let mut query: String = format!( - "CREATE ROLE {} CREATEROLE CREATEDB BYPASSRLS IN ROLE neon_superuser", + "CREATE ROLE {} CREATEROLE CREATEDB BYPASSRLS REPLICATION IN ROLE neon_superuser", name.pg_quote() ); info!("role create query: '{}'", &query); diff --git a/compute_tools/tests/pg_helpers_tests.rs b/compute_tools/tests/pg_helpers_tests.rs index 7d27d22a78..4961bc293d 100644 --- a/compute_tools/tests/pg_helpers_tests.rs +++ b/compute_tools/tests/pg_helpers_tests.rs @@ -28,7 +28,7 @@ mod pg_helpers_tests { assert_eq!( spec.cluster.settings.as_pg_settings(), r#"fsync = off -wal_level = replica +wal_level = logical hot_standby = on neon.safekeepers = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501' wal_log_hints = on diff --git a/control_plane/src/bin/attachment_service.rs b/control_plane/src/bin/attachment_service.rs index d4bca59c7b..65ed78d20b 100644 --- a/control_plane/src/bin/attachment_service.rs +++ b/control_plane/src/bin/attachment_service.rs @@ -13,6 +13,7 @@ use serde::{Deserialize, Serialize}; use std::path::{Path, PathBuf}; use std::{collections::HashMap, sync::Arc}; use utils::logging::{self, LogFormat}; +use utils::signals::{ShutdownSignals, Signal}; use utils::{ http::{ @@ -268,7 +269,16 @@ async fn main() -> anyhow::Result<()> { let server = hyper::Server::from_tcp(http_listener)?.serve(service); tracing::info!("Serving on {0}", args.listen); - server.await?; + + tokio::task::spawn(server); + + ShutdownSignals::handle(|signal| match signal { + Signal::Interrupt | Signal::Terminate | Signal::Quit => { + tracing::info!("Got {}. Terminating", signal.name()); + // We're just a test helper: no graceful shutdown. + std::process::exit(0); + } + })?; Ok(()) } diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index cba364c049..acd9061664 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -253,7 +253,7 @@ impl Endpoint { conf.append("shared_buffers", "1MB"); conf.append("fsync", "off"); conf.append("max_connections", "100"); - conf.append("wal_level", "replica"); + conf.append("wal_level", "logical"); // wal_sender_timeout is the maximum time to wait for WAL replication. // It also defines how often the walreciever will send a feedback message to the wal sender. conf.append("wal_sender_timeout", "5s"); diff --git a/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json b/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json index 565e5e368e..ccf0a91b90 100644 --- a/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json +++ b/docker-compose/compute_wrapper/var/db/postgres/specs/spec.json @@ -25,7 +25,7 @@ }, { "name": "wal_level", - "value": "replica", + "value": "logical", "vartype": "enum" }, { diff --git a/docs/error-handling.md b/docs/error-handling.md index c2231274a1..531c80906b 100644 --- a/docs/error-handling.md +++ b/docs/error-handling.md @@ -188,11 +188,60 @@ that. ## Error message style +### PostgreSQL extensions + PostgreSQL has a style guide for writing error messages: https://www.postgresql.org/docs/current/error-style-guide.html Follow that guide when writing error messages in the PostgreSQL -extension. We don't follow it strictly in the pageserver and -safekeeper, but the advice in the PostgreSQL style guide is generally -good, and you can't go wrong by following it. +extensions. + +### Neon Rust code + +#### Anyhow Context + +When adding anyhow `context()`, use form `present-tense-verb+action`. + +Example: +- Bad: `file.metadata().context("could not get file metadata")?;` +- Good: `file.metadata().context("get file metadata")?;` + +#### Logging Errors + +When logging any error `e`, use `could not {e:#}` or `failed to {e:#}`. + +If `e` is an `anyhow` error and you want to log the backtrace that it contains, +use `{e:?}` instead of `{e:#}`. + +#### Rationale + +The `{:#}` ("alternate Display") of an `anyhow` error chain is concatenation fo the contexts, using `: `. + +For example, the following Rust code will result in output +``` +ERROR failed to list users: load users from server: parse response: invalid json +``` + +This is more concise / less noisy than what happens if you do `.context("could not ...")?` at each level, i.e.: + +``` +ERROR could not list users: could not load users from server: could not parse response: invalid json +``` + + +```rust +fn main() { + match list_users().context("list users") else { + Ok(_) => ..., + Err(e) => tracing::error!("failed to {e:#}"), + } +} +fn list_users() { + http_get_users().context("load users from server")?; +} +fn http_get_users() { + let response = client....?; + response.parse().context("parse response")?; // fails with serde error "invalid json" +} +``` diff --git a/libs/compute_api/tests/cluster_spec.json b/libs/compute_api/tests/cluster_spec.json index 96db13a5da..e2afa17ef0 100644 --- a/libs/compute_api/tests/cluster_spec.json +++ b/libs/compute_api/tests/cluster_spec.json @@ -76,7 +76,7 @@ }, { "name": "wal_level", - "value": "replica", + "value": "logical", "vartype": "enum" }, { diff --git a/libs/metrics/src/wrappers.rs b/libs/metrics/src/wrappers.rs index 1bf1ea0753..c3959cbf16 100644 --- a/libs/metrics/src/wrappers.rs +++ b/libs/metrics/src/wrappers.rs @@ -1,6 +1,6 @@ use std::io::{Read, Result, Write}; -/// A wrapper for an object implementing [Read](std::io::Read) +/// A wrapper for an object implementing [Read] /// which allows a closure to observe the amount of bytes read. /// This is useful in conjunction with metrics (e.g. [IntCounter](crate::IntCounter)). /// @@ -51,17 +51,17 @@ impl<'a, T> CountedReader<'a, T> { } } - /// Get an immutable reference to the underlying [Read](std::io::Read) implementor + /// Get an immutable reference to the underlying [Read] implementor pub fn inner(&self) -> &T { &self.reader } - /// Get a mutable reference to the underlying [Read](std::io::Read) implementor + /// Get a mutable reference to the underlying [Read] implementor pub fn inner_mut(&mut self) -> &mut T { &mut self.reader } - /// Consume the wrapper and return the underlying [Read](std::io::Read) implementor + /// Consume the wrapper and return the underlying [Read] implementor pub fn into_inner(self) -> T { self.reader } @@ -75,7 +75,7 @@ impl Read for CountedReader<'_, T> { } } -/// A wrapper for an object implementing [Write](std::io::Write) +/// A wrapper for an object implementing [Write] /// which allows a closure to observe the amount of bytes written. /// This is useful in conjunction with metrics (e.g. [IntCounter](crate::IntCounter)). /// @@ -122,17 +122,17 @@ impl<'a, T> CountedWriter<'a, T> { } } - /// Get an immutable reference to the underlying [Write](std::io::Write) implementor + /// Get an immutable reference to the underlying [Write] implementor pub fn inner(&self) -> &T { &self.writer } - /// Get a mutable reference to the underlying [Write](std::io::Write) implementor + /// Get a mutable reference to the underlying [Write] implementor pub fn inner_mut(&mut self) -> &mut T { &mut self.writer } - /// Consume the wrapper and return the underlying [Write](std::io::Write) implementor + /// Consume the wrapper and return the underlying [Write] implementor pub fn into_inner(self) -> T { self.writer } diff --git a/libs/postgres_backend/src/lib.rs b/libs/postgres_backend/src/lib.rs index 08c4e03d13..8ce7f85481 100644 --- a/libs/postgres_backend/src/lib.rs +++ b/libs/postgres_backend/src/lib.rs @@ -19,8 +19,8 @@ use tracing::{debug, error, info, trace}; use pq_proto::framed::{ConnectionError, Framed, FramedReader, FramedWriter}; use pq_proto::{ - BeMessage, FeMessage, FeStartupPacket, ProtocolError, SQLSTATE_INTERNAL_ERROR, - SQLSTATE_SUCCESSFUL_COMPLETION, + BeMessage, FeMessage, FeStartupPacket, ProtocolError, SQLSTATE_ADMIN_SHUTDOWN, + SQLSTATE_INTERNAL_ERROR, SQLSTATE_SUCCESSFUL_COMPLETION, }; /// An error, occurred during query processing: @@ -30,6 +30,9 @@ pub enum QueryError { /// The connection was lost while processing the query. #[error(transparent)] Disconnected(#[from] ConnectionError), + /// We were instructed to shutdown while processing the query + #[error("Shutting down")] + Shutdown, /// Some other error #[error(transparent)] Other(#[from] anyhow::Error), @@ -44,7 +47,8 @@ impl From for QueryError { impl QueryError { pub fn pg_error_code(&self) -> &'static [u8; 5] { match self { - Self::Disconnected(_) => b"08006", // connection failure + Self::Disconnected(_) => b"08006", // connection failure + Self::Shutdown => SQLSTATE_ADMIN_SHUTDOWN, Self::Other(_) => SQLSTATE_INTERNAL_ERROR, // internal error } } @@ -396,7 +400,20 @@ impl PostgresBackend { // socket might be already closed, e.g. if previously received error, // so ignore result. self.framed.shutdown().await.ok(); - ret + match ret { + Ok(()) => Ok(()), + Err(QueryError::Shutdown) => { + info!("Stopped due to shutdown"); + Ok(()) + } + Err(QueryError::Disconnected(e)) => { + info!("Disconnected ({e:#})"); + // Disconnection is not an error: we just use it that way internally to drop + // out of loops. + Ok(()) + } + e => e, + } } async fn run_message_loop( @@ -416,15 +433,11 @@ impl PostgresBackend { _ = shutdown_watcher() => { // We were requested to shut down. tracing::info!("shutdown request received during handshake"); - return Ok(()) + return Err(QueryError::Shutdown) }, - result = self.handshake(handler) => { - // Handshake complete. - result?; - if self.state == ProtoState::Closed { - return Ok(()); // EOF during handshake - } + handshake_r = self.handshake(handler) => { + handshake_r?; } ); @@ -435,7 +448,7 @@ impl PostgresBackend { _ = shutdown_watcher() => { // We were requested to shut down. tracing::info!("shutdown request received in run_message_loop"); - Ok(None) + return Err(QueryError::Shutdown) }, msg = self.read_message() => { msg }, )? { @@ -447,7 +460,14 @@ impl PostgresBackend { _ = shutdown_watcher() => { // We were requested to shut down. tracing::info!("shutdown request received during response flush"); - return Ok(()) + + // If we exited process_message with a shutdown error, there may be + // some valid response content on in our transmit buffer: permit sending + // this within a short timeout. This is a best effort thing so we don't + // care about the result. + tokio::time::timeout(std::time::Duration::from_millis(500), self.flush()).await.ok(); + + return Err(QueryError::Shutdown) }, flush_r = self.flush() => { flush_r?; @@ -560,7 +580,9 @@ impl PostgresBackend { self.peer_addr ); self.state = ProtoState::Closed; - return Ok(()); + return Err(QueryError::Disconnected(ConnectionError::Protocol( + ProtocolError::Protocol("EOF during handshake".to_string()), + ))); } } } @@ -599,7 +621,9 @@ impl PostgresBackend { self.peer_addr ); self.state = ProtoState::Closed; - return Ok(()); + return Err(QueryError::Disconnected(ConnectionError::Protocol( + ProtocolError::Protocol("EOF during auth".to_string()), + ))); } } } @@ -923,6 +947,7 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> AsyncWrite for CopyDataWriter<'a, I pub fn short_error(e: &QueryError) -> String { match e { QueryError::Disconnected(connection_error) => connection_error.to_string(), + QueryError::Shutdown => "shutdown".to_string(), QueryError::Other(e) => format!("{e:#}"), } } @@ -939,6 +964,9 @@ fn log_query_error(query: &str, e: &QueryError) { QueryError::Disconnected(other_connection_error) => { error!("query handler for '{query}' failed with connection error: {other_connection_error:?}") } + QueryError::Shutdown => { + info!("query handler for '{query}' cancelled during tenant shutdown") + } QueryError::Other(e) => { error!("query handler for '{query}' failed: {e:?}"); } diff --git a/libs/postgres_ffi/src/lib.rs b/libs/postgres_ffi/src/lib.rs index c9e5df9f04..d0009d0dc9 100644 --- a/libs/postgres_ffi/src/lib.rs +++ b/libs/postgres_ffi/src/lib.rs @@ -131,6 +131,7 @@ pub const MAX_SEND_SIZE: usize = XLOG_BLCKSZ * 16; // Export some version independent functions that are used outside of this mod pub use v14::xlog_utils::encode_logical_message; +pub use v14::xlog_utils::from_pg_timestamp; pub use v14::xlog_utils::get_current_timestamp; pub use v14::xlog_utils::to_pg_timestamp; pub use v14::xlog_utils::XLogFileName; diff --git a/libs/postgres_ffi/src/pg_constants.rs b/libs/postgres_ffi/src/pg_constants.rs index 9690dc0eb6..d59e0e4a15 100644 --- a/libs/postgres_ffi/src/pg_constants.rs +++ b/libs/postgres_ffi/src/pg_constants.rs @@ -220,6 +220,10 @@ pub const XLOG_CHECKPOINT_ONLINE: u8 = 0x10; pub const XLP_FIRST_IS_CONTRECORD: u16 = 0x0001; pub const XLP_LONG_HEADER: u16 = 0x0002; +/* From replication/slot.h */ +pub const REPL_SLOT_ON_DISK_OFFSETOF_RESTART_LSN: usize = 4*4 /* offset of `slotdata` in ReplicationSlotOnDisk */ + + 64 /* NameData */ + 4*4; + /* From fsm_internals.h */ const FSM_NODES_PER_PAGE: usize = BLCKSZ as usize - SIZEOF_PAGE_HEADER_DATA - 4; const FSM_NON_LEAF_NODES_PER_PAGE: usize = BLCKSZ as usize / 2 - 1; diff --git a/libs/postgres_ffi/src/xlog_utils.rs b/libs/postgres_ffi/src/xlog_utils.rs index 61a9c38a84..0ca9bd8b45 100644 --- a/libs/postgres_ffi/src/xlog_utils.rs +++ b/libs/postgres_ffi/src/xlog_utils.rs @@ -136,21 +136,42 @@ pub fn get_current_timestamp() -> TimestampTz { to_pg_timestamp(SystemTime::now()) } -pub fn to_pg_timestamp(time: SystemTime) -> TimestampTz { - const UNIX_EPOCH_JDATE: u64 = 2440588; /* == date2j(1970, 1, 1) */ - const POSTGRES_EPOCH_JDATE: u64 = 2451545; /* == date2j(2000, 1, 1) */ +// Module to reduce the scope of the constants +mod timestamp_conversions { + use std::time::Duration; + + use super::*; + + const UNIX_EPOCH_JDATE: u64 = 2440588; // == date2j(1970, 1, 1) + const POSTGRES_EPOCH_JDATE: u64 = 2451545; // == date2j(2000, 1, 1) const SECS_PER_DAY: u64 = 86400; const USECS_PER_SEC: u64 = 1000000; - match time.duration_since(SystemTime::UNIX_EPOCH) { - Ok(n) => { - ((n.as_secs() - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)) - * USECS_PER_SEC - + n.subsec_micros() as u64) as i64 + const SECS_DIFF_UNIX_TO_POSTGRES_EPOCH: u64 = + (POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY; + + pub fn to_pg_timestamp(time: SystemTime) -> TimestampTz { + match time.duration_since(SystemTime::UNIX_EPOCH) { + Ok(n) => { + ((n.as_secs() - SECS_DIFF_UNIX_TO_POSTGRES_EPOCH) * USECS_PER_SEC + + n.subsec_micros() as u64) as i64 + } + Err(_) => panic!("SystemTime before UNIX EPOCH!"), } - Err(_) => panic!("SystemTime before UNIX EPOCH!"), + } + + pub fn from_pg_timestamp(time: TimestampTz) -> SystemTime { + let time: u64 = time + .try_into() + .expect("timestamp before millenium (postgres epoch)"); + let since_unix_epoch = time + SECS_DIFF_UNIX_TO_POSTGRES_EPOCH * USECS_PER_SEC; + SystemTime::UNIX_EPOCH + .checked_add(Duration::from_micros(since_unix_epoch)) + .expect("SystemTime overflow") } } +pub use timestamp_conversions::{from_pg_timestamp, to_pg_timestamp}; + // Returns (aligned) end_lsn of the last record in data_dir with WAL segments. // start_lsn must point to some previously known record boundary (beginning of // the next record). If no valid record after is found, start_lsn is returned @@ -481,4 +502,24 @@ pub fn encode_logical_message(prefix: &str, message: &str) -> Vec { wal } -// If you need to craft WAL and write tests for this module, put it at wal_craft crate. +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ts_conversion() { + let now = SystemTime::now(); + let round_trip = from_pg_timestamp(to_pg_timestamp(now)); + + let now_since = now.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + let round_trip_since = round_trip.duration_since(SystemTime::UNIX_EPOCH).unwrap(); + assert_eq!(now_since.as_micros(), round_trip_since.as_micros()); + + let now_pg = get_current_timestamp(); + let round_trip_pg = to_pg_timestamp(from_pg_timestamp(now_pg)); + + assert_eq!(now_pg, round_trip_pg); + } + + // If you need to craft WAL and write tests for this module, put it at wal_craft crate. +} diff --git a/libs/pq_proto/src/lib.rs b/libs/pq_proto/src/lib.rs index 47faad363f..94bc7f60f8 100644 --- a/libs/pq_proto/src/lib.rs +++ b/libs/pq_proto/src/lib.rs @@ -670,6 +670,7 @@ pub fn read_cstr(buf: &mut Bytes) -> Result { } pub const SQLSTATE_INTERNAL_ERROR: &[u8; 5] = b"XX000"; +pub const SQLSTATE_ADMIN_SHUTDOWN: &[u8; 5] = b"57P01"; pub const SQLSTATE_SUCCESSFUL_COMPLETION: &[u8; 5] = b"00000"; impl<'a> BeMessage<'a> { diff --git a/libs/remote_storage/Cargo.toml b/libs/remote_storage/Cargo.toml index 5d3bda70af..65b5392389 100644 --- a/libs/remote_storage/Cargo.toml +++ b/libs/remote_storage/Cargo.toml @@ -28,6 +28,7 @@ utils.workspace = true pin-project-lite.workspace = true workspace_hack.workspace = true azure_core.workspace = true +azure_identity.workspace = true azure_storage.workspace = true azure_storage_blobs.workspace = true futures-util.workspace = true diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index 4f7c8ce6f4..152929ecd3 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -1,12 +1,15 @@ //! Azure Blob Storage wrapper +use std::env; use std::num::NonZeroU32; +use std::sync::Arc; use std::{borrow::Cow, collections::HashMap, io::Cursor}; use super::REMOTE_STORAGE_PREFIX_SEPARATOR; use anyhow::Result; use azure_core::request_options::{MaxResults, Metadata, Range}; use azure_core::Header; +use azure_identity::DefaultAzureCredential; use azure_storage::StorageCredentials; use azure_storage_blobs::prelude::ClientBuilder; use azure_storage_blobs::{ @@ -38,12 +41,16 @@ impl AzureBlobStorage { azure_config.container_name ); - let account = - std::env::var("AZURE_STORAGE_ACCOUNT").expect("missing AZURE_STORAGE_ACCOUNT"); - let access_key = - std::env::var("AZURE_STORAGE_ACCESS_KEY").expect("missing AZURE_STORAGE_ACCESS_KEY"); + let account = env::var("AZURE_STORAGE_ACCOUNT").expect("missing AZURE_STORAGE_ACCOUNT"); - let credentials = StorageCredentials::access_key(account.clone(), access_key); + // If the `AZURE_STORAGE_ACCESS_KEY` env var has an access key, use that, + // otherwise try the token based credentials. + let credentials = if let Ok(access_key) = env::var("AZURE_STORAGE_ACCESS_KEY") { + StorageCredentials::access_key(account.clone(), access_key) + } else { + let token_credential = DefaultAzureCredential::default(); + StorageCredentials::token_credential(Arc::new(token_credential)) + }; let builder = ClientBuilder::new(account, credentials); @@ -114,22 +121,7 @@ impl AzureBlobStorage { // https://github.com/neondatabase/neon/issues/5563 let mut buf = Vec::new(); while let Some(part) = response.next().await { - let part = match part { - Ok(l) => l, - Err(e) => { - return Err(if let Some(http_err) = e.as_http_error() { - match http_err.status() { - StatusCode::NotFound => DownloadError::NotFound, - StatusCode::BadRequest => { - DownloadError::BadInput(anyhow::Error::new(e)) - } - _ => DownloadError::Other(anyhow::Error::new(e)), - } - } else { - DownloadError::Other(e.into()) - }); - } - }; + let part = part.map_err(to_download_error)?; let data = part .data .collect() @@ -150,30 +142,16 @@ impl AzureBlobStorage { ) -> Result { let builder = blob_client.get_metadata(); - match builder.into_future().await { - Ok(r) => { - let mut map = HashMap::new(); + let response = builder.into_future().await.map_err(to_download_error)?; + let mut map = HashMap::new(); - for md in r.metadata.iter() { - map.insert( - md.name().as_str().to_string(), - md.value().as_str().to_string(), - ); - } - Ok(StorageMetadata(map)) - } - Err(e) => { - return Err(if let Some(http_err) = e.as_http_error() { - match http_err.status() { - StatusCode::NotFound => DownloadError::NotFound, - StatusCode::BadRequest => DownloadError::BadInput(anyhow::Error::new(e)), - _ => DownloadError::Other(anyhow::Error::new(e)), - } - } else { - DownloadError::Other(e.into()) - }); - } + for md in response.metadata.iter() { + map.insert( + md.name().as_str().to_string(), + md.value().as_str().to_string(), + ); } + Ok(StorageMetadata(map)) } async fn permit(&self, kind: RequestKind) -> tokio::sync::SemaphorePermit<'_> { @@ -192,6 +170,18 @@ fn to_azure_metadata(metadata: StorageMetadata) -> Metadata { res } +fn to_download_error(error: azure_core::Error) -> DownloadError { + if let Some(http_err) = error.as_http_error() { + match http_err.status() { + StatusCode::NotFound => DownloadError::NotFound, + StatusCode::BadRequest => DownloadError::BadInput(anyhow::Error::new(error)), + _ => DownloadError::Other(anyhow::Error::new(error)), + } + } else { + DownloadError::Other(error.into()) + } +} + #[async_trait::async_trait] impl RemoteStorage for AzureBlobStorage { async fn list_prefixes( @@ -226,23 +216,8 @@ impl RemoteStorage for AzureBlobStorage { let mut response = builder.into_stream(); let mut res = Vec::new(); - while let Some(l) = response.next().await { - let entry = match l { - Ok(l) => l, - Err(e) => { - return Err(if let Some(http_err) = e.as_http_error() { - match http_err.status() { - StatusCode::NotFound => DownloadError::NotFound, - StatusCode::BadRequest => { - DownloadError::BadInput(anyhow::Error::new(e)) - } - _ => DownloadError::Other(anyhow::Error::new(e)), - } - } else { - DownloadError::Other(e.into()) - }); - } - }; + while let Some(entry) = response.next().await { + let entry = entry.map_err(to_download_error)?; let name_iter = entry .blobs .prefixes() diff --git a/libs/remote_storage/tests/test_real_azure.rs b/libs/remote_storage/tests/test_real_azure.rs index 5ebbd9e95b..0338270aaf 100644 --- a/libs/remote_storage/tests/test_real_azure.rs +++ b/libs/remote_storage/tests/test_real_azure.rs @@ -267,6 +267,12 @@ async fn azure_upload_download_works(ctx: &mut MaybeEnabledAzure) -> anyhow::Res let buf = download_and_compare(dl).await?; assert_eq!(buf, data); + debug!("Cleanup: deleting file at path {path:?}"); + ctx.client + .delete(&path) + .await + .with_context(|| format!("{path:?} removal"))?; + Ok(()) } diff --git a/libs/walproposer/Cargo.toml b/libs/walproposer/Cargo.toml new file mode 100644 index 0000000000..73aa073c44 --- /dev/null +++ b/libs/walproposer/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "walproposer" +version = "0.1.0" +edition.workspace = true +license.workspace = true + +[dependencies] +anyhow.workspace = true +utils.workspace = true +postgres_ffi.workspace = true + +workspace_hack.workspace = true + +[build-dependencies] +anyhow.workspace = true +bindgen.workspace = true diff --git a/libs/walproposer/bindgen_deps.h b/libs/walproposer/bindgen_deps.h new file mode 100644 index 0000000000..b95788347c --- /dev/null +++ b/libs/walproposer/bindgen_deps.h @@ -0,0 +1 @@ +#include "walproposer.h" diff --git a/libs/walproposer/build.rs b/libs/walproposer/build.rs new file mode 100644 index 0000000000..d32c8ab299 --- /dev/null +++ b/libs/walproposer/build.rs @@ -0,0 +1,113 @@ +use std::{env, path::PathBuf, process::Command}; + +use anyhow::{anyhow, Context}; +use bindgen::CargoCallbacks; + +fn main() -> anyhow::Result<()> { + // Tell cargo to invalidate the built crate whenever the wrapper changes + println!("cargo:rerun-if-changed=bindgen_deps.h"); + + // Finding the location of built libraries and Postgres C headers: + // - if POSTGRES_INSTALL_DIR is set look into it, otherwise look into `/pg_install` + // - if there's a `bin/pg_config` file use it for getting include server, otherwise use `/pg_install/{PG_MAJORVERSION}/include/postgresql/server` + let pg_install_dir = if let Some(postgres_install_dir) = env::var_os("POSTGRES_INSTALL_DIR") { + postgres_install_dir.into() + } else { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../pg_install") + }; + + let pg_install_abs = std::fs::canonicalize(pg_install_dir)?; + let walproposer_lib_dir = pg_install_abs.join("build/walproposer-lib"); + let walproposer_lib_search_str = walproposer_lib_dir + .to_str() + .ok_or(anyhow!("Bad non-UTF path"))?; + + let pgxn_neon = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("../../pgxn/neon"); + let pgxn_neon = std::fs::canonicalize(pgxn_neon)?; + let pgxn_neon = pgxn_neon.to_str().ok_or(anyhow!("Bad non-UTF path"))?; + + println!("cargo:rustc-link-lib=static=pgport"); + println!("cargo:rustc-link-lib=static=pgcommon"); + println!("cargo:rustc-link-lib=static=walproposer"); + println!("cargo:rustc-link-search={walproposer_lib_search_str}"); + + let pg_config_bin = pg_install_abs.join("v16").join("bin").join("pg_config"); + let inc_server_path: String = if pg_config_bin.exists() { + let output = Command::new(pg_config_bin) + .arg("--includedir-server") + .output() + .context("failed to execute `pg_config --includedir-server`")?; + + if !output.status.success() { + panic!("`pg_config --includedir-server` failed") + } + + String::from_utf8(output.stdout) + .context("pg_config output is not UTF-8")? + .trim_end() + .into() + } else { + let server_path = pg_install_abs + .join("v16") + .join("include") + .join("postgresql") + .join("server") + .into_os_string(); + server_path + .into_string() + .map_err(|s| anyhow!("Bad postgres server path {s:?}"))? + }; + + // The bindgen::Builder is the main entry point + // to bindgen, and lets you build up options for + // the resulting bindings. + let bindings = bindgen::Builder::default() + // The input header we would like to generate + // bindings for. + .header("bindgen_deps.h") + // Tell cargo to invalidate the built crate whenever any of the + // included header files changed. + .parse_callbacks(Box::new(CargoCallbacks)) + .allowlist_type("WalProposer") + .allowlist_type("WalProposerConfig") + .allowlist_type("walproposer_api") + .allowlist_function("WalProposerCreate") + .allowlist_function("WalProposerStart") + .allowlist_function("WalProposerBroadcast") + .allowlist_function("WalProposerPoll") + .allowlist_function("WalProposerFree") + .allowlist_var("DEBUG5") + .allowlist_var("DEBUG4") + .allowlist_var("DEBUG3") + .allowlist_var("DEBUG2") + .allowlist_var("DEBUG1") + .allowlist_var("LOG") + .allowlist_var("INFO") + .allowlist_var("NOTICE") + .allowlist_var("WARNING") + .allowlist_var("ERROR") + .allowlist_var("FATAL") + .allowlist_var("PANIC") + .allowlist_var("WPEVENT") + .allowlist_var("WL_LATCH_SET") + .allowlist_var("WL_SOCKET_READABLE") + .allowlist_var("WL_SOCKET_WRITEABLE") + .allowlist_var("WL_TIMEOUT") + .allowlist_var("WL_SOCKET_CLOSED") + .allowlist_var("WL_SOCKET_MASK") + .clang_arg("-DWALPROPOSER_LIB") + .clang_arg(format!("-I{pgxn_neon}")) + .clang_arg(format!("-I{inc_server_path}")) + // Finish the builder and generate the bindings. + .generate() + // Unwrap the Result and panic on failure. + .expect("Unable to generate bindings"); + + // Write the bindings to the $OUT_DIR/bindings.rs file. + let out_path = PathBuf::from(env::var("OUT_DIR").unwrap()).join("bindings.rs"); + bindings + .write_to_file(out_path) + .expect("Couldn't write bindings!"); + + Ok(()) +} diff --git a/libs/walproposer/src/api_bindings.rs b/libs/walproposer/src/api_bindings.rs new file mode 100644 index 0000000000..b0ac2a879e --- /dev/null +++ b/libs/walproposer/src/api_bindings.rs @@ -0,0 +1,455 @@ +#![allow(dead_code)] + +use std::ffi::CStr; +use std::ffi::CString; + +use crate::bindings::uint32; +use crate::bindings::walproposer_api; +use crate::bindings::PGAsyncReadResult; +use crate::bindings::PGAsyncWriteResult; +use crate::bindings::Safekeeper; +use crate::bindings::Size; +use crate::bindings::StringInfoData; +use crate::bindings::TimeLineID; +use crate::bindings::TimestampTz; +use crate::bindings::WalProposer; +use crate::bindings::WalProposerConnStatusType; +use crate::bindings::WalProposerConnectPollStatusType; +use crate::bindings::WalProposerExecStatusType; +use crate::bindings::WalproposerShmemState; +use crate::bindings::XLogRecPtr; +use crate::walproposer::ApiImpl; +use crate::walproposer::WaitResult; + +extern "C" fn get_shmem_state(wp: *mut WalProposer) -> *mut WalproposerShmemState { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_shmem_state() + } +} + +extern "C" fn start_streaming(wp: *mut WalProposer, startpos: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).start_streaming(startpos) + } +} + +extern "C" fn get_flush_rec_ptr(wp: *mut WalProposer) -> XLogRecPtr { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_flush_rec_ptr() + } +} + +extern "C" fn get_current_timestamp(wp: *mut WalProposer) -> TimestampTz { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_current_timestamp() + } +} + +extern "C" fn conn_error_message(sk: *mut Safekeeper) -> *mut ::std::os::raw::c_char { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + let msg = (*api).conn_error_message(&mut (*sk)); + let msg = CString::new(msg).unwrap(); + // TODO: fix leaking error message + msg.into_raw() + } +} + +extern "C" fn conn_status(sk: *mut Safekeeper) -> WalProposerConnStatusType { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_status(&mut (*sk)) + } +} + +extern "C" fn conn_connect_start(sk: *mut Safekeeper) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_connect_start(&mut (*sk)) + } +} + +extern "C" fn conn_connect_poll(sk: *mut Safekeeper) -> WalProposerConnectPollStatusType { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_connect_poll(&mut (*sk)) + } +} + +extern "C" fn conn_send_query(sk: *mut Safekeeper, query: *mut ::std::os::raw::c_char) -> bool { + let query = unsafe { CStr::from_ptr(query) }; + let query = query.to_str().unwrap(); + + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_send_query(&mut (*sk), query) + } +} + +extern "C" fn conn_get_query_result(sk: *mut Safekeeper) -> WalProposerExecStatusType { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_get_query_result(&mut (*sk)) + } +} + +extern "C" fn conn_flush(sk: *mut Safekeeper) -> ::std::os::raw::c_int { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_flush(&mut (*sk)) + } +} + +extern "C" fn conn_finish(sk: *mut Safekeeper) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_finish(&mut (*sk)) + } +} + +extern "C" fn conn_async_read( + sk: *mut Safekeeper, + buf: *mut *mut ::std::os::raw::c_char, + amount: *mut ::std::os::raw::c_int, +) -> PGAsyncReadResult { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + let (res, result) = (*api).conn_async_read(&mut (*sk)); + + // This function has guarantee that returned buf will be valid until + // the next call. So we can store a Vec in each Safekeeper and reuse + // it on the next call. + let mut inbuf = take_vec_u8(&mut (*sk).inbuf).unwrap_or_default(); + + inbuf.clear(); + inbuf.extend_from_slice(res); + + // Put a Vec back to sk->inbuf and return data ptr. + *buf = store_vec_u8(&mut (*sk).inbuf, inbuf); + *amount = res.len() as i32; + + result + } +} + +extern "C" fn conn_async_write( + sk: *mut Safekeeper, + buf: *const ::std::os::raw::c_void, + size: usize, +) -> PGAsyncWriteResult { + unsafe { + let buf = std::slice::from_raw_parts(buf as *const u8, size); + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_async_write(&mut (*sk), buf) + } +} + +extern "C" fn conn_blocking_write( + sk: *mut Safekeeper, + buf: *const ::std::os::raw::c_void, + size: usize, +) -> bool { + unsafe { + let buf = std::slice::from_raw_parts(buf as *const u8, size); + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).conn_blocking_write(&mut (*sk), buf) + } +} + +extern "C" fn recovery_download( + sk: *mut Safekeeper, + _timeline: TimeLineID, + startpos: XLogRecPtr, + endpos: XLogRecPtr, +) -> bool { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).recovery_download(&mut (*sk), startpos, endpos) + } +} + +extern "C" fn wal_read( + sk: *mut Safekeeper, + buf: *mut ::std::os::raw::c_char, + startptr: XLogRecPtr, + count: Size, +) { + unsafe { + let buf = std::slice::from_raw_parts_mut(buf as *mut u8, count); + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).wal_read(&mut (*sk), buf, startptr) + } +} + +extern "C" fn wal_reader_allocate(sk: *mut Safekeeper) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).wal_reader_allocate(&mut (*sk)); + } +} + +extern "C" fn free_event_set(wp: *mut WalProposer) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).free_event_set(&mut (*wp)); + } +} + +extern "C" fn init_event_set(wp: *mut WalProposer) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).init_event_set(&mut (*wp)); + } +} + +extern "C" fn update_event_set(sk: *mut Safekeeper, events: uint32) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).update_event_set(&mut (*sk), events); + } +} + +extern "C" fn add_safekeeper_event_set(sk: *mut Safekeeper, events: uint32) { + unsafe { + let callback_data = (*(*(*sk).wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).add_safekeeper_event_set(&mut (*sk), events); + } +} + +extern "C" fn wait_event_set( + wp: *mut WalProposer, + timeout: ::std::os::raw::c_long, + event_sk: *mut *mut Safekeeper, + events: *mut uint32, +) -> ::std::os::raw::c_int { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + let result = (*api).wait_event_set(&mut (*wp), timeout); + match result { + WaitResult::Latch => { + *event_sk = std::ptr::null_mut(); + *events = crate::bindings::WL_LATCH_SET; + 1 + } + WaitResult::Timeout => { + *event_sk = std::ptr::null_mut(); + *events = crate::bindings::WL_TIMEOUT; + 0 + } + WaitResult::Network(sk, event_mask) => { + *event_sk = sk; + *events = event_mask; + 1 + } + } + } +} + +extern "C" fn strong_random( + wp: *mut WalProposer, + buf: *mut ::std::os::raw::c_void, + len: usize, +) -> bool { + unsafe { + let buf = std::slice::from_raw_parts_mut(buf as *mut u8, len); + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).strong_random(buf) + } +} + +extern "C" fn get_redo_start_lsn(wp: *mut WalProposer) -> XLogRecPtr { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).get_redo_start_lsn() + } +} + +extern "C" fn finish_sync_safekeepers(wp: *mut WalProposer, lsn: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).finish_sync_safekeepers(lsn) + } +} + +extern "C" fn process_safekeeper_feedback(wp: *mut WalProposer, commit_lsn: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).process_safekeeper_feedback(&mut (*wp), commit_lsn) + } +} + +extern "C" fn confirm_wal_streamed(wp: *mut WalProposer, lsn: XLogRecPtr) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).confirm_wal_streamed(&mut (*wp), lsn) + } +} + +extern "C" fn log_internal( + wp: *mut WalProposer, + level: ::std::os::raw::c_int, + line: *const ::std::os::raw::c_char, +) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + let line = CStr::from_ptr(line); + let line = line.to_str().unwrap(); + (*api).log_internal(&mut (*wp), Level::from(level as u32), line) + } +} + +extern "C" fn after_election(wp: *mut WalProposer) { + unsafe { + let callback_data = (*(*wp).config).callback_data; + let api = callback_data as *mut Box; + (*api).after_election(&mut (*wp)) + } +} + +#[derive(Debug)] +pub enum Level { + Debug5, + Debug4, + Debug3, + Debug2, + Debug1, + Log, + Info, + Notice, + Warning, + Error, + Fatal, + Panic, + WPEvent, +} + +impl Level { + pub fn from(elevel: u32) -> Level { + use crate::bindings::*; + + match elevel { + DEBUG5 => Level::Debug5, + DEBUG4 => Level::Debug4, + DEBUG3 => Level::Debug3, + DEBUG2 => Level::Debug2, + DEBUG1 => Level::Debug1, + LOG => Level::Log, + INFO => Level::Info, + NOTICE => Level::Notice, + WARNING => Level::Warning, + ERROR => Level::Error, + FATAL => Level::Fatal, + PANIC => Level::Panic, + WPEVENT => Level::WPEvent, + _ => panic!("unknown log level {}", elevel), + } + } +} + +pub(crate) fn create_api() -> walproposer_api { + walproposer_api { + get_shmem_state: Some(get_shmem_state), + start_streaming: Some(start_streaming), + get_flush_rec_ptr: Some(get_flush_rec_ptr), + get_current_timestamp: Some(get_current_timestamp), + conn_error_message: Some(conn_error_message), + conn_status: Some(conn_status), + conn_connect_start: Some(conn_connect_start), + conn_connect_poll: Some(conn_connect_poll), + conn_send_query: Some(conn_send_query), + conn_get_query_result: Some(conn_get_query_result), + conn_flush: Some(conn_flush), + conn_finish: Some(conn_finish), + conn_async_read: Some(conn_async_read), + conn_async_write: Some(conn_async_write), + conn_blocking_write: Some(conn_blocking_write), + recovery_download: Some(recovery_download), + wal_read: Some(wal_read), + wal_reader_allocate: Some(wal_reader_allocate), + free_event_set: Some(free_event_set), + init_event_set: Some(init_event_set), + update_event_set: Some(update_event_set), + add_safekeeper_event_set: Some(add_safekeeper_event_set), + wait_event_set: Some(wait_event_set), + strong_random: Some(strong_random), + get_redo_start_lsn: Some(get_redo_start_lsn), + finish_sync_safekeepers: Some(finish_sync_safekeepers), + process_safekeeper_feedback: Some(process_safekeeper_feedback), + confirm_wal_streamed: Some(confirm_wal_streamed), + log_internal: Some(log_internal), + after_election: Some(after_election), + } +} + +impl std::fmt::Display for Level { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{:?}", self) + } +} + +/// Take ownership of `Vec` from StringInfoData. +pub(crate) fn take_vec_u8(pg: &mut StringInfoData) -> Option> { + if pg.data.is_null() { + return None; + } + + let ptr = pg.data as *mut u8; + let length = pg.len as usize; + let capacity = pg.maxlen as usize; + + pg.data = std::ptr::null_mut(); + pg.len = 0; + pg.maxlen = 0; + + unsafe { Some(Vec::from_raw_parts(ptr, length, capacity)) } +} + +/// Store `Vec` in StringInfoData. +fn store_vec_u8(pg: &mut StringInfoData, vec: Vec) -> *mut ::std::os::raw::c_char { + let ptr = vec.as_ptr() as *mut ::std::os::raw::c_char; + let length = vec.len(); + let capacity = vec.capacity(); + + assert!(pg.data.is_null()); + + pg.data = ptr; + pg.len = length as i32; + pg.maxlen = capacity as i32; + + std::mem::forget(vec); + + ptr +} diff --git a/libs/walproposer/src/lib.rs b/libs/walproposer/src/lib.rs new file mode 100644 index 0000000000..f26a13458b --- /dev/null +++ b/libs/walproposer/src/lib.rs @@ -0,0 +1,14 @@ +pub mod bindings { + #![allow(non_upper_case_globals)] + #![allow(non_camel_case_types)] + #![allow(non_snake_case)] + // bindgen creates some unsafe code with no doc comments. + #![allow(clippy::missing_safety_doc)] + // noted at 1.63 that in many cases there's a u32 -> u32 transmutes in bindgen code. + #![allow(clippy::useless_transmute)] + + include!(concat!(env!("OUT_DIR"), "/bindings.rs")); +} + +pub mod api_bindings; +pub mod walproposer; diff --git a/libs/walproposer/src/walproposer.rs b/libs/walproposer/src/walproposer.rs new file mode 100644 index 0000000000..4be15344c5 --- /dev/null +++ b/libs/walproposer/src/walproposer.rs @@ -0,0 +1,485 @@ +use std::ffi::CString; + +use postgres_ffi::WAL_SEGMENT_SIZE; +use utils::id::TenantTimelineId; + +use crate::{ + api_bindings::{create_api, take_vec_u8, Level}, + bindings::{ + Safekeeper, WalProposer, WalProposerConfig, WalProposerCreate, WalProposerFree, + WalProposerStart, + }, +}; + +/// Rust high-level wrapper for C walproposer API. Many methods are not required +/// for simple cases, hence todo!() in default implementations. +/// +/// Refer to `pgxn/neon/walproposer.h` for documentation. +pub trait ApiImpl { + fn get_shmem_state(&self) -> &mut crate::bindings::WalproposerShmemState { + todo!() + } + + fn start_streaming(&self, _startpos: u64) { + todo!() + } + + fn get_flush_rec_ptr(&self) -> u64 { + todo!() + } + + fn get_current_timestamp(&self) -> i64 { + todo!() + } + + fn conn_error_message(&self, _sk: &mut Safekeeper) -> String { + todo!() + } + + fn conn_status(&self, _sk: &mut Safekeeper) -> crate::bindings::WalProposerConnStatusType { + todo!() + } + + fn conn_connect_start(&self, _sk: &mut Safekeeper) { + todo!() + } + + fn conn_connect_poll( + &self, + _sk: &mut Safekeeper, + ) -> crate::bindings::WalProposerConnectPollStatusType { + todo!() + } + + fn conn_send_query(&self, _sk: &mut Safekeeper, _query: &str) -> bool { + todo!() + } + + fn conn_get_query_result( + &self, + _sk: &mut Safekeeper, + ) -> crate::bindings::WalProposerExecStatusType { + todo!() + } + + fn conn_flush(&self, _sk: &mut Safekeeper) -> i32 { + todo!() + } + + fn conn_finish(&self, _sk: &mut Safekeeper) { + todo!() + } + + fn conn_async_read(&self, _sk: &mut Safekeeper) -> (&[u8], crate::bindings::PGAsyncReadResult) { + todo!() + } + + fn conn_async_write( + &self, + _sk: &mut Safekeeper, + _buf: &[u8], + ) -> crate::bindings::PGAsyncWriteResult { + todo!() + } + + fn conn_blocking_write(&self, _sk: &mut Safekeeper, _buf: &[u8]) -> bool { + todo!() + } + + fn recovery_download(&self, _sk: &mut Safekeeper, _startpos: u64, _endpos: u64) -> bool { + todo!() + } + + fn wal_read(&self, _sk: &mut Safekeeper, _buf: &mut [u8], _startpos: u64) { + todo!() + } + + fn wal_reader_allocate(&self, _sk: &mut Safekeeper) { + todo!() + } + + fn free_event_set(&self, _wp: &mut WalProposer) { + todo!() + } + + fn init_event_set(&self, _wp: &mut WalProposer) { + todo!() + } + + fn update_event_set(&self, _sk: &mut Safekeeper, _events_mask: u32) { + todo!() + } + + fn add_safekeeper_event_set(&self, _sk: &mut Safekeeper, _events_mask: u32) { + todo!() + } + + fn wait_event_set(&self, _wp: &mut WalProposer, _timeout_millis: i64) -> WaitResult { + todo!() + } + + fn strong_random(&self, _buf: &mut [u8]) -> bool { + todo!() + } + + fn get_redo_start_lsn(&self) -> u64 { + todo!() + } + + fn finish_sync_safekeepers(&self, _lsn: u64) { + todo!() + } + + fn process_safekeeper_feedback(&self, _wp: &mut WalProposer, _commit_lsn: u64) { + todo!() + } + + fn confirm_wal_streamed(&self, _wp: &mut WalProposer, _lsn: u64) { + todo!() + } + + fn log_internal(&self, _wp: &mut WalProposer, _level: Level, _msg: &str) { + todo!() + } + + fn after_election(&self, _wp: &mut WalProposer) { + todo!() + } +} + +pub enum WaitResult { + Latch, + Timeout, + Network(*mut Safekeeper, u32), +} + +pub struct Config { + /// Tenant and timeline id + pub ttid: TenantTimelineId, + /// List of safekeepers in format `host:port` + pub safekeepers_list: Vec, + /// Safekeeper reconnect timeout in milliseconds + pub safekeeper_reconnect_timeout: i32, + /// Safekeeper connection timeout in milliseconds + pub safekeeper_connection_timeout: i32, + /// walproposer mode, finish when all safekeepers are synced or subscribe + /// to WAL streaming + pub sync_safekeepers: bool, +} + +/// WalProposer main struct. C methods are reexported as Rust functions. +pub struct Wrapper { + wp: *mut WalProposer, + _safekeepers_list_vec: Vec, +} + +impl Wrapper { + pub fn new(api: Box, config: Config) -> Wrapper { + let neon_tenant = CString::new(config.ttid.tenant_id.to_string()) + .unwrap() + .into_raw(); + let neon_timeline = CString::new(config.ttid.timeline_id.to_string()) + .unwrap() + .into_raw(); + + let mut safekeepers_list_vec = CString::new(config.safekeepers_list.join(",")) + .unwrap() + .into_bytes_with_nul(); + assert!(safekeepers_list_vec.len() == safekeepers_list_vec.capacity()); + let safekeepers_list = safekeepers_list_vec.as_mut_ptr() as *mut i8; + + let callback_data = Box::into_raw(Box::new(api)) as *mut ::std::os::raw::c_void; + + let c_config = WalProposerConfig { + neon_tenant, + neon_timeline, + safekeepers_list, + safekeeper_reconnect_timeout: config.safekeeper_reconnect_timeout, + safekeeper_connection_timeout: config.safekeeper_connection_timeout, + wal_segment_size: WAL_SEGMENT_SIZE as i32, // default 16MB + syncSafekeepers: config.sync_safekeepers, + systemId: 0, + pgTimeline: 1, + callback_data, + }; + let c_config = Box::into_raw(Box::new(c_config)); + + let api = create_api(); + let wp = unsafe { WalProposerCreate(c_config, api) }; + Wrapper { + wp, + _safekeepers_list_vec: safekeepers_list_vec, + } + } + + pub fn start(&self) { + unsafe { WalProposerStart(self.wp) } + } +} + +impl Drop for Wrapper { + fn drop(&mut self) { + unsafe { + let config = (*self.wp).config; + drop(Box::from_raw( + (*config).callback_data as *mut Box, + )); + drop(CString::from_raw((*config).neon_tenant)); + drop(CString::from_raw((*config).neon_timeline)); + drop(Box::from_raw(config)); + + for i in 0..(*self.wp).n_safekeepers { + let sk = &mut (*self.wp).safekeeper[i as usize]; + take_vec_u8(&mut sk.inbuf); + } + + WalProposerFree(self.wp); + } + } +} + +#[cfg(test)] +mod tests { + use std::{ + cell::Cell, + sync::{atomic::AtomicUsize, mpsc::sync_channel}, + }; + + use utils::id::TenantTimelineId; + + use crate::{api_bindings::Level, walproposer::Wrapper}; + + use super::ApiImpl; + + #[derive(Clone, Copy, Debug)] + struct WaitEventsData { + sk: *mut crate::bindings::Safekeeper, + event_mask: u32, + } + + struct MockImpl { + // data to return from wait_event_set + wait_events: Cell, + // walproposer->safekeeper messages + expected_messages: Vec>, + expected_ptr: AtomicUsize, + // safekeeper->walproposer messages + safekeeper_replies: Vec>, + replies_ptr: AtomicUsize, + // channel to send LSN to the main thread + sync_channel: std::sync::mpsc::SyncSender, + } + + impl MockImpl { + fn check_walproposer_msg(&self, msg: &[u8]) { + let ptr = self + .expected_ptr + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + + if ptr >= self.expected_messages.len() { + panic!("unexpected message from walproposer"); + } + + let expected_msg = &self.expected_messages[ptr]; + assert_eq!(msg, expected_msg.as_slice()); + } + + fn next_safekeeper_reply(&self) -> &[u8] { + let ptr = self + .replies_ptr + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + + if ptr >= self.safekeeper_replies.len() { + panic!("no more safekeeper replies"); + } + + &self.safekeeper_replies[ptr] + } + } + + impl ApiImpl for MockImpl { + fn get_current_timestamp(&self) -> i64 { + println!("get_current_timestamp"); + 0 + } + + fn conn_status( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> crate::bindings::WalProposerConnStatusType { + println!("conn_status"); + crate::bindings::WalProposerConnStatusType_WP_CONNECTION_OK + } + + fn conn_connect_start(&self, _: &mut crate::bindings::Safekeeper) { + println!("conn_connect_start"); + } + + fn conn_connect_poll( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> crate::bindings::WalProposerConnectPollStatusType { + println!("conn_connect_poll"); + crate::bindings::WalProposerConnectPollStatusType_WP_CONN_POLLING_OK + } + + fn conn_send_query(&self, _: &mut crate::bindings::Safekeeper, query: &str) -> bool { + println!("conn_send_query: {}", query); + true + } + + fn conn_get_query_result( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> crate::bindings::WalProposerExecStatusType { + println!("conn_get_query_result"); + crate::bindings::WalProposerExecStatusType_WP_EXEC_SUCCESS_COPYBOTH + } + + fn conn_async_read( + &self, + _: &mut crate::bindings::Safekeeper, + ) -> (&[u8], crate::bindings::PGAsyncReadResult) { + println!("conn_async_read"); + let reply = self.next_safekeeper_reply(); + println!("conn_async_read result: {:?}", reply); + ( + reply, + crate::bindings::PGAsyncReadResult_PG_ASYNC_READ_SUCCESS, + ) + } + + fn conn_blocking_write(&self, _: &mut crate::bindings::Safekeeper, buf: &[u8]) -> bool { + println!("conn_blocking_write: {:?}", buf); + self.check_walproposer_msg(buf); + true + } + + fn wal_reader_allocate(&self, _: &mut crate::bindings::Safekeeper) { + println!("wal_reader_allocate") + } + + fn free_event_set(&self, _: &mut crate::bindings::WalProposer) { + println!("free_event_set") + } + + fn init_event_set(&self, _: &mut crate::bindings::WalProposer) { + println!("init_event_set") + } + + fn update_event_set(&self, sk: &mut crate::bindings::Safekeeper, event_mask: u32) { + println!( + "update_event_set, sk={:?}, events_mask={:#b}", + sk as *mut crate::bindings::Safekeeper, event_mask + ); + self.wait_events.set(WaitEventsData { sk, event_mask }); + } + + fn add_safekeeper_event_set(&self, sk: &mut crate::bindings::Safekeeper, event_mask: u32) { + println!( + "add_safekeeper_event_set, sk={:?}, events_mask={:#b}", + sk as *mut crate::bindings::Safekeeper, event_mask + ); + self.wait_events.set(WaitEventsData { sk, event_mask }); + } + + fn wait_event_set( + &self, + _: &mut crate::bindings::WalProposer, + timeout_millis: i64, + ) -> super::WaitResult { + let data = self.wait_events.get(); + println!( + "wait_event_set, timeout_millis={}, res={:?}", + timeout_millis, data + ); + super::WaitResult::Network(data.sk, data.event_mask) + } + + fn strong_random(&self, buf: &mut [u8]) -> bool { + println!("strong_random"); + buf.fill(0); + true + } + + fn finish_sync_safekeepers(&self, lsn: u64) { + self.sync_channel.send(lsn).unwrap(); + panic!("sync safekeepers finished at lsn={}", lsn); + } + + fn log_internal(&self, _wp: &mut crate::bindings::WalProposer, level: Level, msg: &str) { + println!("walprop_log[{}] {}", level, msg); + } + + fn after_election(&self, _wp: &mut crate::bindings::WalProposer) { + println!("after_election"); + } + } + + /// Test that walproposer can successfully connect to safekeeper and finish + /// sync_safekeepers. API is mocked in MockImpl. + /// + /// Run this test with valgrind to detect leaks: + /// `valgrind --leak-check=full target/debug/deps/walproposer-` + #[test] + fn test_simple_sync_safekeepers() -> anyhow::Result<()> { + let ttid = TenantTimelineId::new( + "9e4c8f36063c6c6e93bc20d65a820f3d".parse()?, + "9e4c8f36063c6c6e93bc20d65a820f3d".parse()?, + ); + + let (sender, receiver) = sync_channel(1); + + let my_impl: Box = Box::new(MockImpl { + wait_events: Cell::new(WaitEventsData { + sk: std::ptr::null_mut(), + event_mask: 0, + }), + expected_messages: vec![ + // Greeting(ProposerGreeting { protocol_version: 2, pg_version: 160000, proposer_id: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], system_id: 0, timeline_id: 9e4c8f36063c6c6e93bc20d65a820f3d, tenant_id: 9e4c8f36063c6c6e93bc20d65a820f3d, tli: 1, wal_seg_size: 16777216 }) + vec![ + 103, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 113, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 158, 76, 143, 54, 6, 60, 108, 110, + 147, 188, 32, 214, 90, 130, 15, 61, 158, 76, 143, 54, 6, 60, 108, 110, 147, + 188, 32, 214, 90, 130, 15, 61, 1, 0, 0, 0, 0, 0, 0, 1, + ], + // VoteRequest(VoteRequest { term: 3 }) + vec![ + 118, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, + ], + ], + expected_ptr: AtomicUsize::new(0), + safekeeper_replies: vec![ + // Greeting(AcceptorGreeting { term: 2, node_id: NodeId(1) }) + vec![ + 103, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, + ], + // VoteResponse(VoteResponse { term: 3, vote_given: 1, flush_lsn: 0/539, truncate_lsn: 0/539, term_history: [(2, 0/539)], timeline_start_lsn: 0/539 }) + vec![ + 118, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 57, + 5, 0, 0, 0, 0, 0, 0, 57, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, + 0, 57, 5, 0, 0, 0, 0, 0, 0, 57, 5, 0, 0, 0, 0, 0, 0, + ], + ], + replies_ptr: AtomicUsize::new(0), + sync_channel: sender, + }); + let config = crate::walproposer::Config { + ttid, + safekeepers_list: vec!["localhost:5000".to_string()], + safekeeper_reconnect_timeout: 1000, + safekeeper_connection_timeout: 10000, + sync_safekeepers: true, + }; + + let wp = Wrapper::new(my_impl, config); + + // walproposer will panic when it finishes sync_safekeepers + std::panic::catch_unwind(|| wp.start()).unwrap_err(); + // validate the resulting LSN + assert_eq!(receiver.recv()?, 1337); + Ok(()) + // drop() will free up resources here + } +} diff --git a/pageserver/benches/bench_walredo.rs b/pageserver/benches/bench_walredo.rs index 9bcd3fa708..ba41866935 100644 --- a/pageserver/benches/bench_walredo.rs +++ b/pageserver/benches/bench_walredo.rs @@ -32,9 +32,15 @@ fn redo_scenarios(c: &mut Criterion) { let manager = Arc::new(manager); - tracing::info!("executing first"); - short().execute(&manager).unwrap(); - tracing::info!("first executed"); + { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + tracing::info!("executing first"); + short().execute(rt.handle(), &manager).unwrap(); + tracing::info!("first executed"); + } let thread_counts = [1, 2, 4, 8, 16]; @@ -77,9 +83,14 @@ fn add_multithreaded_walredo_requesters( assert_ne!(threads, 0); if threads == 1 { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let handle = rt.handle(); b.iter_batched_ref( || Some(input_factory()), - |input| execute_all(input.take(), manager), + |input| execute_all(input.take(), handle, manager), criterion::BatchSize::PerIteration, ); } else { @@ -95,19 +106,26 @@ fn add_multithreaded_walredo_requesters( let manager = manager.clone(); let barrier = barrier.clone(); let work_rx = work_rx.clone(); - move || loop { - // queue up and wait if we want to go another round - if work_rx.lock().unwrap().recv().is_err() { - break; + move || { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let handle = rt.handle(); + loop { + // queue up and wait if we want to go another round + if work_rx.lock().unwrap().recv().is_err() { + break; + } + + let input = Some(input_factory()); + + barrier.wait(); + + execute_all(input, handle, &manager).unwrap(); + + barrier.wait(); } - - let input = Some(input_factory()); - - barrier.wait(); - - execute_all(input, &manager).unwrap(); - - barrier.wait(); } }) }) @@ -149,13 +167,17 @@ impl Drop for JoinOnDrop { } } -fn execute_all(input: I, manager: &PostgresRedoManager) -> anyhow::Result<()> +fn execute_all( + input: I, + handle: &tokio::runtime::Handle, + manager: &PostgresRedoManager, +) -> anyhow::Result<()> where I: IntoIterator, { // just fire all requests as fast as possible input.into_iter().try_for_each(|req| { - let page = req.execute(manager)?; + let page = req.execute(handle, manager)?; assert_eq!(page.remaining(), 8192); anyhow::Ok(()) }) @@ -470,9 +492,11 @@ struct Request { } impl Request { - fn execute(self, manager: &PostgresRedoManager) -> anyhow::Result { - use pageserver::walredo::WalRedoManager; - + fn execute( + self, + rt: &tokio::runtime::Handle, + manager: &PostgresRedoManager, + ) -> anyhow::Result { let Request { key, lsn, @@ -481,6 +505,6 @@ impl Request { pg_version, } = self; - manager.request_redo(key, lsn, base_img, records, pg_version) + rt.block_on(manager.request_redo(key, lsn, base_img, records, pg_version)) } } diff --git a/pageserver/src/basebackup.rs b/pageserver/src/basebackup.rs index a959f1cddc..ed452eae7d 100644 --- a/pageserver/src/basebackup.rs +++ b/pageserver/src/basebackup.rs @@ -13,6 +13,7 @@ use anyhow::{anyhow, bail, ensure, Context}; use bytes::{BufMut, BytesMut}; use fail::fail_point; +use postgres_ffi::pg_constants; use std::fmt::Write as FmtWrite; use std::time::SystemTime; use tokio::io; @@ -180,6 +181,7 @@ where } } + let mut min_restart_lsn: Lsn = Lsn::MAX; // Create tablespace directories for ((spcnode, dbnode), has_relmap_file) in self.timeline.list_dbdirs(self.lsn, self.ctx).await? @@ -213,6 +215,34 @@ where self.add_rel(rel, rel).await?; } } + + for (path, content) in self.timeline.list_aux_files(self.lsn, self.ctx).await? { + if path.starts_with("pg_replslot") { + let offs = pg_constants::REPL_SLOT_ON_DISK_OFFSETOF_RESTART_LSN; + let restart_lsn = Lsn(u64::from_le_bytes( + content[offs..offs + 8].try_into().unwrap(), + )); + info!("Replication slot {} restart LSN={}", path, restart_lsn); + min_restart_lsn = Lsn::min(min_restart_lsn, restart_lsn); + } + let header = new_tar_header(&path, content.len() as u64)?; + self.ar + .append(&header, &*content) + .await + .context("could not add aux file to basebackup tarball")?; + } + } + if min_restart_lsn != Lsn::MAX { + info!( + "Min restart LSN for logical replication is {}", + min_restart_lsn + ); + let data = min_restart_lsn.0.to_le_bytes(); + let header = new_tar_header("restart.lsn", data.len() as u64)?; + self.ar + .append(&header, &data[..]) + .await + .context("could not add restart.lsn file to basebackup tarball")?; } for xid in self .timeline diff --git a/pageserver/src/bin/pageserver.rs b/pageserver/src/bin/pageserver.rs index 827f74fcce..798b9f258b 100644 --- a/pageserver/src/bin/pageserver.rs +++ b/pageserver/src/bin/pageserver.rs @@ -2,6 +2,7 @@ use std::env::{var, VarError}; use std::sync::Arc; +use std::time::Duration; use std::{env, ops::ControlFlow, str::FromStr}; use anyhow::{anyhow, Context}; @@ -200,6 +201,51 @@ fn initialize_config( }) } +struct WaitForPhaseResult { + timeout_remaining: Duration, + skipped: Option, +} + +/// During startup, we apply a timeout to our waits for readiness, to avoid +/// stalling the whole service if one Tenant experiences some problem. Each +/// phase may consume some of the timeout: this function returns the updated +/// timeout for use in the next call. +async fn wait_for_phase(phase: &str, mut fut: F, timeout: Duration) -> WaitForPhaseResult +where + F: std::future::Future + Unpin, +{ + let initial_t = Instant::now(); + let skipped = match tokio::time::timeout(timeout, &mut fut).await { + Ok(_) => None, + Err(_) => { + tracing::info!( + timeout_millis = timeout.as_millis(), + %phase, + "Startup phase timed out, proceeding anyway" + ); + Some(fut) + } + }; + + WaitForPhaseResult { + timeout_remaining: timeout + .checked_sub(Instant::now().duration_since(initial_t)) + .unwrap_or(Duration::ZERO), + skipped, + } +} + +fn startup_checkpoint(started_at: Instant, phase: &str, human_phase: &str) { + let elapsed = started_at.elapsed(); + let secs = elapsed.as_secs_f64(); + STARTUP_DURATION.with_label_values(&[phase]).set(secs); + + info!( + elapsed_ms = elapsed.as_millis(), + "{human_phase} ({secs:.3}s since start)" + ) +} + fn start_pageserver( launch_ts: &'static LaunchTimestamp, conf: &'static PageServerConf, @@ -207,16 +253,6 @@ fn start_pageserver( // Monotonic time for later calculating startup duration let started_startup_at = Instant::now(); - let startup_checkpoint = move |phase: &str, human_phase: &str| { - let elapsed = started_startup_at.elapsed(); - let secs = elapsed.as_secs_f64(); - STARTUP_DURATION.with_label_values(&[phase]).set(secs); - info!( - elapsed_ms = elapsed.as_millis(), - "{human_phase} ({secs:.3}s since start)" - ) - }; - // Print version and launch timestamp to the log, // and expose them as prometheus metrics. // A changed version string indicates changed software. @@ -341,7 +377,7 @@ fn start_pageserver( // Up to this point no significant I/O has been done: this should have been fast. Record // duration prior to starting I/O intensive phase of startup. - startup_checkpoint("initial", "Starting loading tenants"); + startup_checkpoint(started_startup_at, "initial", "Starting loading tenants"); STARTUP_IS_LOADING.set(1); // Startup staging or optimizing: @@ -388,58 +424,93 @@ fn start_pageserver( let shutdown_pageserver = shutdown_pageserver.clone(); let drive_init = async move { // NOTE: unlike many futures in pageserver, this one is cancellation-safe - let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial load completed")); + let guard = scopeguard::guard_on_success((), |_| { + tracing::info!("Cancelled before initial load completed") + }); - init_remote_done_rx.wait().await; - startup_checkpoint("initial_tenant_load_remote", "Remote part of initial load completed"); + let timeout = conf.background_task_maximum_delay; - init_done_rx.wait().await; - startup_checkpoint("initial_tenant_load", "Initial load completed"); - STARTUP_IS_LOADING.set(0); + let init_remote_done = std::pin::pin!(async { + init_remote_done_rx.wait().await; + startup_checkpoint( + started_startup_at, + "initial_tenant_load_remote", + "Remote part of initial load completed", + ); + }); + + let WaitForPhaseResult { + timeout_remaining: timeout, + skipped: init_remote_skipped, + } = wait_for_phase("initial_tenant_load_remote", init_remote_done, timeout).await; + + let init_load_done = std::pin::pin!(async { + init_done_rx.wait().await; + startup_checkpoint( + started_startup_at, + "initial_tenant_load", + "Initial load completed", + ); + STARTUP_IS_LOADING.set(0); + }); + + let WaitForPhaseResult { + timeout_remaining: timeout, + skipped: init_load_skipped, + } = wait_for_phase("initial_tenant_load", init_load_done, timeout).await; // initial logical sizes can now start, as they were waiting on init_done_rx. scopeguard::ScopeGuard::into_inner(guard); - let mut init_sizes_done = std::pin::pin!(init_logical_size_done_rx.wait()); + let guard = scopeguard::guard_on_success((), |_| { + tracing::info!("Cancelled before initial logical sizes completed") + }); - let timeout = conf.background_task_maximum_delay; + let logical_sizes_done = std::pin::pin!(async { + init_logical_size_done_rx.wait().await; + startup_checkpoint( + started_startup_at, + "initial_logical_sizes", + "Initial logical sizes completed", + ); + }); - let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial logical sizes completed")); - - let init_sizes_done = match tokio::time::timeout(timeout, &mut init_sizes_done).await { - Ok(_) => { - startup_checkpoint("initial_logical_sizes", "Initial logical sizes completed"); - None - } - Err(_) => { - tracing::info!( - timeout_millis = timeout.as_millis(), - "Initial logical size timeout elapsed; starting background jobs" - ); - Some(init_sizes_done) - } - }; + let WaitForPhaseResult { + timeout_remaining: _, + skipped: logical_sizes_skipped, + } = wait_for_phase("initial_logical_sizes", logical_sizes_done, timeout).await; scopeguard::ScopeGuard::into_inner(guard); - // allow background jobs to start + // allow background jobs to start: we either completed prior stages, or they reached timeout + // and were skipped. It is important that we do not let them block background jobs indefinitely, + // because things like consumption metrics for billing are blocked by this barrier. drop(background_jobs_can_start); - startup_checkpoint("background_jobs_can_start", "Starting background jobs"); - - if let Some(init_sizes_done) = init_sizes_done { - // ending up here is not a bug; at the latest logical sizes will be queried by - // consumption metrics. - let guard = scopeguard::guard_on_success((), |_| tracing::info!("Cancelled before initial logical sizes completed")); - init_sizes_done.await; - - scopeguard::ScopeGuard::into_inner(guard); - - startup_checkpoint("initial_logical_sizes", "Initial logical sizes completed after timeout (background jobs already started)"); + startup_checkpoint( + started_startup_at, + "background_jobs_can_start", + "Starting background jobs", + ); + // We are done. If we skipped any phases due to timeout, run them to completion here so that + // they will eventually update their startup_checkpoint, and so that we do not declare the + // 'complete' stage until all the other stages are really done. + let guard = scopeguard::guard_on_success((), |_| { + tracing::info!("Cancelled before waiting for skipped phases done") + }); + if let Some(f) = init_remote_skipped { + f.await; } + if let Some(f) = init_load_skipped { + f.await; + } + if let Some(f) = logical_sizes_skipped { + f.await; + } + scopeguard::ScopeGuard::into_inner(guard); - startup_checkpoint("complete", "Startup complete"); + startup_checkpoint(started_startup_at, "complete", "Startup complete"); }; async move { @@ -579,6 +650,7 @@ fn start_pageserver( pageserver_listener, conf.pg_auth_type, libpq_ctx, + task_mgr::shutdown_token(), ) .await }, diff --git a/pageserver/src/consumption_metrics.rs b/pageserver/src/consumption_metrics.rs index 13f7977946..061045eb76 100644 --- a/pageserver/src/consumption_metrics.rs +++ b/pageserver/src/consumption_metrics.rs @@ -11,6 +11,7 @@ use reqwest::Url; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, SystemTime}; +use tokio::time::Instant; use tracing::*; use utils::id::NodeId; @@ -88,22 +89,12 @@ pub async fn collect_metrics( let node_id = node_id.to_string(); - // reminder: ticker is ready immediatedly - let mut ticker = tokio::time::interval(metric_collection_interval); - loop { - let tick_at = tokio::select! { - _ = cancel.cancelled() => return Ok(()), - tick_at = ticker.tick() => tick_at, - }; + let started_at = Instant::now(); // these are point in time, with variable "now" let metrics = metrics::collect_all_metrics(&cached_metrics, &ctx).await; - if metrics.is_empty() { - continue; - } - let metrics = Arc::new(metrics); // why not race cancellation here? because we are one of the last tasks, and if we are @@ -142,10 +133,19 @@ pub async fn collect_metrics( let (_, _) = tokio::join!(flush, upload); crate::tenant::tasks::warn_when_period_overrun( - tick_at.elapsed(), + started_at.elapsed(), metric_collection_interval, BackgroundLoopKind::ConsumptionMetricsCollectMetrics, ); + + let res = tokio::time::timeout_at( + started_at + metric_collection_interval, + task_mgr::shutdown_token().cancelled(), + ) + .await; + if res.is_ok() { + return Ok(()); + } } } @@ -244,16 +244,14 @@ async fn calculate_synthetic_size_worker( ctx: &RequestContext, ) -> anyhow::Result<()> { info!("starting calculate_synthetic_size_worker"); + scopeguard::defer! { + info!("calculate_synthetic_size_worker stopped"); + }; - // reminder: ticker is ready immediatedly - let mut ticker = tokio::time::interval(synthetic_size_calculation_interval); let cause = LogicalSizeCalculationCause::ConsumptionMetricsSyntheticSize; loop { - let tick_at = tokio::select! { - _ = task_mgr::shutdown_watcher() => return Ok(()), - tick_at = ticker.tick() => tick_at, - }; + let started_at = Instant::now(); let tenants = match mgr::list_tenants().await { Ok(tenants) => tenants, @@ -281,9 +279,18 @@ async fn calculate_synthetic_size_worker( } crate::tenant::tasks::warn_when_period_overrun( - tick_at.elapsed(), + started_at.elapsed(), synthetic_size_calculation_interval, BackgroundLoopKind::ConsumptionMetricsSyntheticSizeWorker, ); + + let res = tokio::time::timeout_at( + started_at + synthetic_size_calculation_interval, + task_mgr::shutdown_token().cancelled(), + ) + .await; + if res.is_ok() { + return Ok(()); + } } } diff --git a/pageserver/src/deletion_queue.rs b/pageserver/src/deletion_queue.rs index 0bf851a8d7..22efa23f10 100644 --- a/pageserver/src/deletion_queue.rs +++ b/pageserver/src/deletion_queue.rs @@ -1298,10 +1298,6 @@ pub(crate) mod mock { } } - pub fn get_executed(&self) -> usize { - self.executed.load(Ordering::Relaxed) - } - #[allow(clippy::await_holding_lock)] pub async fn pump(&self) { if let Some(remote_storage) = &self.remote_storage { diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 477a2d378d..f0bf2666a7 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -306,6 +306,67 @@ paths: schema: $ref: "#/components/schemas/ServiceUnavailableError" + /v1/tenant/{tenant_id}/timeline/{timeline_id}/get_timestamp_of_lsn: + parameters: + - name: tenant_id + in: path + required: true + schema: + type: string + format: hex + - name: timeline_id + in: path + required: true + schema: + type: string + format: hex + get: + description: Get timestamp for a given LSN + parameters: + - name: lsn + in: query + required: true + schema: + type: integer + description: A LSN to get the timestamp + responses: + "200": + description: OK + content: + application/json: + schema: + type: string + format: date-time + "400": + description: Error when no tenant id found in path, no timeline id or invalid timestamp + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "401": + description: Unauthorized Error + content: + application/json: + schema: + $ref: "#/components/schemas/UnauthorizedError" + "403": + description: Forbidden Error + content: + application/json: + schema: + $ref: "#/components/schemas/ForbiddenError" + "404": + description: Timeline not found, or there is no timestamp information for the given lsn + content: + application/json: + schema: + $ref: "#/components/schemas/NotFoundError" + "500": + description: Generic operation error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" /v1/tenant/{tenant_id}/timeline/{timeline_id}/get_lsn_by_timestamp: parameters: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 38304eb16b..142e85d203 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -2,10 +2,12 @@ //! Management HTTP API //! use std::collections::HashMap; +use std::str::FromStr; use std::sync::Arc; use anyhow::{anyhow, Context, Result}; use futures::TryFutureExt; +use humantime::format_rfc3339; use hyper::header::CONTENT_TYPE; use hyper::StatusCode; use hyper::{Body, Request, Response, Uri}; @@ -502,6 +504,33 @@ async fn get_lsn_by_timestamp_handler( json_response(StatusCode::OK, result) } +async fn get_timestamp_of_lsn_handler( + request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + let tenant_id: TenantId = parse_request_param(&request, "tenant_id")?; + check_permission(&request, Some(tenant_id))?; + + let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?; + + let lsn_str = must_get_query_param(&request, "lsn")?; + let lsn = Lsn::from_str(&lsn_str) + .with_context(|| format!("Invalid LSN: {lsn_str:?}")) + .map_err(ApiError::BadRequest)?; + + let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Download); + let timeline = active_timeline_of_active_tenant(tenant_id, timeline_id).await?; + let result = timeline.get_timestamp_for_lsn(lsn, &ctx).await?; + + match result { + Some(time) => { + let time = format_rfc3339(postgres_ffi::from_pg_timestamp(time)).to_string(); + json_response(StatusCode::OK, time) + } + None => json_response(StatusCode::NOT_FOUND, ()), + } +} + async fn tenant_attach_handler( mut request: Request, _cancel: CancellationToken, @@ -1034,9 +1063,17 @@ async fn put_tenant_location_config_handler( // The `Detached` state is special, it doesn't upsert a tenant, it removes // its local disk content and drops it from memory. if let LocationConfigMode::Detached = request_data.config.mode { - mgr::detach_tenant(conf, tenant_id, true, &state.deletion_queue_client) + if let Err(e) = mgr::detach_tenant(conf, tenant_id, true, &state.deletion_queue_client) .instrument(info_span!("tenant_detach", %tenant_id)) - .await?; + .await + { + match e { + TenantStateError::NotFound(_) => { + // This API is idempotent: a NotFound on a detach is fine. + } + _ => return Err(e.into()), + } + } return json_response(StatusCode::OK, ()); } @@ -1672,6 +1709,10 @@ pub fn make_router( "/v1/tenant/:tenant_id/timeline/:timeline_id/get_lsn_by_timestamp", |r| api_handler(r, get_lsn_by_timestamp_handler), ) + .get( + "/v1/tenant/:tenant_id/timeline/:timeline_id/get_timestamp_of_lsn", + |r| api_handler(r, get_timestamp_of_lsn_handler), + ) .put("/v1/tenant/:tenant_id/timeline/:timeline_id/do_gc", |r| { api_handler(r, timeline_gc_handler) }) diff --git a/pageserver/src/page_cache.rs b/pageserver/src/page_cache.rs index f6acf64f22..0702057766 100644 --- a/pageserver/src/page_cache.rs +++ b/pageserver/src/page_cache.rs @@ -318,15 +318,6 @@ impl std::ops::Deref for PageWriteGuard<'_> { } } -impl AsMut<[u8; PAGE_SZ]> for PageWriteGuard<'_> { - fn as_mut(&mut self) -> &mut [u8; PAGE_SZ] { - match &mut self.state { - PageWriteGuardState::Invalid { inner, _permit } => inner.buf, - PageWriteGuardState::Downgraded => unreachable!(), - } - } -} - impl<'a> PageWriteGuard<'a> { /// Mark that the buffer contents are now valid. #[must_use] diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 5ab4fbbd4c..536334d051 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -122,6 +122,7 @@ pub async fn libpq_listener_main( listener: TcpListener, auth_type: AuthType, listener_ctx: RequestContext, + cancel: CancellationToken, ) -> anyhow::Result<()> { listener.set_nonblocking(true)?; let tokio_listener = tokio::net::TcpListener::from_std(listener)?; @@ -130,7 +131,7 @@ pub async fn libpq_listener_main( while let Some(res) = tokio::select! { biased; - _ = task_mgr::shutdown_watcher() => { + _ = cancel.cancelled() => { // We were requested to shut down. None } @@ -299,7 +300,7 @@ impl PageServerHandler { Ok(flush_r?) }, _ = self.cancel.cancelled() => { - Err(QueryError::Other(anyhow::anyhow!("Shutting down"))) + Err(QueryError::Shutdown) } ) } @@ -316,11 +317,11 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = task_mgr::shutdown_watcher() => { + _ = self.cancel.cancelled() => { // We were requested to shut down. let msg = "pageserver is shutting down"; let _ = pgb.write_message_noflush(&BeMessage::ErrorResponse(msg, None)); - Err(QueryError::Other(anyhow::anyhow!(msg))) + Err(QueryError::Shutdown) } msg = pgb.read_message() => { msg.map_err(QueryError::from)} @@ -414,10 +415,10 @@ impl PageServerHandler { let msg = tokio::select! { biased; - _ = task_mgr::shutdown_watcher() => { + _ = self.cancel.cancelled() => { // We were requested to shut down. info!("shutdown request received in page handler"); - break; + return Err(QueryError::Shutdown) } msg = pgb.read_message() => { msg } diff --git a/pageserver/src/pgdatadir_mapping.rs b/pageserver/src/pgdatadir_mapping.rs index 9a1281a522..d27c8a3d5d 100644 --- a/pageserver/src/pgdatadir_mapping.rs +++ b/pageserver/src/pgdatadir_mapping.rs @@ -19,6 +19,7 @@ use postgres_ffi::BLCKSZ; use postgres_ffi::{Oid, TimestampTz, TransactionId}; use serde::{Deserialize, Serialize}; use std::collections::{hash_map, HashMap, HashSet}; +use std::ops::ControlFlow; use std::ops::Range; use tokio_util::sync::CancellationToken; use tracing::{debug, trace, warn}; @@ -370,7 +371,6 @@ impl Timeline { } } - /// /// Subroutine of find_lsn_for_timestamp(). Returns true, if there are any /// commits that committed after 'search_timestamp', at LSN 'probe_lsn'. /// @@ -385,6 +385,50 @@ impl Timeline { found_larger: &mut bool, ctx: &RequestContext, ) -> Result { + self.map_all_timestamps(probe_lsn, ctx, |timestamp| { + if timestamp >= search_timestamp { + *found_larger = true; + return ControlFlow::Break(true); + } else { + *found_smaller = true; + } + ControlFlow::Continue(()) + }) + .await + } + + /// Obtain the possible timestamp range for the given lsn. + /// + /// If the lsn has no timestamps, returns None. returns `(min, max, median)` if it has timestamps. + pub async fn get_timestamp_for_lsn( + &self, + probe_lsn: Lsn, + ctx: &RequestContext, + ) -> Result, PageReconstructError> { + let mut max: Option = None; + self.map_all_timestamps(probe_lsn, ctx, |timestamp| { + if let Some(max_prev) = max { + max = Some(max_prev.max(timestamp)); + } else { + max = Some(timestamp); + } + ControlFlow::Continue(()) + }) + .await?; + + Ok(max) + } + + /// Runs the given function on all the timestamps for a given lsn + /// + /// The return value is either given by the closure, or set to the `Default` + /// impl's output. + async fn map_all_timestamps( + &self, + probe_lsn: Lsn, + ctx: &RequestContext, + mut f: impl FnMut(TimestampTz) -> ControlFlow, + ) -> Result { for segno in self .list_slru_segments(SlruKind::Clog, probe_lsn, ctx) .await? @@ -402,16 +446,14 @@ impl Timeline { timestamp_bytes.copy_from_slice(&clog_page[BLCKSZ as usize..]); let timestamp = TimestampTz::from_be_bytes(timestamp_bytes); - if timestamp >= search_timestamp { - *found_larger = true; - return Ok(true); - } else { - *found_smaller = true; + match f(timestamp) { + ControlFlow::Break(b) => return Ok(b), + ControlFlow::Continue(()) => (), } } } } - Ok(false) + Ok(Default::default()) } /// Get a list of SLRU segments @@ -499,6 +541,23 @@ impl Timeline { self.get(CHECKPOINT_KEY, lsn, ctx).await } + pub async fn list_aux_files( + &self, + lsn: Lsn, + ctx: &RequestContext, + ) -> Result, PageReconstructError> { + match self.get(AUX_FILES_KEY, lsn, ctx).await { + Ok(buf) => match AuxFilesDirectory::des(&buf).context("deserialization failure") { + Ok(dir) => Ok(dir.files), + Err(e) => Err(PageReconstructError::from(e)), + }, + Err(e) => { + warn!("Failed to get info about AUX files: {}", e); + Ok(HashMap::new()) + } + } + } + /// Does the same as get_current_logical_size but counted on demand. /// Used to initialize the logical size tracking on startup. /// @@ -616,6 +675,7 @@ impl Timeline { result.add_key(CONTROLFILE_KEY); result.add_key(CHECKPOINT_KEY); + result.add_key(AUX_FILES_KEY); Ok(result.to_keyspace()) } @@ -692,6 +752,12 @@ impl<'a> DatadirModification<'a> { })?; self.put(DBDIR_KEY, Value::Image(buf.into())); + // Create AuxFilesDirectory + let buf = AuxFilesDirectory::ser(&AuxFilesDirectory { + files: HashMap::new(), + })?; + self.put(AUX_FILES_KEY, Value::Image(Bytes::from(buf))); + let buf = TwoPhaseDirectory::ser(&TwoPhaseDirectory { xids: HashSet::new(), })?; @@ -796,6 +862,12 @@ impl<'a> DatadirModification<'a> { // 'true', now write the updated 'dbdirs' map back. let buf = DbDirectory::ser(&dbdir)?; self.put(DBDIR_KEY, Value::Image(buf.into())); + + // Create AuxFilesDirectory as well + let buf = AuxFilesDirectory::ser(&AuxFilesDirectory { + files: HashMap::new(), + })?; + self.put(AUX_FILES_KEY, Value::Image(Bytes::from(buf))); } if r.is_none() { // Create RelDirectory @@ -1120,6 +1192,36 @@ impl<'a> DatadirModification<'a> { Ok(()) } + pub async fn put_file( + &mut self, + path: &str, + content: &[u8], + ctx: &RequestContext, + ) -> anyhow::Result<()> { + let mut dir = match self.get(AUX_FILES_KEY, ctx).await { + Ok(buf) => AuxFilesDirectory::des(&buf)?, + Err(e) => { + warn!("Failed to get info about AUX files: {}", e); + AuxFilesDirectory { + files: HashMap::new(), + } + } + }; + let path = path.to_string(); + if content.is_empty() { + dir.files.remove(&path); + } else { + dir.files.insert(path, Bytes::copy_from_slice(content)); + } + self.put( + AUX_FILES_KEY, + Value::Image(Bytes::from( + AuxFilesDirectory::ser(&dir).context("serialize")?, + )), + ); + Ok(()) + } + /// /// Flush changes accumulated so far to the underlying repository. /// @@ -1255,6 +1357,11 @@ struct RelDirectory { rels: HashSet<(Oid, u8)>, } +#[derive(Debug, Serialize, Deserialize, Default)] +struct AuxFilesDirectory { + files: HashMap, +} + #[derive(Debug, Serialize, Deserialize)] struct RelSizeEntry { nblocks: u32, @@ -1303,10 +1410,12 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; BLCKSZ as usize]); // 02 pg_twophase // // 03 misc -// controlfile +// Controlfile // checkpoint // pg_version // +// 04 aux files +// // Below is a full list of the keyspace allocation: // // DbDir: @@ -1344,6 +1453,11 @@ static ZERO_PAGE: Bytes = Bytes::from_static(&[0u8; BLCKSZ as usize]); // // Checkpoint: // 03 00000000 00000000 00000000 00 00000001 +// +// AuxFiles: +// 03 00000000 00000000 00000000 00 00000002 +// + //-- Section 01: relation data and metadata const DBDIR_KEY: Key = Key { @@ -1567,6 +1681,15 @@ const CHECKPOINT_KEY: Key = Key { field6: 1, }; +const AUX_FILES_KEY: Key = Key { + field1: 0x03, + field2: 0, + field3: 0, + field4: 0, + field5: 0, + field6: 2, +}; + // Reverse mappings for a few Keys. // These are needed by WAL redo manager. diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 88b2423eec..82f82dcd09 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -18,6 +18,7 @@ use pageserver_api::models::TimelineState; use remote_storage::DownloadError; use remote_storage::GenericRemoteStorage; use storage_broker::BrokerClientChannel; +use tokio::runtime::Handle; use tokio::sync::watch; use tokio::task::JoinSet; use tokio_util::sync::CancellationToken; @@ -77,12 +78,12 @@ use crate::tenant::remote_timeline_client::MaybeDeletedIndexPart; use crate::tenant::storage_layer::DeltaLayer; use crate::tenant::storage_layer::ImageLayer; use crate::InitializationOrder; +use crate::METADATA_FILE_NAME; use crate::tenant::timeline::delete::DeleteTimelineFlow; use crate::tenant::timeline::uninit::cleanup_timeline_directory; use crate::virtual_file::VirtualFile; use crate::walredo::PostgresRedoManager; -use crate::walredo::WalRedoManager; use crate::TEMP_FILE_SUFFIX; pub use pageserver_api::models::TenantState; @@ -229,7 +230,7 @@ pub struct Tenant { // with timelines, which in turn may cause dropping replication connection, expiration of wait_for_lsn // timeout... gc_cs: tokio::sync::Mutex<()>, - walredo_mgr: Arc, + walredo_mgr: Arc, // provides access to timeline data sitting in the remote storage pub(crate) remote_storage: Option, @@ -246,67 +247,43 @@ pub struct Tenant { pub(crate) delete_progress: Arc>, } -// We should not blindly overwrite local metadata with remote one. -// For example, consider the following case: -// Image layer is flushed to disk as a new delta layer, we update local metadata and start upload task but after that -// pageserver crashes. During startup we'll load new metadata, and then reset it -// to the state of remote one. But current layermap will have layers from the old -// metadata which is inconsistent. -// And with current logic it wont disgard them during load because during layermap -// load it sees local disk consistent lsn which is ahead of layer lsns. -// If we treat remote as source of truth we need to completely sync with it, -// i e delete local files which are missing on the remote. This will add extra work, -// wal for these layers needs to be reingested for example -// -// So the solution is to take remote metadata only when we're attaching. -pub fn merge_local_remote_metadata<'a>( - local: Option<&'a TimelineMetadata>, - remote: Option<&'a TimelineMetadata>, -) -> anyhow::Result<(&'a TimelineMetadata, bool)> { - match (local, remote) { - (None, None) => anyhow::bail!("we should have either local metadata or remote"), - (Some(local), None) => Ok((local, true)), - // happens if we crash during attach, before writing out the metadata file - (None, Some(remote)) => Ok((remote, false)), - // This is the regular case where we crash/exit before finishing queued uploads. - // Also, it happens if we crash during attach after writing the metadata file - // but before removing the attaching marker file. - (Some(local), Some(remote)) => { - let consistent_lsn_cmp = local - .disk_consistent_lsn() - .cmp(&remote.disk_consistent_lsn()); - let gc_cutoff_lsn_cmp = local - .latest_gc_cutoff_lsn() - .cmp(&remote.latest_gc_cutoff_lsn()); - use std::cmp::Ordering::*; - match (consistent_lsn_cmp, gc_cutoff_lsn_cmp) { - // It wouldn't matter, but pick the local one so that we don't rewrite the metadata file. - (Equal, Equal) => Ok((local, true)), - // Local state is clearly ahead of the remote. - (Greater, Greater) => Ok((local, true)), - // We have local layer files that aren't on the remote, but GC horizon is on par. - (Greater, Equal) => Ok((local, true)), - // Local GC started running but we couldn't sync it to the remote. - (Equal, Greater) => Ok((local, true)), +pub(crate) enum WalRedoManager { + Prod(PostgresRedoManager), + #[cfg(test)] + Test(harness::TestRedoManager), +} - // We always update the local value first, so something else must have - // updated the remote value, probably a different pageserver. - // The control plane is supposed to prevent this from happening. - // Bail out. - (Less, Less) - | (Less, Equal) - | (Equal, Less) - | (Less, Greater) - | (Greater, Less) => { - anyhow::bail!( - r#"remote metadata appears to be ahead of local metadata: -local: - {local:#?} -remote: - {remote:#?} -"# - ); - } +impl From for WalRedoManager { + fn from(mgr: PostgresRedoManager) -> Self { + Self::Prod(mgr) + } +} + +#[cfg(test)] +impl From for WalRedoManager { + fn from(mgr: harness::TestRedoManager) -> Self { + Self::Test(mgr) + } +} + +impl WalRedoManager { + pub async fn request_redo( + &self, + key: crate::repository::Key, + lsn: Lsn, + base_img: Option<(Lsn, bytes::Bytes)>, + records: Vec<(Lsn, crate::walrecord::NeonWalRecord)>, + pg_version: u32, + ) -> anyhow::Result { + match self { + Self::Prod(mgr) => { + mgr.request_redo(key, lsn, base_img, records, pg_version) + .await + } + #[cfg(test)] + Self::Test(mgr) => { + mgr.request_redo(key, lsn, base_img, records, pg_version) + .await } } } @@ -375,11 +352,6 @@ impl Debug for SetStoppingError { } } -struct RemoteStartupData { - index_part: IndexPart, - remote_metadata: TimelineMetadata, -} - #[derive(Debug, thiserror::Error)] pub(crate) enum WaitToBecomeActiveError { WillNotBecomeActive { @@ -420,6 +392,12 @@ pub enum CreateTimelineError { Other(#[from] anyhow::Error), } +/// spawn_attach argument for whether the caller is using attachment markers +pub(super) enum AttachMarkerMode { + Expect, + Ignore, +} + struct TenantDirectoryScan { sorted_timelines_to_load: Vec<(TimelineId, TimelineMetadata)>, timelines_to_resume_deletion: Vec<(TimelineId, Option)>, @@ -446,24 +424,17 @@ impl Tenant { &self, timeline_id: TimelineId, resources: TimelineResources, - remote_startup_data: Option, - local_metadata: Option, + index_part: Option, + metadata: TimelineMetadata, ancestor: Option>, init_order: Option<&InitializationOrder>, _ctx: &RequestContext, ) -> anyhow::Result<()> { let tenant_id = self.tenant_id; - let (up_to_date_metadata, picked_local) = merge_local_remote_metadata( - local_metadata.as_ref(), - remote_startup_data.as_ref().map(|r| &r.remote_metadata), - ) - .context("merge_local_remote_metadata")? - .to_owned(); - let timeline = self.create_timeline_struct( timeline_id, - up_to_date_metadata, + &metadata, ancestor.clone(), resources, init_order, @@ -476,20 +447,11 @@ impl Tenant { ); assert_eq!( disk_consistent_lsn, - up_to_date_metadata.disk_consistent_lsn(), + metadata.disk_consistent_lsn(), "these are used interchangeably" ); - // Save the metadata file to local disk. - if !picked_local { - save_metadata(self.conf, &tenant_id, &timeline_id, up_to_date_metadata) - .await - .context("save_metadata")?; - } - - let index_part = remote_startup_data.as_ref().map(|x| &x.index_part); - - if let Some(index_part) = index_part { + if let Some(index_part) = index_part.as_ref() { timeline .remote_client .as_ref() @@ -502,15 +464,12 @@ impl Tenant { // If control plane retries timeline creation in the meantime, the mgmt API handler // for timeline creation will coalesce on the upload we queue here. let rtc = timeline.remote_client.as_ref().unwrap(); - rtc.init_upload_queue_for_empty_remote(up_to_date_metadata)?; - rtc.schedule_index_upload_for_metadata_update(up_to_date_metadata)?; + rtc.init_upload_queue_for_empty_remote(&metadata)?; + rtc.schedule_index_upload_for_metadata_update(&metadata)?; } timeline - .load_layer_map( - disk_consistent_lsn, - remote_startup_data.map(|x| x.index_part), - ) + .load_layer_map(disk_consistent_lsn, index_part) .await .with_context(|| { format!("Failed to load layermap for timeline {tenant_id}/{timeline_id}") @@ -567,10 +526,13 @@ impl Tenant { resources: TenantSharedResources, attached_conf: AttachedTenantConf, tenants: &'static tokio::sync::RwLock, + expect_marker: AttachMarkerMode, ctx: &RequestContext, ) -> anyhow::Result> { // TODO dedup with spawn_load - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( + conf, tenant_id, + ))); let TenantSharedResources { broker_client, @@ -648,7 +610,7 @@ impl Tenant { } } - match tenant_clone.attach(&ctx).await { + match tenant_clone.attach(&ctx, expect_marker).await { Ok(()) => { info!("attach finished, activating"); tenant_clone.activate(broker_client, None, &ctx); @@ -673,17 +635,23 @@ impl Tenant { /// /// No background tasks are started as part of this routine. /// - async fn attach(self: &Arc, ctx: &RequestContext) -> anyhow::Result<()> { + async fn attach( + self: &Arc, + ctx: &RequestContext, + expect_marker: AttachMarkerMode, + ) -> anyhow::Result<()> { span::debug_assert_current_span_has_tenant_id(); let marker_file = self.conf.tenant_attaching_mark_file_path(&self.tenant_id); - if !tokio::fs::try_exists(&marker_file) - .await - .context("check for existence of marker file")? - { - anyhow::bail!( - "implementation error: marker file should exist at beginning of this function" - ); + if let AttachMarkerMode::Expect = expect_marker { + if !tokio::fs::try_exists(&marker_file) + .await + .context("check for existence of marker file")? + { + anyhow::bail!( + "implementation error: marker file should exist at beginning of this function" + ); + } } // Get list of remote timelines @@ -805,10 +773,12 @@ impl Tenant { .map_err(LoadLocalTimelineError::ResumeDeletion)?; } - std::fs::remove_file(&marker_file) - .with_context(|| format!("unlink attach marker file {marker_file}"))?; - crashsafe::fsync(marker_file.parent().expect("marker file has parent dir")) - .context("fsync tenant directory after unlinking attach marker file")?; + if let AttachMarkerMode::Expect = expect_marker { + std::fs::remove_file(&marker_file) + .with_context(|| format!("unlink attach marker file {marker_file}"))?; + crashsafe::fsync(marker_file.parent().expect("marker file has parent dir")) + .context("fsync tenant directory after unlinking attach marker file")?; + } crate::failpoint_support::sleep_millis_async!("attach-before-activate"); @@ -861,21 +831,23 @@ impl Tenant { None }; - // Even if there is local metadata it cannot be ahead of the remote one - // since we're attaching. Even if we resume interrupted attach remote one - // cannot be older than the local one - let local_metadata = None; + // we can load remote timelines during init, but they are assumed to be so rare that + // initialization order is not passed to here. + let init_order = None; + + // timeline loading after attach expects to find metadata file for each metadata + save_metadata(self.conf, &self.tenant_id, &timeline_id, &remote_metadata) + .await + .context("save_metadata") + .map_err(LoadLocalTimelineError::Load)?; self.timeline_init_and_sync( timeline_id, resources, - Some(RemoteStartupData { - index_part, - remote_metadata, - }), - local_metadata, + Some(index_part), + remote_metadata, ancestor, - None, + init_order, ctx, ) .await @@ -887,7 +859,9 @@ impl Tenant { tenant_id: TenantId, reason: String, ) -> Arc { - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( + conf, tenant_id, + ))); Arc::new(Tenant::new( TenantState::Broken { reason, @@ -926,7 +900,9 @@ impl Tenant { let broker_client = resources.broker_client; let remote_storage = resources.remote_storage; - let wal_redo_manager = Arc::new(PostgresRedoManager::new(conf, tenant_id)); + let wal_redo_manager = Arc::new(WalRedoManager::from(PostgresRedoManager::new( + conf, tenant_id, + ))); let tenant = Tenant::new( TenantState::Loading, conf, @@ -1349,8 +1325,8 @@ impl Tenant { LoadLocalTimelineError::Load(source) => { // We tried to load deleted timeline, this is a bug. return Err(anyhow::anyhow!(source).context( - "This is a bug. We tried to load deleted timeline which is wrong and loading failed. Timeline: {timeline_id}" - )); + format!("This is a bug. We tried to load deleted timeline which is wrong and loading failed. Timeline: {timeline_id}") + )); } LoadLocalTimelineError::ResumeDeletion(source) => { // Make sure resumed deletion wont fail loading for entire tenant. @@ -1384,6 +1360,11 @@ impl Tenant { let mut resources = self.build_timeline_resources(timeline_id); + struct RemoteStartupData { + index_part: IndexPart, + remote_metadata: TimelineMetadata, + } + let (remote_startup_data, remote_client) = match preload { Some(preload) => { let TimelinePreload { @@ -1439,7 +1420,7 @@ impl Tenant { ) } Err(DownloadError::NotFound) => { - info!("no index file was found on the remote, found_delete_mark: {found_delete_mark}"); + info!(found_delete_mark, "no index file was found on the remote, resuming deletion or cleaning unuploaded up"); if found_delete_mark { // We could've resumed at a point where remote index was deleted, but metadata file wasnt. @@ -1453,14 +1434,73 @@ impl Tenant { .map_err(LoadLocalTimelineError::ResumeDeletion); } - // We're loading fresh timeline that didnt yet make it into remote. - (None, Some(remote_client)) + // as the remote index_part.json did not exist, this timeline is a + // not-yet-uploaded one. it should be deleted now, because the branching might + // not have been valid as it's ancestor may have been restored to earlier state + // as well. in practice, control plane will keep retrying. + // + // first ensure that the un-uploaded timeline looks like it should, as in we + // are not accidentially deleting a timeline which was ever active: + // - root timelines have metadata and one possibly partial layer + // - branched timelines have metadata + // + // if the timeline does not look like expected, fail loading of the tenant. + // cleaning the timeline up manually and reloading the tenant is possible via + // the above log message. + let path = self.conf.timeline_path(&self.tenant_id, &timeline_id); + + let span = tracing::Span::current(); + + return tokio::task::spawn_blocking({ + move || { + use std::str::FromStr; + use crate::tenant::storage_layer::LayerFileName; + + let _e = span.entered(); + let mut metadata = false; + let mut layers = 0; + let mut others = 0; + for dentry in path.read_dir_utf8()? { + let dentry = dentry?; + let file_name = dentry.file_name(); + + if file_name == METADATA_FILE_NAME { + metadata = true; + continue; + } + + if LayerFileName::from_str(file_name).is_ok() + { + layers += 1; + continue; + } + + others += 1; + } + + // bootstrapped have the one image layer file, or one partial temp + // file, branched have just the metadata + if !(metadata && layers + others <= 1) { + anyhow::bail!("unexpected assumed unuploaded, never been active timeline: found metadata={}, layers={}, others={}", metadata, layers, others); + } + + let tmp_path = + path.with_file_name(format!("{timeline_id}{}", TEMP_FILE_SUFFIX)); + std::fs::rename(path, &tmp_path)?; + std::fs::remove_dir_all(&tmp_path)?; + Ok(()) + } + }) + .await + .map_err(anyhow::Error::new) + .and_then(|x| x) + .context("delete assumed unuploaded fresh timeline") + .map_err(LoadLocalTimelineError::Load); } Err(e) => return Err(LoadLocalTimelineError::Load(anyhow::Error::new(e))), } } None => { - // No remote client if found_delete_mark { // There is no remote client, we found local metadata. // Continue cleaning up local disk. @@ -1492,11 +1532,27 @@ impl Tenant { None }; + let (index_part, metadata) = match remote_startup_data { + Some(RemoteStartupData { + index_part, + remote_metadata, + }) => { + // always choose the remote metadata to be crash consistent (see RFC 27) + save_metadata(self.conf, &self.tenant_id, &timeline_id, &remote_metadata) + .await + .context("save_metadata") + .map_err(LoadLocalTimelineError::Load)?; + + (Some(index_part), remote_metadata) + } + None => (None, local_metadata), + }; + self.timeline_init_and_sync( timeline_id, resources, - remote_startup_data, - Some(local_metadata), + index_part, + metadata, ancestor, init_order, ctx, @@ -2411,7 +2467,7 @@ impl Tenant { state: TenantState, conf: &'static PageServerConf, attached_conf: AttachedTenantConf, - walredo_mgr: Arc, + walredo_mgr: Arc, tenant_id: TenantId, remote_storage: Option, deletion_queue_client: DeletionQueueClient, @@ -2559,6 +2615,7 @@ impl Tenant { ) -> anyhow::Result<()> { let legacy_config_path = conf.tenant_config_path(tenant_id); let config_path = conf.tenant_location_config_path(tenant_id); + Self::persist_tenant_config_at(tenant_id, &config_path, &legacy_config_path, location_conf) .await } @@ -2597,12 +2654,20 @@ impl Tenant { // Convert the config to a toml file. conf_content += &toml_edit::ser::to_string_pretty(&location_conf)?; - let conf_content = conf_content.as_bytes(); - let temp_path = path_with_suffix_extension(config_path, TEMP_FILE_SUFFIX); - VirtualFile::crashsafe_overwrite(config_path, &temp_path, conf_content) - .await - .with_context(|| format!("write tenant {tenant_id} config to {config_path}"))?; + + let tenant_id = *tenant_id; + let config_path = config_path.to_owned(); + tokio::task::spawn_blocking(move || { + Handle::current().block_on(async move { + let conf_content = conf_content.as_bytes(); + VirtualFile::crashsafe_overwrite(&config_path, &temp_path, conf_content) + .await + .with_context(|| format!("write tenant {tenant_id} config to {config_path}")) + }) + }) + .await??; + Ok(()) } @@ -2624,12 +2689,21 @@ impl Tenant { // Convert the config to a toml file. conf_content += &toml_edit::ser::to_string(&tenant_conf)?; - let conf_content = conf_content.as_bytes(); - let temp_path = path_with_suffix_extension(target_config_path, TEMP_FILE_SUFFIX); - VirtualFile::crashsafe_overwrite(target_config_path, &temp_path, conf_content) - .await - .with_context(|| format!("write tenant {tenant_id} config to {target_config_path}"))?; + + let tenant_id = *tenant_id; + let target_config_path = target_config_path.to_owned(); + tokio::task::spawn_blocking(move || { + Handle::current().block_on(async move { + let conf_content = conf_content.as_bytes(); + VirtualFile::crashsafe_overwrite(&target_config_path, &temp_path, conf_content) + .await + .with_context(|| { + format!("write tenant {tenant_id} config to {target_config_path}") + }) + }) + }) + .await??; Ok(()) } @@ -3542,7 +3616,7 @@ pub async fn dump_layerfile_from_path( } #[cfg(test)] -pub mod harness { +pub(crate) mod harness { use bytes::{Bytes, BytesMut}; use once_cell::sync::OnceCell; use std::fs; @@ -3553,7 +3627,6 @@ pub mod harness { use crate::deletion_queue::mock::MockDeletionQueue; use crate::{ config::PageServerConf, repository::Key, tenant::Tenant, walrecord::NeonWalRecord, - walredo::WalRedoManager, }; use super::*; @@ -3614,17 +3687,21 @@ pub mod harness { static LOG_HANDLE: OnceCell<()> = OnceCell::new(); + pub(crate) fn setup_logging() { + LOG_HANDLE.get_or_init(|| { + logging::init( + logging::LogFormat::Test, + // enable it in case the tests exercise code paths that use + // debug_assert_current_span_has_tenant_and_timeline_id + logging::TracingErrorLayerEnablement::EnableWithRustLogFilter, + ) + .expect("Failed to init test logging") + }); + } + impl TenantHarness { pub fn create(test_name: &'static str) -> anyhow::Result { - LOG_HANDLE.get_or_init(|| { - logging::init( - logging::LogFormat::Test, - // enable it in case in case the tests exercise code paths that use - // debug_assert_current_span_has_tenant_and_timeline_id - logging::TracingErrorLayerEnablement::EnableWithRustLogFilter, - ) - .expect("Failed to init test logging") - }); + setup_logging(); let repo_dir = PageServerConf::test_repo_dir(test_name); let _ = fs::remove_dir_all(&repo_dir); @@ -3682,7 +3759,7 @@ pub mod harness { } pub async fn try_load(&self, ctx: &RequestContext) -> anyhow::Result> { - let walredo_mgr = Arc::new(TestRedoManager); + let walredo_mgr = Arc::new(WalRedoManager::from(TestRedoManager)); let tenant = Arc::new(Tenant::new( TenantState::Loading, @@ -3716,10 +3793,10 @@ pub mod harness { } // Mock WAL redo manager that doesn't do much - pub struct TestRedoManager; + pub(crate) struct TestRedoManager; - impl WalRedoManager for TestRedoManager { - fn request_redo( + impl TestRedoManager { + pub async fn request_redo( &self, key: Key, lsn: Lsn, diff --git a/pageserver/src/tenant/delete.rs b/pageserver/src/tenant/delete.rs index 0db6213bbf..989f2cd779 100644 --- a/pageserver/src/tenant/delete.rs +++ b/pageserver/src/tenant/delete.rs @@ -458,7 +458,10 @@ impl DeleteTenantFlow { .await .expect("cant be stopping or broken"); - tenant.attach(ctx).await.context("attach")?; + tenant + .attach(ctx, super::AttachMarkerMode::Expect) + .await + .context("attach")?; Self::background( guard, diff --git a/pageserver/src/tenant/ephemeral_file.rs b/pageserver/src/tenant/ephemeral_file.rs index 5b99a1dd03..9a06d9df61 100644 --- a/pageserver/src/tenant/ephemeral_file.rs +++ b/pageserver/src/tenant/ephemeral_file.rs @@ -354,8 +354,7 @@ mod tests { } // Test a large blob that spans multiple pages - let mut large_data = Vec::new(); - large_data.resize(20000, 0); + let mut large_data = vec![0; 20000]; thread_rng().fill_bytes(&mut large_data); let pos_large = file.write_blob(&large_data, &ctx).await?; let result = file.block_cursor().read_blob(pos_large, &ctx).await?; diff --git a/pageserver/src/tenant/mgr.rs b/pageserver/src/tenant/mgr.rs index 83ba49e1f5..380b610a4c 100644 --- a/pageserver/src/tenant/mgr.rs +++ b/pageserver/src/tenant/mgr.rs @@ -1,7 +1,7 @@ //! This module acts as a switchboard to access different repositories managed by this //! page server. -use camino::{Utf8Path, Utf8PathBuf}; +use camino::{Utf8DirEntry, Utf8Path, Utf8PathBuf}; use rand::{distributions::Alphanumeric, Rng}; use std::collections::{hash_map, HashMap}; use std::sync::Arc; @@ -27,7 +27,8 @@ use crate::task_mgr::{self, TaskKind}; use crate::tenant::config::{AttachmentMode, LocationConf, LocationMode, TenantConfOpt}; use crate::tenant::delete::DeleteTenantFlow; use crate::tenant::{ - create_tenant_files, AttachedTenantConf, CreateTenantFilesMode, Tenant, TenantState, + create_tenant_files, AttachMarkerMode, AttachedTenantConf, CreateTenantFilesMode, Tenant, + TenantState, }; use crate::{InitializationOrder, IGNORED_TENANT_FILE_NAME, TEMP_FILE_SUFFIX}; @@ -151,6 +152,49 @@ async fn safe_rename_tenant_dir(path: impl AsRef) -> std::io::Result> = Lazy::new(|| RwLock::new(TenantsMap::Initializing)); +/// Create a directory, including parents. This does no fsyncs and makes +/// no guarantees about the persistence of the resulting metadata: for +/// use when creating dirs for use as cache. +async fn unsafe_create_dir_all(path: &Utf8PathBuf) -> std::io::Result<()> { + let mut dirs_to_create = Vec::new(); + let mut path: &Utf8Path = path.as_ref(); + + // Figure out which directories we need to create. + loop { + let meta = tokio::fs::metadata(path).await; + match meta { + Ok(metadata) if metadata.is_dir() => break, + Ok(_) => { + return Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + format!("non-directory found in path: {path}"), + )); + } + Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + + dirs_to_create.push(path); + + match path.parent() { + Some(parent) => path = parent, + None => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("can't find parent of path '{path}'"), + )); + } + } + } + + // Create directories from parent to child. + for &path in dirs_to_create.iter().rev() { + tokio::fs::create_dir(path).await?; + } + + Ok(()) +} + fn emergency_generations( tenant_confs: &HashMap>, ) -> HashMap { @@ -212,83 +256,99 @@ async fn init_load_generations( Ok(Some(generations)) } +/// Given a directory discovered in the pageserver's tenants/ directory, attempt +/// to load a tenant config from it. +/// +/// If file is missing, return Ok(None) +fn load_tenant_config( + conf: &'static PageServerConf, + dentry: Utf8DirEntry, +) -> anyhow::Result)>> { + let tenant_dir_path = dentry.path().to_path_buf(); + if crate::is_temporary(&tenant_dir_path) { + info!("Found temporary tenant directory, removing: {tenant_dir_path}"); + // No need to use safe_remove_tenant_dir_all because this is already + // a temporary path + if let Err(e) = std::fs::remove_dir_all(&tenant_dir_path) { + error!( + "Failed to remove temporary directory '{}': {:?}", + tenant_dir_path, e + ); + } + return Ok(None); + } + + // This case happens if we crash during attachment before writing a config into the dir + let is_empty = tenant_dir_path + .is_empty_dir() + .with_context(|| format!("Failed to check whether {tenant_dir_path:?} is an empty dir"))?; + if is_empty { + info!("removing empty tenant directory {tenant_dir_path:?}"); + if let Err(e) = std::fs::remove_dir(&tenant_dir_path) { + error!( + "Failed to remove empty tenant directory '{}': {e:#}", + tenant_dir_path + ) + } + return Ok(None); + } + + let tenant_ignore_mark_file = tenant_dir_path.join(IGNORED_TENANT_FILE_NAME); + if tenant_ignore_mark_file.exists() { + info!("Found an ignore mark file {tenant_ignore_mark_file:?}, skipping the tenant"); + return Ok(None); + } + + let tenant_id = match tenant_dir_path + .file_name() + .unwrap_or_default() + .parse::() + { + Ok(id) => id, + Err(_) => { + warn!("Invalid tenant path (garbage in our repo directory?): {tenant_dir_path}",); + return Ok(None); + } + }; + + Ok(Some(( + tenant_id, + Tenant::load_tenant_config(conf, &tenant_id), + ))) +} + /// Initial stage of load: walk the local tenants directory, clean up any temp files, /// and load configurations for the tenants we found. +/// +/// Do this in parallel, because we expect 10k+ tenants, so serial execution can take +/// seconds even on reasonably fast drives. async fn init_load_tenant_configs( conf: &'static PageServerConf, ) -> anyhow::Result>> { let tenants_dir = conf.tenants_path(); - let mut dir_entries = tenants_dir - .read_dir_utf8() - .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?; + let dentries = tokio::task::spawn_blocking(move || -> anyhow::Result> { + let dir_entries = tenants_dir + .read_dir_utf8() + .with_context(|| format!("Failed to list tenants dir {tenants_dir:?}"))?; + + Ok(dir_entries.collect::, std::io::Error>>()?) + }) + .await??; let mut configs = HashMap::new(); - loop { - match dir_entries.next() { - None => break, - Some(Ok(dentry)) => { - let tenant_dir_path = dentry.path().to_path_buf(); - if crate::is_temporary(&tenant_dir_path) { - info!("Found temporary tenant directory, removing: {tenant_dir_path}"); - // No need to use safe_remove_tenant_dir_all because this is already - // a temporary path - if let Err(e) = fs::remove_dir_all(&tenant_dir_path).await { - error!( - "Failed to remove temporary directory '{}': {:?}", - tenant_dir_path, e - ); - } - continue; - } + let mut join_set = JoinSet::new(); + for dentry in dentries { + join_set.spawn_blocking(move || load_tenant_config(conf, dentry)); + } - // This case happens if we: - // * crash during attach before creating the attach marker file - // * crash during tenant delete before removing tenant directory - let is_empty = tenant_dir_path.is_empty_dir().with_context(|| { - format!("Failed to check whether {tenant_dir_path:?} is an empty dir") - })?; - if is_empty { - info!("removing empty tenant directory {tenant_dir_path:?}"); - if let Err(e) = fs::remove_dir(&tenant_dir_path).await { - error!( - "Failed to remove empty tenant directory '{}': {e:#}", - tenant_dir_path - ) - } - continue; - } - - let tenant_ignore_mark_file = tenant_dir_path.join(IGNORED_TENANT_FILE_NAME); - if tenant_ignore_mark_file.exists() { - info!("Found an ignore mark file {tenant_ignore_mark_file:?}, skipping the tenant"); - continue; - } - - let tenant_id = match tenant_dir_path - .file_name() - .unwrap_or_default() - .parse::() - { - Ok(id) => id, - Err(_) => { - warn!( - "Invalid tenant path (garbage in our repo directory?): {tenant_dir_path}", - ); - continue; - } - }; - - configs.insert(tenant_id, Tenant::load_tenant_config(conf, &tenant_id)); - } - Some(Err(e)) => { - // An error listing the top level directory indicates serious problem - // with local filesystem: we will fail to load, and fail to start. - anyhow::bail!(e); - } + while let Some(r) = join_set.join_next().await { + if let Some((tenant_id, tenant_config)) = r?? { + configs.insert(tenant_id, tenant_config); } } + Ok(configs) } @@ -446,7 +506,15 @@ pub(crate) fn schedule_local_tenant_processing( "attaching mark file present but no remote storage configured".to_string(), ) } else { - match Tenant::spawn_attach(conf, tenant_id, resources, location_conf, tenants, ctx) { + match Tenant::spawn_attach( + conf, + tenant_id, + resources, + location_conf, + tenants, + AttachMarkerMode::Expect, + ctx, + ) { Ok(tenant) => tenant, Err(e) => { error!("Failed to spawn_attach tenant {tenant_id}, reason: {e:#}"); @@ -655,7 +723,7 @@ pub(crate) async fn set_new_tenant_config( Ok(()) } -#[instrument(skip_all, fields(tenant_id, new_location_config))] +#[instrument(skip_all, fields(%tenant_id))] pub(crate) async fn upsert_location( conf: &'static PageServerConf, tenant_id: TenantId, @@ -734,36 +802,61 @@ pub(crate) async fn upsert_location( } let new_slot = match &new_location_config.mode { - LocationMode::Secondary(_) => TenantSlot::Secondary, - LocationMode::Attached(_attach_config) => { - // Do a schedule_local_tenant_processing - // FIXME: should avoid doing this disk I/O inside the TenantsMap lock, - // we have the same problem in load_tenant/attach_tenant. Probably - // need a lock in TenantSlot to fix this. + LocationMode::Secondary(_) => { + let tenant_path = conf.tenant_path(&tenant_id); + // Directory doesn't need to be fsync'd because if we crash it can + // safely be recreated next time this tenant location is configured. + unsafe_create_dir_all(&tenant_path) + .await + .with_context(|| format!("Creating {tenant_path}"))?; + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) .await .map_err(SetNewTenantConfigError::Persist)?; - let tenant_path = conf.tenant_path(&tenant_id); - let resources = TenantSharedResources { - broker_client, - remote_storage, - deletion_queue_client, - }; - let new_tenant = schedule_local_tenant_processing( + + TenantSlot::Secondary + } + LocationMode::Attached(_attach_config) => { + // FIXME: should avoid doing this disk I/O inside the TenantsMap lock, + // we have the same problem in load_tenant/attach_tenant. Probably + // need a lock in TenantSlot to fix this. + let timelines_path = conf.timelines_path(&tenant_id); + + // Directory doesn't need to be fsync'd because we do not depend on + // it to exist after crashes: it may be recreated when tenant is + // re-attached, see https://github.com/neondatabase/neon/issues/5550 + unsafe_create_dir_all(&timelines_path) + .await + .with_context(|| format!("Creating {timelines_path}"))?; + + Tenant::persist_tenant_config(conf, &tenant_id, &new_location_config) + .await + .map_err(SetNewTenantConfigError::Persist)?; + + let tenant = match Tenant::spawn_attach( conf, tenant_id, - &tenant_path, + TenantSharedResources { + broker_client, + remote_storage, + deletion_queue_client, + }, AttachedTenantConf::try_from(new_location_config)?, - resources, - None, &TENANTS, + // The LocationConf API does not use marker files, because we have Secondary + // locations where the directory's existence is not a signal that it contains + // all timelines. See https://github.com/neondatabase/neon/issues/5550 + AttachMarkerMode::Ignore, ctx, - ) - .with_context(|| { - format!("Failed to schedule tenant processing in path {tenant_path:?}") - })?; + ) { + Ok(tenant) => tenant, + Err(e) => { + error!("Failed to spawn_attach tenant {tenant_id}, reason: {e:#}"); + Tenant::create_broken_tenant(conf, tenant_id, format!("{e:#}")) + } + }; - TenantSlot::Attached(new_tenant) + TenantSlot::Attached(tenant) } }; @@ -771,7 +864,6 @@ pub(crate) async fn upsert_location( }) .await?; } - Ok(()) } diff --git a/pageserver/src/tenant/remote_timeline_client/download.rs b/pageserver/src/tenant/remote_timeline_client/download.rs index ef8d217be4..ca6e3293d6 100644 --- a/pageserver/src/tenant/remote_timeline_client/download.rs +++ b/pageserver/src/tenant/remote_timeline_client/download.rs @@ -18,7 +18,7 @@ use crate::config::PageServerConf; use crate::tenant::remote_timeline_client::{remote_layer_path, remote_timelines_path}; use crate::tenant::storage_layer::LayerFileName; use crate::tenant::timeline::span::debug_assert_current_span_has_tenant_and_timeline_id; -use crate::tenant::Generation; +use crate::tenant::{Generation, TENANT_DELETED_MARKER_FILE_NAME}; use remote_storage::{DownloadError, GenericRemoteStorage}; use utils::crashsafe::path_with_suffix_extension; use utils::id::{TenantId, TimelineId}; @@ -190,6 +190,12 @@ pub async fn list_remote_timelines( let mut timeline_ids = HashSet::new(); for timeline_remote_storage_key in timelines { + if timeline_remote_storage_key.object_name() == Some(TENANT_DELETED_MARKER_FILE_NAME) { + // A `deleted` key within `timelines/` is a marker file, not a timeline. Ignore it. + // This code will be removed in https://github.com/neondatabase/neon/pull/5580 + continue; + } + let object_name = timeline_remote_storage_key.object_name().ok_or_else(|| { anyhow::anyhow!("failed to get timeline id for remote tenant {tenant_id}") })?; diff --git a/pageserver/src/tenant/storage_layer/filename.rs b/pageserver/src/tenant/storage_layer/filename.rs index 9fb0c23dd7..a98be0842b 100644 --- a/pageserver/src/tenant/storage_layer/filename.rs +++ b/pageserver/src/tenant/storage_layer/filename.rs @@ -226,6 +226,14 @@ impl LayerFileName { _ => false, } } + + pub(crate) fn kind(&self) -> &'static str { + use LayerFileName::*; + match self { + Delta(_) => "delta", + Image(_) => "image", + } + } } impl fmt::Display for LayerFileName { diff --git a/pageserver/src/tenant/storage_layer/remote_layer.rs b/pageserver/src/tenant/storage_layer/remote_layer.rs index cafe5f6bb6..94bebb5f6e 100644 --- a/pageserver/src/tenant/storage_layer/remote_layer.rs +++ b/pageserver/src/tenant/storage_layer/remote_layer.rs @@ -25,7 +25,7 @@ use super::{ }; /// RemoteLayer is a not yet downloaded [`ImageLayer`] or -/// [`DeltaLayer`](super::DeltaLayer). +/// [`DeltaLayer`]. /// /// RemoteLayer might be downloaded on-demand during operations which are /// allowed download remote layers and during which, it gets replaced with a diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index a6f92646a9..2c76155e2a 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -81,7 +81,6 @@ use crate::repository::GcResult; use crate::repository::{Key, Value}; use crate::task_mgr; use crate::task_mgr::TaskKind; -use crate::walredo::WalRedoManager; use crate::ZERO_PAGE; use self::delete::DeleteTimelineFlow; @@ -201,7 +200,7 @@ pub struct Timeline { last_freeze_ts: RwLock, // WAL redo manager - walredo_mgr: Arc, + walredo_mgr: Arc, /// Remote storage client. /// See [`remote_timeline_client`](super::remote_timeline_client) module comment for details. @@ -1471,7 +1470,7 @@ impl Timeline { timeline_id: TimelineId, tenant_id: TenantId, generation: Generation, - walredo_mgr: Arc, + walredo_mgr: Arc, resources: TimelineResources, pg_version: u32, initial_logical_size_can_start: Option, @@ -1700,7 +1699,7 @@ impl Timeline { disk_consistent_lsn: Lsn, index_part: Option, ) -> anyhow::Result<()> { - use init::{Decision::*, Discovered, FutureLayer}; + use init::{Decision::*, Discovered, DismissedLayer}; use LayerFileName::*; let mut guard = self.layers.write().await; @@ -1716,7 +1715,7 @@ impl Timeline { // Copy to move into the task we're about to spawn let generation = self.generation; - let (loaded_layers, to_sync, total_physical_size) = tokio::task::spawn_blocking({ + let (loaded_layers, needs_cleanup, total_physical_size) = tokio::task::spawn_blocking({ move || { let _g = span.entered(); let discovered = init::scan_timeline_dir(&timeline_path)?; @@ -1765,7 +1764,6 @@ impl Timeline { ); let mut loaded_layers = Vec::new(); - let mut needs_upload = Vec::new(); let mut needs_cleanup = Vec::new(); let mut total_physical_size = 0; @@ -1786,7 +1784,7 @@ impl Timeline { } } Ok(decision) => decision, - Err(FutureLayer { local }) => { + Err(DismissedLayer::Future { local }) => { if local.is_some() { path.push(name.file_name()); init::cleanup_future_layer(&path, &name, disk_consistent_lsn)?; @@ -1795,6 +1793,13 @@ impl Timeline { needs_cleanup.push(name); continue; } + Err(DismissedLayer::LocalOnly(local)) => { + path.push(name.file_name()); + init::cleanup_local_only_file(&path, &name, &local)?; + path.pop(); + // this file never existed remotely, we will have to do rework + continue; + } }; match &name { @@ -1803,14 +1808,16 @@ impl Timeline { } let status = match &decision { - UseLocal(_) | NeedsUpload(_) => LayerResidenceStatus::Resident, + UseLocal(_) => LayerResidenceStatus::Resident, Evicted(_) | UseRemote { .. } => LayerResidenceStatus::Evicted, }; + tracing::debug!(layer=%name, ?decision, ?status, "applied"); + let stats = LayerAccessStats::for_loading_layer(status); let layer: Arc = match (name, &decision) { - (Delta(d), UseLocal(m) | NeedsUpload(m)) => { + (Delta(d), UseLocal(m)) => { total_physical_size += m.file_size(); Arc::new(DeltaLayer::new( conf, @@ -1821,7 +1828,7 @@ impl Timeline { stats, )) } - (Image(i), UseLocal(m) | NeedsUpload(m)) => { + (Image(i), UseLocal(m)) => { total_physical_size += m.file_size(); Arc::new(ImageLayer::new( conf, @@ -1840,17 +1847,9 @@ impl Timeline { ), }; - if let NeedsUpload(m) = decision { - needs_upload.push((layer.clone(), m)); - } - loaded_layers.push(layer); } - Ok(( - loaded_layers, - (needs_upload, needs_cleanup), - total_physical_size, - )) + Ok((loaded_layers, needs_cleanup, total_physical_size)) } }) .await @@ -1862,10 +1861,6 @@ impl Timeline { guard.initialize_local_layers(loaded_layers, disk_consistent_lsn + 1); if let Some(rtc) = self.remote_client.as_ref() { - let (needs_upload, needs_cleanup) = to_sync; - for (layer, m) in needs_upload { - rtc.schedule_layer_file_upload(&layer.layer_desc().filename(), &m)?; - } rtc.schedule_layer_file_deletion(needs_cleanup)?; rtc.schedule_index_upload_for_file_changes()?; // Tenant::create_timeline will wait for these uploads to happen before returning, or @@ -4328,6 +4323,7 @@ impl Timeline { let img = match self .walredo_mgr .request_redo(key, request_lsn, data.img, data.records, self.pg_version) + .await .context("Failed to reconstruct a page image:") { Ok(img) => img, diff --git a/pageserver/src/tenant/timeline/init.rs b/pageserver/src/tenant/timeline/init.rs index 3902afe89a..96bf847fbb 100644 --- a/pageserver/src/tenant/timeline/init.rs +++ b/pageserver/src/tenant/timeline/init.rs @@ -72,7 +72,7 @@ pub(super) fn scan_timeline_dir(path: &Utf8Path) -> anyhow::Result, +pub(super) enum DismissedLayer { + /// The related layer is is in future compared to disk_consistent_lsn, it must not be loaded. + Future { + /// The local metadata. `None` if the layer is only known through [`IndexPart`]. + local: Option, + }, + /// The layer only exists locally. + /// + /// In order to make crash safe updates to layer map, we must dismiss layers which are only + /// found locally or not yet included in the remote `index_part.json`. + LocalOnly(LayerFileMetadata), } /// Merges local discoveries and remote [`IndexPart`] to a collection of decisions. -/// -/// This function should not gain additional reasons to fail than [`FutureLayer`], consider adding -/// the checks earlier to [`scan_timeline_dir`]. pub(super) fn reconcile( discovered: Vec<(LayerFileName, u64)>, index_part: Option<&IndexPart>, disk_consistent_lsn: Lsn, generation: Generation, -) -> Vec<(LayerFileName, Result)> { +) -> Vec<(LayerFileName, Result)> { use Decision::*; // name => (local, remote) @@ -142,17 +145,19 @@ pub(super) fn reconcile( .into_iter() .map(|(name, (local, remote))| { let decision = if name.is_in_future(disk_consistent_lsn) { - Err(FutureLayer { local }) + Err(DismissedLayer::Future { local }) } else { - Ok(match (local, remote) { - (Some(local), Some(remote)) if local != remote => UseRemote { local, remote }, - (Some(x), Some(_)) => UseLocal(x), - (None, Some(x)) => Evicted(x), - (Some(x), None) => NeedsUpload(x), + match (local, remote) { + (Some(local), Some(remote)) if local != remote => { + Ok(UseRemote { local, remote }) + } + (Some(x), Some(_)) => Ok(UseLocal(x)), + (None, Some(x)) => Ok(Evicted(x)), + (Some(x), None) => Err(DismissedLayer::LocalOnly(x)), (None, None) => { unreachable!("there must not be any non-local non-remote files") } - }) + } }; (name, decision) @@ -192,14 +197,21 @@ pub(super) fn cleanup_future_layer( name: &LayerFileName, disk_consistent_lsn: Lsn, ) -> anyhow::Result<()> { - use LayerFileName::*; - let kind = match name { - Delta(_) => "delta", - Image(_) => "image", - }; // future image layers are allowed to be produced always for not yet flushed to disk // lsns stored in InMemoryLayer. + let kind = name.kind(); tracing::info!("found future {kind} layer {name} disk_consistent_lsn is {disk_consistent_lsn}"); - crate::tenant::timeline::rename_to_backup(path)?; + std::fs::remove_file(path)?; + Ok(()) +} + +pub(super) fn cleanup_local_only_file( + path: &Utf8Path, + name: &LayerFileName, + local: &LayerFileMetadata, +) -> anyhow::Result<()> { + let kind = name.kind(); + tracing::info!("found local-only {kind} layer {name}, metadata {local:?}"); + std::fs::remove_file(path)?; Ok(()) } diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index d290715938..fb1dbcd6ba 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -338,11 +338,20 @@ impl<'a> WalIngest<'a> { } else if decoded.xl_rmid == pg_constants::RM_LOGICALMSG_ID { let info = decoded.xl_info & pg_constants::XLR_RMGR_INFO_MASK; if info == pg_constants::XLOG_LOGICAL_MESSAGE { - // This is a convenient way to make the WAL ingestion pause at - // particular point in the WAL. For more fine-grained control, - // we could peek into the message and only pause if it contains - // a particular string, for example, but this is enough for now. - crate::failpoint_support::sleep_millis_async!("wal-ingest-logical-message-sleep"); + let xlrec = XlLogicalMessage::decode(&mut buf); + let prefix = std::str::from_utf8(&buf[0..xlrec.prefix_size - 1])?; + let message = &buf[xlrec.prefix_size..xlrec.prefix_size + xlrec.message_size]; + if prefix == "neon-test" { + // This is a convenient way to make the WAL ingestion pause at + // particular point in the WAL. For more fine-grained control, + // we could peek into the message and only pause if it contains + // a particular string, for example, but this is enough for now. + crate::failpoint_support::sleep_millis_async!( + "wal-ingest-logical-message-sleep" + ); + } else if let Some(path) = prefix.strip_prefix("neon-file:") { + modification.put_file(path, message, ctx).await?; + } } } @@ -459,7 +468,6 @@ impl<'a> WalIngest<'a> { } } else if info == pg_constants::XLOG_HEAP_DELETE { let xlrec = v14::XlHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } @@ -527,7 +535,6 @@ impl<'a> WalIngest<'a> { } } else if info == pg_constants::XLOG_HEAP_DELETE { let xlrec = v15::XlHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } @@ -595,7 +602,6 @@ impl<'a> WalIngest<'a> { } } else if info == pg_constants::XLOG_HEAP_DELETE { let xlrec = v16::XlHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } @@ -771,7 +777,6 @@ impl<'a> WalIngest<'a> { } pg_constants::XLOG_NEON_HEAP_DELETE => { let xlrec = v16::rm_neon::XlNeonHeapDelete::decode(buf); - assert_eq!(0, buf.remaining()); if (xlrec.flags & pg_constants::XLH_DELETE_ALL_VISIBLE_CLEARED) != 0 { new_heap_blkno = Some(decoded.blocks[0].blkno); } diff --git a/pageserver/src/walrecord.rs b/pageserver/src/walrecord.rs index 9c2e522f17..ff6bc9194b 100644 --- a/pageserver/src/walrecord.rs +++ b/pageserver/src/walrecord.rs @@ -748,6 +748,26 @@ impl XlMultiXactTruncate { } } +#[repr(C)] +#[derive(Debug)] +pub struct XlLogicalMessage { + pub db_id: Oid, + pub transactional: bool, + pub prefix_size: usize, + pub message_size: usize, +} + +impl XlLogicalMessage { + pub fn decode(buf: &mut Bytes) -> XlLogicalMessage { + XlLogicalMessage { + db_id: buf.get_u32_le(), + transactional: buf.get_u32_le() != 0, // 4-bytes alignment + prefix_size: buf.get_u64_le() as usize, + message_size: buf.get_u64_le() as usize, + } + } +} + /// Main routine to decode a WAL record and figure out which blocks are modified // // See xlogrecord.h for details diff --git a/pageserver/src/walredo.rs b/pageserver/src/walredo.rs index 28e5f997bb..cffd912e16 100644 --- a/pageserver/src/walredo.rs +++ b/pageserver/src/walredo.rs @@ -26,15 +26,15 @@ use serde::Serialize; use std::collections::VecDeque; use std::io; use std::io::prelude::*; -use std::io::{Error, ErrorKind}; use std::ops::{Deref, DerefMut}; -use std::os::unix::io::{AsRawFd, RawFd}; +use std::os::unix::io::AsRawFd; use std::os::unix::prelude::CommandExt; use std::process::Stdio; -use std::process::{Child, ChildStderr, ChildStdin, ChildStdout, Command}; -use std::sync::{Mutex, MutexGuard}; +use std::process::{Child, ChildStdin, ChildStdout, Command}; +use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use std::time::Duration; use std::time::Instant; +use tokio_util::sync::CancellationToken; use tracing::*; use utils::{bin_ser::BeSer, id::TenantId, lsn::Lsn, nonblock::set_nonblock}; @@ -48,7 +48,6 @@ use crate::metrics::{ }; use crate::pgdatadir_mapping::{key_to_rel_block, key_to_slru_block}; use crate::repository::Key; -use crate::task_mgr::BACKGROUND_RUNTIME; use crate::walrecord::NeonWalRecord; use pageserver_api::reltag::{RelTag, SlruKind}; use postgres_ffi::pg_constants; @@ -71,32 +70,8 @@ pub(crate) struct BufferTag { pub blknum: u32, } -/// -/// WAL Redo Manager is responsible for replaying WAL records. -/// -/// Callers use the WAL redo manager through this abstract interface, -/// which makes it easy to mock it in tests. -pub trait WalRedoManager: Send + Sync { - /// Apply some WAL records. - /// - /// The caller passes an old page image, and WAL records that should be - /// applied over it. The return value is a new page image, after applying - /// the reords. - fn request_redo( - &self, - key: Key, - lsn: Lsn, - base_img: Option<(Lsn, Bytes)>, - records: Vec<(Lsn, NeonWalRecord)>, - pg_version: u32, - ) -> anyhow::Result; -} - struct ProcessInput { - child: NoLeakChild, stdin: ChildStdin, - stderr_fd: RawFd, - stdout_fd: RawFd, n_requests: usize, } @@ -116,13 +91,7 @@ struct ProcessOutput { pub struct PostgresRedoManager { tenant_id: TenantId, conf: &'static PageServerConf, - /// Counter to separate same sized walredo inputs failing at the same millisecond. - #[cfg(feature = "testing")] - dump_sequence: AtomicUsize, - - stdout: Mutex>, - stdin: Mutex>, - stderr: Mutex>, + redo_process: RwLock>>, } /// Can this request be served by neon redo functions @@ -143,14 +112,15 @@ fn can_apply_in_neon(rec: &NeonWalRecord) -> bool { /// /// Public interface of WAL redo manager /// -impl WalRedoManager for PostgresRedoManager { +impl PostgresRedoManager { /// /// Request the WAL redo manager to apply some WAL records /// /// The WAL redo is handled by a separate thread, so this just sends a request /// to the thread and waits for response. /// - fn request_redo( + /// CANCEL SAFETY: NOT CANCEL SAFE. + pub async fn request_redo( &self, key: Key, lsn: Lsn, @@ -182,6 +152,7 @@ impl WalRedoManager for PostgresRedoManager { self.conf.wal_redo_timeout, pg_version, ) + .await }; img = Some(result?); @@ -202,6 +173,7 @@ impl WalRedoManager for PostgresRedoManager { self.conf.wal_redo_timeout, pg_version, ) + .await } } } @@ -215,11 +187,7 @@ impl PostgresRedoManager { PostgresRedoManager { tenant_id, conf, - #[cfg(feature = "testing")] - dump_sequence: AtomicUsize::default(), - stdin: Mutex::new(None), - stdout: Mutex::new(None), - stderr: Mutex::new(None), + redo_process: RwLock::new(None), } } @@ -227,7 +195,7 @@ impl PostgresRedoManager { /// Process one request for WAL redo using wal-redo postgres /// #[allow(clippy::too_many_arguments)] - fn apply_batch_postgres( + async fn apply_batch_postgres( &self, key: Key, lsn: Lsn, @@ -242,20 +210,38 @@ impl PostgresRedoManager { let start_time = Instant::now(); let mut n_attempts = 0u32; loop { - let mut proc = self.stdin.lock().unwrap(); let lock_time = Instant::now(); // launch the WAL redo process on first use - if proc.is_none() { - self.launch(&mut proc, pg_version) - .context("launch process")?; - } + let proc: Arc = { + let proc_guard = self.redo_process.read().unwrap(); + match &*proc_guard { + None => { + // "upgrade" to write lock to launch the process + drop(proc_guard); + let mut proc_guard = self.redo_process.write().unwrap(); + match &*proc_guard { + None => { + let proc = Arc::new( + WalRedoProcess::launch(self.conf, self.tenant_id, pg_version) + .context("launch walredo process")?, + ); + *proc_guard = Some(Arc::clone(&proc)); + proc + } + Some(proc) => Arc::clone(proc), + } + } + Some(proc) => Arc::clone(proc), + } + }; + WAL_REDO_WAIT_TIME.observe(lock_time.duration_since(start_time).as_secs_f64()); // Relational WAL records are applied using wal-redo-postgres let buf_tag = BufferTag { rel, blknum }; - let result = self - .apply_wal_records(proc, buf_tag, &base_img, records, wal_redo_timeout) + let result = proc + .apply_wal_records(buf_tag, &base_img, records, wal_redo_timeout) .context("apply_wal_records"); let end_time = Instant::now(); @@ -296,22 +282,40 @@ impl PostgresRedoManager { n_attempts, e, ); - // self.stdin only holds stdin & stderr as_raw_fd(). - // Dropping it as part of take() doesn't close them. - // The owning objects (ChildStdout and ChildStderr) are stored in - // self.stdout and self.stderr, respsectively. - // We intentionally keep them open here to avoid a race between - // currently running `apply_wal_records()` and a `launch()` call - // after we return here. - // The currently running `apply_wal_records()` must not read from - // the newly launched process. - // By keeping self.stdout and self.stderr open here, `launch()` will - // get other file descriptors for the new child's stdout and stderr, - // and hence the current `apply_wal_records()` calls will observe - // `output.stdout.as_raw_fd() != stdout_fd` . - if let Some(proc) = self.stdin.lock().unwrap().take() { - proc.child.kill_and_wait(); + // Avoid concurrent callers hitting the same issue. + // We can't prevent it from happening because we want to enable parallelism. + { + let mut guard = self.redo_process.write().unwrap(); + match &*guard { + Some(current_field_value) => { + if Arc::ptr_eq(current_field_value, &proc) { + // We're the first to observe an error from `proc`, it's our job to take it out of rotation. + *guard = None; + } + } + None => { + // Another thread was faster to observe the error, and already took the process out of rotation. + } + } } + // NB: there may still be other concurrent threads using `proc`. + // The last one will send SIGKILL when the underlying Arc reaches refcount 0. + // NB: it's important to drop(proc) after drop(guard). Otherwise we'd keep + // holding the lock while waiting for the process to exit. + // NB: the drop impl blocks the current threads with a wait() system call for + // the child process. We dropped the `guard` above so that other threads aren't + // affected. But, it's good that the current thread _does_ block to wait. + // If we instead deferred the waiting into the background / to tokio, it could + // happen that if walredo always fails immediately, we spawn processes faster + // than we can SIGKILL & `wait` for them to exit. By doing it the way we do here, + // we limit this risk of run-away to at most $num_runtimes * $num_executor_threads. + // This probably needs revisiting at some later point. + let mut wait_done = proc.stderr_logger_task_done.clone(); + drop(proc); + wait_done + .wait_for(|v| *v) + .await + .expect("we use scopeguard to ensure we always send `true` to the channel before dropping the sender"); } else if n_attempts != 0 { info!(n_attempts, "retried walredo succeeded"); } @@ -614,24 +618,33 @@ impl CloseFileDescriptors for C { } } -impl PostgresRedoManager { +struct WalRedoProcess { + #[allow(dead_code)] + conf: &'static PageServerConf, + tenant_id: TenantId, + // Some() on construction, only becomes None on Drop. + child: Option, + stdout: Mutex, + stdin: Mutex, + stderr_logger_cancel: CancellationToken, + stderr_logger_task_done: tokio::sync::watch::Receiver, + /// Counter to separate same sized walredo inputs failing at the same millisecond. + #[cfg(feature = "testing")] + dump_sequence: AtomicUsize, +} + +impl WalRedoProcess { // // Start postgres binary in special WAL redo mode. // - #[instrument(skip_all,fields(tenant_id=%self.tenant_id, pg_version=pg_version))] + #[instrument(skip_all,fields(tenant_id=%tenant_id, pg_version=pg_version))] fn launch( - &self, - input: &mut MutexGuard>, + conf: &'static PageServerConf, + tenant_id: TenantId, pg_version: u32, - ) -> Result<(), Error> { - let pg_bin_dir_path = self - .conf - .pg_bin_dir(pg_version) - .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_bin_dir path: {e}")))?; - let pg_lib_dir_path = self - .conf - .pg_lib_dir(pg_version) - .map_err(|e| Error::new(ErrorKind::Other, format!("incorrect pg_lib_dir path: {e}")))?; + ) -> anyhow::Result { + let pg_bin_dir_path = conf.pg_bin_dir(pg_version).context("pg_bin_dir")?; // TODO these should be infallible. + let pg_lib_dir_path = conf.pg_lib_dir(pg_version).context("pg_lib_dir")?; // Start postgres itself let child = Command::new(pg_bin_dir_path.join("postgres")) @@ -652,13 +665,8 @@ impl PostgresRedoManager { // as close-on-exec by default, but that's not enough, since we use // libraries that directly call libc open without setting that flag. .close_fds() - .spawn_no_leak_child(self.tenant_id) - .map_err(|e| { - Error::new( - e.kind(), - format!("postgres --wal-redo command failed to start: {}", e), - ) - })?; + .spawn_no_leak_child(tenant_id) + .context("spawn process")?; let mut child = scopeguard::guard(child, |child| { error!("killing wal-redo-postgres process due to a problem during launch"); @@ -668,7 +676,6 @@ impl PostgresRedoManager { let stdin = child.stdin.take().unwrap(); let stdout = child.stdout.take().unwrap(); let stderr = child.stderr.take().unwrap(); - macro_rules! set_nonblock_or_log_err { ($file:ident) => {{ let res = set_nonblock($file.as_raw_fd()); @@ -682,39 +689,108 @@ impl PostgresRedoManager { set_nonblock_or_log_err!(stdout)?; set_nonblock_or_log_err!(stderr)?; + let mut stderr = tokio::io::unix::AsyncFd::new(stderr).context("AsyncFd::with_interest")?; + // all fallible operations post-spawn are complete, so get rid of the guard let child = scopeguard::ScopeGuard::into_inner(child); - **input = Some(ProcessInput { - child, - stdout_fd: stdout.as_raw_fd(), - stderr_fd: stderr.as_raw_fd(), - stdin, - n_requests: 0, + let stderr_logger_cancel = CancellationToken::new(); + let (stderr_logger_task_done_tx, stderr_logger_task_done_rx) = + tokio::sync::watch::channel(false); + tokio::spawn({ + let stderr_logger_cancel = stderr_logger_cancel.clone(); + async move { + scopeguard::defer! { + debug!("wal-redo-postgres stderr_logger_task finished"); + let _ = stderr_logger_task_done_tx.send(true); + } + debug!("wal-redo-postgres stderr_logger_task started"); + loop { + // NB: we purposefully don't do a select! for the cancellation here. + // The cancellation would likely cause us to miss stderr messages. + // We can rely on this to return from .await because when we SIGKILL + // the child, the writing end of the stderr pipe gets closed. + match stderr.readable_mut().await { + Ok(mut guard) => { + let mut errbuf = [0; 16384]; + let res = guard.try_io(|fd| { + use std::io::Read; + fd.get_mut().read(&mut errbuf) + }); + match res { + Ok(Ok(0)) => { + // it closed the stderr pipe + break; + } + Ok(Ok(n)) => { + // The message might not be split correctly into lines here. But this is + // good enough, the important thing is to get the message to the log. + let output = String::from_utf8_lossy(&errbuf[0..n]).to_string(); + error!(output, "received output"); + }, + Ok(Err(e)) => { + error!(error = ?e, "read() error, waiting for cancellation"); + stderr_logger_cancel.cancelled().await; + error!(error = ?e, "read() error, cancellation complete"); + break; + } + Err(e) => { + let _e: tokio::io::unix::TryIoError = e; + // the read() returned WouldBlock, that's expected + } + } + } + Err(e) => { + error!(error = ?e, "read() error, waiting for cancellation"); + stderr_logger_cancel.cancelled().await; + error!(error = ?e, "read() error, cancellation complete"); + break; + } + } + } + }.instrument(tracing::info_span!(parent: None, "wal-redo-postgres-stderr", pid = child.id(), tenant_id = %tenant_id, %pg_version)) }); - *self.stdout.lock().unwrap() = Some(ProcessOutput { - stdout, - pending_responses: VecDeque::new(), - n_processed_responses: 0, - }); - *self.stderr.lock().unwrap() = Some(stderr); + Ok(Self { + conf, + tenant_id, + child: Some(child), + stdin: Mutex::new(ProcessInput { + stdin, + n_requests: 0, + }), + stdout: Mutex::new(ProcessOutput { + stdout, + pending_responses: VecDeque::new(), + n_processed_responses: 0, + }), + stderr_logger_cancel, + stderr_logger_task_done: stderr_logger_task_done_rx, + #[cfg(feature = "testing")] + dump_sequence: AtomicUsize::default(), + }) + } - Ok(()) + fn id(&self) -> u32 { + self.child + .as_ref() + .expect("must not call this during Drop") + .id() } // Apply given WAL records ('records') over an old page image. Returns // new page image. // - #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%input.as_ref().unwrap().child.id()))] + #[instrument(skip_all, fields(tenant_id=%self.tenant_id, pid=%self.id()))] fn apply_wal_records( &self, - input: MutexGuard>, tag: BufferTag, base_img: &Option, records: &[(Lsn, NeonWalRecord)], wal_redo_timeout: Duration, ) -> anyhow::Result { + let input = self.stdin.lock().unwrap(); + // Serialize all the messages to send the WAL redo process first. // // This could be problematic if there are millions of records to replay, @@ -757,26 +833,17 @@ impl PostgresRedoManager { fn apply_wal_records0( &self, writebuf: &[u8], - mut input: MutexGuard>, + input: MutexGuard, wal_redo_timeout: Duration, ) -> anyhow::Result { - let proc = input.as_mut().unwrap(); + let mut proc = { input }; // TODO: remove this legacy rename, but this keep the patch small. let mut nwrite = 0usize; - let stdout_fd = proc.stdout_fd; - // Prepare for calling poll() - let mut pollfds = [ - PollFd::new(proc.stdin.as_raw_fd(), PollFlags::POLLOUT), - PollFd::new(proc.stderr_fd, PollFlags::POLLIN), - PollFd::new(stdout_fd, PollFlags::POLLIN), - ]; + let mut stdin_pollfds = [PollFd::new(proc.stdin.as_raw_fd(), PollFlags::POLLOUT)]; - // We do two things simultaneously: send the old base image and WAL records to - // the child process's stdin and forward any logging - // information that the child writes to its stderr to the page server's log. while nwrite < writebuf.len() { let n = loop { - match nix::poll::poll(&mut pollfds[0..2], wal_redo_timeout.as_millis() as i32) { + match nix::poll::poll(&mut stdin_pollfds[..], wal_redo_timeout.as_millis() as i32) { Err(nix::errno::Errno::EINTR) => continue, res => break res, } @@ -786,32 +853,8 @@ impl PostgresRedoManager { anyhow::bail!("WAL redo timed out"); } - // If we have some messages in stderr, forward them to the log. - let err_revents = pollfds[1].revents().unwrap(); - if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { - let mut errbuf: [u8; 16384] = [0; 16384]; - let mut stderr_guard = self.stderr.lock().unwrap(); - let stderr = stderr_guard.as_mut().unwrap(); - let len = stderr.read(&mut errbuf)?; - - // The message might not be split correctly into lines here. But this is - // good enough, the important thing is to get the message to the log. - if len > 0 { - error!( - "wal-redo-postgres: {}", - String::from_utf8_lossy(&errbuf[0..len]) - ); - - // To make sure we capture all log from the process if it fails, keep - // reading from the stderr, before checking the stdout. - continue; - } - } else if err_revents.contains(PollFlags::POLLHUP) { - anyhow::bail!("WAL redo process closed its stderr unexpectedly"); - } - // If 'stdin' is writeable, do write. - let in_revents = pollfds[0].revents().unwrap(); + let in_revents = stdin_pollfds[0].revents().unwrap(); if in_revents & (PollFlags::POLLERR | PollFlags::POLLOUT) != PollFlags::empty() { nwrite += proc.stdin.write(&writebuf[nwrite..])?; } else if in_revents.contains(PollFlags::POLLHUP) { @@ -821,7 +864,7 @@ impl PostgresRedoManager { } let request_no = proc.n_requests; proc.n_requests += 1; - drop(input); + drop(proc); // To improve walredo performance we separate sending requests and receiving // responses. Them are protected by different mutexes (output and input). @@ -835,20 +878,8 @@ impl PostgresRedoManager { // pending responses ring buffer and truncate all empty elements from the front, // advancing processed responses number. - let mut output_guard = self.stdout.lock().unwrap(); - let output = output_guard.as_mut().unwrap(); - if output.stdout.as_raw_fd() != stdout_fd { - // If stdout file descriptor is changed then it means that walredo process is crashed and restarted. - // As far as ProcessInput and ProcessOutout are protected by different mutexes, - // it can happen that we send request to one process and waiting response from another. - // To prevent such situation we compare stdout file descriptors. - // As far as old stdout pipe is destroyed only after new one is created, - // it can not reuse the same file descriptor, so this check is safe. - // - // Cross-read this with the comment in apply_batch_postgres if result.is_err(). - // That's where we kill the child process. - anyhow::bail!("WAL redo process closed its stdout unexpectedly"); - } + let mut output = self.stdout.lock().unwrap(); + let mut stdout_pollfds = [PollFd::new(output.stdout.as_raw_fd(), PollFlags::POLLIN)]; let n_processed_responses = output.n_processed_responses; while n_processed_responses + output.pending_responses.len() <= request_no { // We expect the WAL redo process to respond with an 8k page image. We read it @@ -859,7 +890,10 @@ impl PostgresRedoManager { // We do two things simultaneously: reading response from stdout // and forward any logging information that the child writes to its stderr to the page server's log. let n = loop { - match nix::poll::poll(&mut pollfds[1..3], wal_redo_timeout.as_millis() as i32) { + match nix::poll::poll( + &mut stdout_pollfds[..], + wal_redo_timeout.as_millis() as i32, + ) { Err(nix::errno::Errno::EINTR) => continue, res => break res, } @@ -869,32 +903,8 @@ impl PostgresRedoManager { anyhow::bail!("WAL redo timed out"); } - // If we have some messages in stderr, forward them to the log. - let err_revents = pollfds[1].revents().unwrap(); - if err_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { - let mut errbuf: [u8; 16384] = [0; 16384]; - let mut stderr_guard = self.stderr.lock().unwrap(); - let stderr = stderr_guard.as_mut().unwrap(); - let len = stderr.read(&mut errbuf)?; - - // The message might not be split correctly into lines here. But this is - // good enough, the important thing is to get the message to the log. - if len > 0 { - error!( - "wal-redo-postgres: {}", - String::from_utf8_lossy(&errbuf[0..len]) - ); - - // To make sure we capture all log from the process if it fails, keep - // reading from the stderr, before checking the stdout. - continue; - } - } else if err_revents.contains(PollFlags::POLLHUP) { - anyhow::bail!("WAL redo process closed its stderr unexpectedly"); - } - // If we have some data in stdout, read it to the result buffer. - let out_revents = pollfds[2].revents().unwrap(); + let out_revents = stdout_pollfds[0].revents().unwrap(); if out_revents & (PollFlags::POLLERR | PollFlags::POLLIN) != PollFlags::empty() { nresult += output.stdout.read(&mut resultbuf[nresult..])?; } else if out_revents.contains(PollFlags::POLLHUP) { @@ -984,6 +994,17 @@ impl PostgresRedoManager { fn record_and_log(&self, _: &[u8]) {} } +impl Drop for WalRedoProcess { + fn drop(&mut self) { + self.child + .take() + .expect("we only do this once") + .kill_and_wait(); + self.stderr_logger_cancel.cancel(); + // no way to wait for stderr_logger_task from Drop because that is async only + } +} + /// Wrapper type around `std::process::Child` which guarantees that the child /// will be killed and waited-for by this process before being dropped. struct NoLeakChild { @@ -1062,7 +1083,7 @@ impl Drop for NoLeakChild { // Offload the kill+wait of the child process into the background. // If someone stops the runtime, we'll leak the child process. // We can ignore that case because we only stop the runtime on pageserver exit. - BACKGROUND_RUNTIME.spawn(async move { + tokio::runtime::Handle::current().spawn(async move { tokio::task::spawn_blocking(move || { // Intentionally don't inherit the tracing context from whoever is dropping us. // This thread here is going to outlive of our dropper. @@ -1131,15 +1152,15 @@ fn build_get_page_msg(tag: BufferTag, buf: &mut Vec) { #[cfg(test)] mod tests { - use super::{PostgresRedoManager, WalRedoManager}; + use super::PostgresRedoManager; use crate::repository::Key; use crate::{config::PageServerConf, walrecord::NeonWalRecord}; use bytes::Bytes; use std::str::FromStr; use utils::{id::TenantId, lsn::Lsn}; - #[test] - fn short_v14_redo() { + #[tokio::test] + async fn short_v14_redo() { let expected = std::fs::read("fixtures/short_v14_redo.page").unwrap(); let h = RedoHarness::new().unwrap(); @@ -1160,13 +1181,14 @@ mod tests { short_records(), 14, ) + .await .unwrap(); assert_eq!(&expected, &*page); } - #[test] - fn short_v14_fails_for_wrong_key_but_returns_zero_page() { + #[tokio::test] + async fn short_v14_fails_for_wrong_key_but_returns_zero_page() { let h = RedoHarness::new().unwrap(); let page = h @@ -1186,6 +1208,7 @@ mod tests { short_records(), 14, ) + .await .unwrap(); // TODO: there will be some stderr printout, which is forwarded to tracing that could @@ -1193,6 +1216,22 @@ mod tests { assert_eq!(page, crate::ZERO_PAGE); } + #[tokio::test] + async fn test_stderr() { + let h = RedoHarness::new().unwrap(); + h + .manager + .request_redo( + Key::from_i128(0), + Lsn::INVALID, + None, + short_records(), + 16, /* 16 currently produces stderr output on startup, which adds a nice extra edge */ + ) + .await + .unwrap_err(); + } + #[allow(clippy::octal_escapes)] fn short_records() -> Vec<(Lsn, NeonWalRecord)> { vec![ @@ -1221,6 +1260,8 @@ mod tests { impl RedoHarness { fn new() -> anyhow::Result { + crate::tenant::harness::setup_logging(); + let repo_dir = camino_tempfile::tempdir()?; let conf = PageServerConf::dummy_conf(repo_dir.path().to_path_buf()); let conf = Box::leak(Box::new(conf)); diff --git a/pgxn/neon/Makefile b/pgxn/neon/Makefile index e88901ed78..84835843bc 100644 --- a/pgxn/neon/Makefile +++ b/pgxn/neon/Makefile @@ -23,6 +23,23 @@ EXTENSION = neon DATA = neon--1.0.sql PGFILEDESC = "neon - cloud storage for PostgreSQL" +EXTRA_CLEAN = \ + libwalproposer.a + +WALPROP_OBJS = \ + $(WIN32RES) \ + walproposer.o \ + neon_utils.o \ + walproposer_compat.o + +.PHONY: walproposer-lib +walproposer-lib: CPPFLAGS += -DWALPROPOSER_LIB +walproposer-lib: libwalproposer.a; + +.PHONY: libwalproposer.a +libwalproposer.a: $(WALPROP_OBJS) + rm -f $@ + $(AR) $(AROPT) $@ $^ PG_CONFIG = pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index 5e172a0be4..3a841b04ec 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -63,7 +63,6 @@ #include "storage/md.h" #include "pgstat.h" - #if PG_VERSION_NUM >= 150000 #include "access/xlogutils.h" #include "access/xlogrecovery.h" @@ -1395,12 +1394,6 @@ neon_get_request_lsn(bool *latest, NRelFileInfo rinfo, ForkNumber forknum, Block elog(DEBUG1, "neon_get_request_lsn GetXLogReplayRecPtr %X/%X request lsn 0 ", (uint32) ((lsn) >> 32), (uint32) (lsn)); } - else if (am_walsender) - { - *latest = true; - lsn = InvalidXLogRecPtr; - elog(DEBUG1, "am walsender neon_get_request_lsn lsn 0 "); - } else { XLogRecPtr flushlsn; diff --git a/pgxn/neon/walproposer.c b/pgxn/neon/walproposer.c index c1fd5e3ef3..10544ba7a8 100644 --- a/pgxn/neon/walproposer.c +++ b/pgxn/neon/walproposer.c @@ -79,7 +79,7 @@ static int CompareLsn(const void *a, const void *b); static char *FormatSafekeeperState(SafekeeperState state); static void AssertEventsOkForState(uint32 events, Safekeeper *sk); static uint32 SafekeeperStateDesiredEvents(SafekeeperState state); -static char *FormatEvents(uint32 events); +static char *FormatEvents(WalProposer *wp, uint32 events); WalProposer * WalProposerCreate(WalProposerConfig *config, walproposer_api api) @@ -98,7 +98,7 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) port = strchr(host, ':'); if (port == NULL) { - elog(FATAL, "port is not specified"); + walprop_log(FATAL, "port is not specified"); } *port++ = '\0'; sep = strchr(port, ','); @@ -106,12 +106,11 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) *sep++ = '\0'; if (wp->n_safekeepers + 1 >= MAX_SAFEKEEPERS) { - elog(FATAL, "Too many safekeepers"); + walprop_log(FATAL, "Too many safekeepers"); } wp->safekeeper[wp->n_safekeepers].host = host; wp->safekeeper[wp->n_safekeepers].port = port; wp->safekeeper[wp->n_safekeepers].state = SS_OFFLINE; - wp->safekeeper[wp->n_safekeepers].conn = NULL; wp->safekeeper[wp->n_safekeepers].wp = wp; { @@ -122,13 +121,11 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) "host=%s port=%s dbname=replication options='-c timeline_id=%s tenant_id=%s'", sk->host, sk->port, wp->config->neon_timeline, wp->config->neon_tenant); if (written > MAXCONNINFO || written < 0) - elog(FATAL, "could not create connection string for safekeeper %s:%s", sk->host, sk->port); + walprop_log(FATAL, "could not create connection string for safekeeper %s:%s", sk->host, sk->port); } initStringInfo(&wp->safekeeper[wp->n_safekeepers].outbuf); - wp->safekeeper[wp->n_safekeepers].xlogreader = wp->api.wal_reader_allocate(); - if (wp->safekeeper[wp->n_safekeepers].xlogreader == NULL) - elog(FATAL, "Failed to allocate xlog reader"); + wp->api.wal_reader_allocate(&wp->safekeeper[wp->n_safekeepers]); wp->safekeeper[wp->n_safekeepers].flushWrite = false; wp->safekeeper[wp->n_safekeepers].startStreamingAt = InvalidXLogRecPtr; wp->safekeeper[wp->n_safekeepers].streamingAt = InvalidXLogRecPtr; @@ -136,7 +133,7 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) } if (wp->n_safekeepers < 1) { - elog(FATAL, "Safekeepers addresses are not specified"); + walprop_log(FATAL, "Safekeepers addresses are not specified"); } wp->quorum = wp->n_safekeepers / 2 + 1; @@ -144,27 +141,47 @@ WalProposerCreate(WalProposerConfig *config, walproposer_api api) wp->greetRequest.tag = 'g'; wp->greetRequest.protocolVersion = SK_PROTOCOL_VERSION; wp->greetRequest.pgVersion = PG_VERSION_NUM; - wp->api.strong_random(&wp->greetRequest.proposerId, sizeof(wp->greetRequest.proposerId)); + wp->api.strong_random(wp, &wp->greetRequest.proposerId, sizeof(wp->greetRequest.proposerId)); wp->greetRequest.systemId = wp->config->systemId; if (!wp->config->neon_timeline) - elog(FATAL, "neon.timeline_id is not provided"); + walprop_log(FATAL, "neon.timeline_id is not provided"); if (*wp->config->neon_timeline != '\0' && !HexDecodeString(wp->greetRequest.timeline_id, wp->config->neon_timeline, 16)) - elog(FATAL, "Could not parse neon.timeline_id, %s", wp->config->neon_timeline); + walprop_log(FATAL, "Could not parse neon.timeline_id, %s", wp->config->neon_timeline); if (!wp->config->neon_tenant) - elog(FATAL, "neon.tenant_id is not provided"); + walprop_log(FATAL, "neon.tenant_id is not provided"); if (*wp->config->neon_tenant != '\0' && !HexDecodeString(wp->greetRequest.tenant_id, wp->config->neon_tenant, 16)) - elog(FATAL, "Could not parse neon.tenant_id, %s", wp->config->neon_tenant); + walprop_log(FATAL, "Could not parse neon.tenant_id, %s", wp->config->neon_tenant); - wp->greetRequest.timeline = wp->api.get_timeline_id(); + wp->greetRequest.timeline = wp->config->pgTimeline; wp->greetRequest.walSegSize = wp->config->wal_segment_size; - wp->api.init_event_set(wp->n_safekeepers); + wp->api.init_event_set(wp); return wp; } +void +WalProposerFree(WalProposer *wp) +{ + for (int i = 0; i < wp->n_safekeepers; i++) + { + Safekeeper *sk = &wp->safekeeper[i]; + + Assert(sk->outbuf.data != NULL); + pfree(sk->outbuf.data); + if (sk->voteResponse.termHistory.entries) + pfree(sk->voteResponse.termHistory.entries); + sk->voteResponse.termHistory.entries = NULL; + } + if (wp->propTermHistory.entries != NULL) + pfree(wp->propTermHistory.entries); + wp->propTermHistory.entries = NULL; + + pfree(wp); +} + /* * Create new AppendRequest message and start sending it. This function is * called from walsender every time the new WAL is available. @@ -190,10 +207,10 @@ WalProposerPoll(WalProposer *wp) Safekeeper *sk = NULL; int rc = 0; uint32 events = 0; - TimestampTz now = wp->api.get_current_timestamp(); + TimestampTz now = wp->api.get_current_timestamp(wp); long timeout = TimeToReconnect(wp, now); - rc = wp->api.wait_event_set(timeout, &sk, &events); + rc = wp->api.wait_event_set(wp, timeout, &sk, &events); /* Exit loop if latch is set (we got new WAL) */ if ((rc == 1 && events & WL_LATCH_SET)) @@ -224,14 +241,14 @@ WalProposerPoll(WalProposer *wp) */ if (!wp->config->syncSafekeepers) { - XLogRecPtr flushed = wp->api.get_flush_rec_ptr(); + XLogRecPtr flushed = wp->api.get_flush_rec_ptr(wp); if (flushed > wp->availableLsn) break; } } - now = wp->api.get_current_timestamp(); + now = wp->api.get_current_timestamp(wp); /* timeout expired: poll state */ if (rc == 0 || TimeToReconnect(wp, now) <= 0) { @@ -249,7 +266,7 @@ WalProposerPoll(WalProposer *wp) /* * Abandon connection attempts which take too long. */ - now = wp->api.get_current_timestamp(); + now = wp->api.get_current_timestamp(wp); for (int i = 0; i < wp->n_safekeepers; i++) { Safekeeper *sk = &wp->safekeeper[i]; @@ -257,7 +274,7 @@ WalProposerPoll(WalProposer *wp) if (TimestampDifferenceExceeds(sk->latestMsgReceivedAt, now, wp->config->safekeeper_connection_timeout)) { - elog(WARNING, "terminating connection to safekeeper '%s:%s' in '%s' state: no messages received during the last %dms or connection attempt took longer than that", + walprop_log(WARNING, "terminating connection to safekeeper '%s:%s' in '%s' state: no messages received during the last %dms or connection attempt took longer than that", sk->host, sk->port, FormatSafekeeperState(sk->state), wp->config->safekeeper_connection_timeout); ShutdownConnection(sk); } @@ -296,10 +313,10 @@ HackyRemoveWalProposerEvent(Safekeeper *to_remove) { WalProposer *wp = to_remove->wp; - /* Remove the existing event set */ - wp->api.free_event_set(); + /* Remove the existing event set, assign sk->eventPos = -1 */ + wp->api.free_event_set(wp); /* Re-initialize it without adding any safekeeper events */ - wp->api.init_event_set(wp->n_safekeepers); + wp->api.init_event_set(wp); /* * loop through the existing safekeepers. If they aren't the one we're @@ -311,13 +328,11 @@ HackyRemoveWalProposerEvent(Safekeeper *to_remove) uint32 desired_events = WL_NO_EVENTS; Safekeeper *sk = &wp->safekeeper[i]; - sk->eventPos = -1; - if (sk == to_remove) continue; /* If this safekeeper isn't offline, add an event for it! */ - if (sk->conn != NULL) + if (sk->state != SS_OFFLINE) { desired_events = SafekeeperStateDesiredEvents(sk->state); /* will set sk->eventPos */ @@ -330,9 +345,7 @@ HackyRemoveWalProposerEvent(Safekeeper *to_remove) static void ShutdownConnection(Safekeeper *sk) { - if (sk->conn) - sk->wp->api.conn_finish(sk->conn); - sk->conn = NULL; + sk->wp->api.conn_finish(sk); sk->state = SS_OFFLINE; sk->flushWrite = false; sk->streamingAt = InvalidXLogRecPtr; @@ -361,23 +374,16 @@ ResetConnection(Safekeeper *sk) } /* - * Try to establish new connection + * Try to establish new connection, it will update sk->conn. */ - sk->conn = wp->api.conn_connect_start((char *) &sk->conninfo); - - /* - * "If the result is null, then libpq has been unable to allocate a new - * PGconn structure" - */ - if (!sk->conn) - elog(FATAL, "failed to allocate new PGconn object"); + wp->api.conn_connect_start(sk); /* * PQconnectStart won't actually start connecting until we run * PQconnectPoll. Before we do that though, we need to check that it * didn't immediately fail. */ - if (wp->api.conn_status(sk->conn) == WP_CONNECTION_BAD) + if (wp->api.conn_status(sk) == WP_CONNECTION_BAD) { /*--- * According to libpq docs: @@ -388,15 +394,14 @@ ResetConnection(Safekeeper *sk) * * https://www.postgresql.org/docs/devel/libpq-connect.html#LIBPQ-PQCONNECTSTARTPARAMS */ - elog(WARNING, "Immediate failure to connect with node '%s:%s':\n\terror: %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "Immediate failure to connect with node '%s:%s':\n\terror: %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); /* * Even though the connection failed, we still need to clean up the * object */ - wp->api.conn_finish(sk->conn); - sk->conn = NULL; + wp->api.conn_finish(sk); return; } @@ -413,10 +418,10 @@ ResetConnection(Safekeeper *sk) * (see libpqrcv_connect, defined in * src/backend/replication/libpqwalreceiver/libpqwalreceiver.c) */ - elog(LOG, "connecting with node %s:%s", sk->host, sk->port); + walprop_log(LOG, "connecting with node %s:%s", sk->host, sk->port); sk->state = SS_CONNECTING_WRITE; - sk->latestMsgReceivedAt = wp->api.get_current_timestamp(); + sk->latestMsgReceivedAt = wp->api.get_current_timestamp(wp); wp->api.add_safekeeper_event_set(sk, WL_SOCKET_WRITEABLE); return; @@ -447,7 +452,7 @@ TimeToReconnect(WalProposer *wp, TimestampTz now) static void ReconnectSafekeepers(WalProposer *wp) { - TimestampTz now = wp->api.get_current_timestamp(); + TimestampTz now = wp->api.get_current_timestamp(wp); if (TimeToReconnect(wp, now) == 0) { @@ -467,6 +472,8 @@ ReconnectSafekeepers(WalProposer *wp) static void AdvancePollState(Safekeeper *sk, uint32 events) { + WalProposer *wp = sk->wp; + /* * Sanity check. We assume further down that the operations don't block * because the socket is ready. @@ -481,7 +488,7 @@ AdvancePollState(Safekeeper *sk, uint32 events) * ResetConnection */ case SS_OFFLINE: - elog(FATAL, "Unexpected safekeeper %s:%s state advancement: is offline", + walprop_log(FATAL, "Unexpected safekeeper %s:%s state advancement: is offline", sk->host, sk->port); break; /* actually unreachable, but prevents * -Wimplicit-fallthrough */ @@ -517,7 +524,7 @@ AdvancePollState(Safekeeper *sk, uint32 events) * requests. */ case SS_VOTING: - elog(WARNING, "EOF from node %s:%s in %s state", sk->host, + walprop_log(WARNING, "EOF from node %s:%s in %s state", sk->host, sk->port, FormatSafekeeperState(sk->state)); ResetConnection(sk); return; @@ -546,7 +553,7 @@ AdvancePollState(Safekeeper *sk, uint32 events) * Idle state for waiting votes from quorum. */ case SS_IDLE: - elog(WARNING, "EOF from node %s:%s in %s state", sk->host, + walprop_log(WARNING, "EOF from node %s:%s in %s state", sk->host, sk->port, FormatSafekeeperState(sk->state)); ResetConnection(sk); return; @@ -564,7 +571,7 @@ static void HandleConnectionEvent(Safekeeper *sk) { WalProposer *wp = sk->wp; - WalProposerConnectPollStatusType result = wp->api.conn_connect_poll(sk->conn); + WalProposerConnectPollStatusType result = wp->api.conn_connect_poll(sk); /* The new set of events we'll wait on, after updating */ uint32 new_events = WL_NO_EVENTS; @@ -572,9 +579,9 @@ HandleConnectionEvent(Safekeeper *sk) switch (result) { case WP_CONN_POLLING_OK: - elog(LOG, "connected with node %s:%s", sk->host, + walprop_log(LOG, "connected with node %s:%s", sk->host, sk->port); - sk->latestMsgReceivedAt = wp->api.get_current_timestamp(); + sk->latestMsgReceivedAt = wp->api.get_current_timestamp(wp); /* * We have to pick some event to update event set. We'll @@ -596,8 +603,8 @@ HandleConnectionEvent(Safekeeper *sk) break; case WP_CONN_POLLING_FAILED: - elog(WARNING, "failed to connect to node '%s:%s': %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "failed to connect to node '%s:%s': %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); /* * If connecting failed, we don't want to restart the connection @@ -631,10 +638,10 @@ SendStartWALPush(Safekeeper *sk) { WalProposer *wp = sk->wp; - if (!wp->api.conn_send_query(sk->conn, "START_WAL_PUSH")) + if (!wp->api.conn_send_query(sk, "START_WAL_PUSH")) { - elog(WARNING, "Failed to send 'START_WAL_PUSH' query to safekeeper %s:%s: %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "Failed to send 'START_WAL_PUSH' query to safekeeper %s:%s: %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); ShutdownConnection(sk); return; } @@ -647,7 +654,7 @@ RecvStartWALPushResult(Safekeeper *sk) { WalProposer *wp = sk->wp; - switch (wp->api.conn_get_query_result(sk->conn)) + switch (wp->api.conn_get_query_result(sk)) { /* * Successful result, move on to starting the handshake @@ -670,8 +677,8 @@ RecvStartWALPushResult(Safekeeper *sk) break; case WP_EXEC_FAILED: - elog(WARNING, "Failed to send query to safekeeper %s:%s: %s", - sk->host, sk->port, wp->api.conn_error_message(sk->conn)); + walprop_log(WARNING, "Failed to send query to safekeeper %s:%s: %s", + sk->host, sk->port, wp->api.conn_error_message(sk)); ShutdownConnection(sk); return; @@ -681,7 +688,7 @@ RecvStartWALPushResult(Safekeeper *sk) * wrong" */ case WP_EXEC_UNEXPECTED_SUCCESS: - elog(WARNING, "Received bad response from safekeeper %s:%s query execution", + walprop_log(WARNING, "Received bad response from safekeeper %s:%s query execution", sk->host, sk->port); ShutdownConnection(sk); return; @@ -717,7 +724,7 @@ RecvAcceptorGreeting(Safekeeper *sk) if (!AsyncReadMessage(sk, (AcceptorProposerMessage *) &sk->greetResponse)) return; - elog(LOG, "received AcceptorGreeting from safekeeper %s:%s", sk->host, sk->port); + walprop_log(LOG, "received AcceptorGreeting from safekeeper %s:%s", sk->host, sk->port); /* Protocol is all good, move to voting. */ sk->state = SS_VOTING; @@ -737,7 +744,7 @@ RecvAcceptorGreeting(Safekeeper *sk) if (wp->n_connected == wp->quorum) { wp->propTerm++; - elog(LOG, "proposer connected to quorum (%d) safekeepers, propTerm=" INT64_FORMAT, wp->quorum, wp->propTerm); + walprop_log(LOG, "proposer connected to quorum (%d) safekeepers, propTerm=" INT64_FORMAT, wp->quorum, wp->propTerm); wp->voteRequest = (VoteRequest) { @@ -750,7 +757,7 @@ RecvAcceptorGreeting(Safekeeper *sk) else if (sk->greetResponse.term > wp->propTerm) { /* Another compute with higher term is running. */ - elog(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", + walprop_log(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", sk->host, sk->port, sk->greetResponse.term, wp->propTerm); } @@ -792,7 +799,7 @@ SendVoteRequest(Safekeeper *sk) WalProposer *wp = sk->wp; /* We have quorum for voting, send our vote request */ - elog(LOG, "requesting vote from %s:%s for term " UINT64_FORMAT, sk->host, sk->port, wp->voteRequest.term); + walprop_log(LOG, "requesting vote from %s:%s for term " UINT64_FORMAT, sk->host, sk->port, wp->voteRequest.term); /* On failure, logging & resetting is handled */ if (!BlockingWrite(sk, &wp->voteRequest, sizeof(wp->voteRequest), SS_WAIT_VERDICT)) return; @@ -809,7 +816,7 @@ RecvVoteResponse(Safekeeper *sk) if (!AsyncReadMessage(sk, (AcceptorProposerMessage *) &sk->voteResponse)) return; - elog(LOG, + walprop_log(LOG, "got VoteResponse from acceptor %s:%s, voteGiven=" UINT64_FORMAT ", epoch=" UINT64_FORMAT ", flushLsn=%X/%X, truncateLsn=%X/%X, timelineStartLsn=%X/%X", sk->host, sk->port, sk->voteResponse.voteGiven, GetHighestTerm(&sk->voteResponse.termHistory), LSN_FORMAT_ARGS(sk->voteResponse.flushLsn), @@ -824,7 +831,7 @@ RecvVoteResponse(Safekeeper *sk) if ((!sk->voteResponse.voteGiven) && (sk->voteResponse.term > wp->propTerm || wp->n_votes < wp->quorum)) { - elog(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", + walprop_log(FATAL, "WAL acceptor %s:%s with term " INT64_FORMAT " rejects our connection request with term " INT64_FORMAT "", sk->host, sk->port, sk->voteResponse.term, wp->propTerm); } @@ -869,19 +876,19 @@ HandleElectedProposer(WalProposer *wp) */ if (wp->truncateLsn < wp->propEpochStartLsn) { - elog(LOG, + walprop_log(LOG, "start recovery because truncateLsn=%X/%X is not " "equal to epochStartLsn=%X/%X", LSN_FORMAT_ARGS(wp->truncateLsn), LSN_FORMAT_ARGS(wp->propEpochStartLsn)); /* Perform recovery */ if (!wp->api.recovery_download(&wp->safekeeper[wp->donor], wp->greetRequest.timeline, wp->truncateLsn, wp->propEpochStartLsn)) - elog(FATAL, "Failed to recover state"); + walprop_log(FATAL, "Failed to recover state"); } else if (wp->config->syncSafekeepers) { /* Sync is not needed: just exit */ - wp->api.finish_sync_safekeepers(wp->propEpochStartLsn); + wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); /* unreachable */ } @@ -982,7 +989,7 @@ DetermineEpochStartLsn(WalProposer *wp) if (wp->timelineStartLsn != InvalidXLogRecPtr && wp->timelineStartLsn != wp->safekeeper[i].voteResponse.timelineStartLsn) { - elog(WARNING, + walprop_log(WARNING, "inconsistent timelineStartLsn: current %X/%X, received %X/%X", LSN_FORMAT_ARGS(wp->timelineStartLsn), LSN_FORMAT_ARGS(wp->safekeeper[i].voteResponse.timelineStartLsn)); @@ -998,12 +1005,12 @@ DetermineEpochStartLsn(WalProposer *wp) */ if (wp->propEpochStartLsn == InvalidXLogRecPtr && !wp->config->syncSafekeepers) { - wp->propEpochStartLsn = wp->truncateLsn = wp->api.get_redo_start_lsn(); + wp->propEpochStartLsn = wp->truncateLsn = wp->api.get_redo_start_lsn(wp); if (wp->timelineStartLsn == InvalidXLogRecPtr) { - wp->timelineStartLsn = wp->api.get_redo_start_lsn(); + wp->timelineStartLsn = wp->api.get_redo_start_lsn(wp); } - elog(LOG, "bumped epochStartLsn to the first record %X/%X", LSN_FORMAT_ARGS(wp->propEpochStartLsn)); + walprop_log(LOG, "bumped epochStartLsn to the first record %X/%X", LSN_FORMAT_ARGS(wp->propEpochStartLsn)); } /* @@ -1030,7 +1037,7 @@ DetermineEpochStartLsn(WalProposer *wp) wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].term = wp->propTerm; wp->propTermHistory.entries[wp->propTermHistory.n_entries - 1].lsn = wp->propEpochStartLsn; - elog(LOG, "got votes from majority (%d) of nodes, term " UINT64_FORMAT ", epochStartLsn %X/%X, donor %s:%s, truncate_lsn %X/%X", + walprop_log(LOG, "got votes from majority (%d) of nodes, term " UINT64_FORMAT ", epochStartLsn %X/%X, donor %s:%s, truncate_lsn %X/%X", wp->quorum, wp->propTerm, LSN_FORMAT_ARGS(wp->propEpochStartLsn), @@ -1044,7 +1051,7 @@ DetermineEpochStartLsn(WalProposer *wp) */ if (!wp->config->syncSafekeepers) { - WalproposerShmemState *walprop_shared = wp->api.get_shmem_state(); + WalproposerShmemState *walprop_shared = wp->api.get_shmem_state(wp); /* * Basebackup LSN always points to the beginning of the record (not @@ -1052,7 +1059,7 @@ DetermineEpochStartLsn(WalProposer *wp) * Safekeepers don't skip header as they need continious stream of * data, so correct LSN for comparison. */ - if (SkipXLogPageHeader(wp, wp->propEpochStartLsn) != wp->api.get_redo_start_lsn()) + if (SkipXLogPageHeader(wp, wp->propEpochStartLsn) != wp->api.get_redo_start_lsn(wp)) { /* * However, allow to proceed if previously elected leader was me; @@ -1062,14 +1069,21 @@ DetermineEpochStartLsn(WalProposer *wp) if (!((dth->n_entries >= 1) && (dth->entries[dth->n_entries - 1].term == walprop_shared->mineLastElectedTerm))) { - elog(PANIC, + walprop_log(PANIC, "collected propEpochStartLsn %X/%X, but basebackup LSN %X/%X", LSN_FORMAT_ARGS(wp->propEpochStartLsn), - LSN_FORMAT_ARGS(wp->api.get_redo_start_lsn())); + LSN_FORMAT_ARGS(wp->api.get_redo_start_lsn(wp))); } } walprop_shared->mineLastElectedTerm = wp->propTerm; } + + /* + * WalProposer has just elected itself and initialized history, so + * we can call election callback. Usually it updates truncateLsn to + * fetch WAL for logical replication. + */ + wp->api.after_election(wp); } /* @@ -1140,7 +1154,7 @@ SendProposerElected(Safekeeper *sk) */ sk->startStreamingAt = wp->truncateLsn; - elog(WARNING, "empty safekeeper joined cluster as %s:%s, historyStart=%X/%X, sk->startStreamingAt=%X/%X", + walprop_log(WARNING, "empty safekeeper joined cluster as %s:%s, historyStart=%X/%X, sk->startStreamingAt=%X/%X", sk->host, sk->port, LSN_FORMAT_ARGS(wp->propTermHistory.entries[0].lsn), LSN_FORMAT_ARGS(sk->startStreamingAt)); } @@ -1175,7 +1189,7 @@ SendProposerElected(Safekeeper *sk) msg.timelineStartLsn = wp->timelineStartLsn; lastCommonTerm = i >= 0 ? wp->propTermHistory.entries[i].term : 0; - elog(LOG, + walprop_log(LOG, "sending elected msg to node " UINT64_FORMAT " term=" UINT64_FORMAT ", startStreamingAt=%X/%X (lastCommonTerm=" UINT64_FORMAT "), termHistory.n_entries=%u to %s:%s, timelineStartLsn=%X/%X", sk->greetResponse.nodeId, msg.term, LSN_FORMAT_ARGS(msg.startStreamingAt), lastCommonTerm, msg.termHistory->n_entries, sk->host, sk->port, LSN_FORMAT_ARGS(msg.timelineStartLsn)); @@ -1340,13 +1354,12 @@ SendAppendRequests(Safekeeper *sk) req = &sk->appendRequest; PrepareAppendRequest(sk->wp, &sk->appendRequest, sk->streamingAt, endLsn); - ereport(DEBUG2, - (errmsg("sending message len %ld beginLsn=%X/%X endLsn=%X/%X commitLsn=%X/%X truncateLsn=%X/%X to %s:%s", + walprop_log(DEBUG2, "sending message len %ld beginLsn=%X/%X endLsn=%X/%X commitLsn=%X/%X truncateLsn=%X/%X to %s:%s", req->endLsn - req->beginLsn, LSN_FORMAT_ARGS(req->beginLsn), LSN_FORMAT_ARGS(req->endLsn), LSN_FORMAT_ARGS(req->commitLsn), - LSN_FORMAT_ARGS(wp->truncateLsn), sk->host, sk->port))); + LSN_FORMAT_ARGS(wp->truncateLsn), sk->host, sk->port); resetStringInfo(&sk->outbuf); @@ -1356,13 +1369,13 @@ SendAppendRequests(Safekeeper *sk) /* write the WAL itself */ enlargeStringInfo(&sk->outbuf, req->endLsn - req->beginLsn); /* wal_read will raise error on failure */ - wp->api.wal_read(sk->xlogreader, + wp->api.wal_read(sk, &sk->outbuf.data[sk->outbuf.len], req->beginLsn, req->endLsn - req->beginLsn); sk->outbuf.len += req->endLsn - req->beginLsn; - writeResult = wp->api.conn_async_write(sk->conn, sk->outbuf.data, sk->outbuf.len); + writeResult = wp->api.conn_async_write(sk, sk->outbuf.data, sk->outbuf.len); /* Mark current message as sent, whatever the result is */ sk->streamingAt = endLsn; @@ -1384,9 +1397,9 @@ SendAppendRequests(Safekeeper *sk) return true; case PG_ASYNC_WRITE_FAIL: - elog(WARNING, "Failed to send to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to send to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; default: @@ -1424,17 +1437,16 @@ RecvAppendResponses(Safekeeper *sk) if (!AsyncReadMessage(sk, (AcceptorProposerMessage *) &sk->appendResponse)) break; - ereport(DEBUG2, - (errmsg("received message term=" INT64_FORMAT " flushLsn=%X/%X commitLsn=%X/%X from %s:%s", + walprop_log(DEBUG2, "received message term=" INT64_FORMAT " flushLsn=%X/%X commitLsn=%X/%X from %s:%s", sk->appendResponse.term, LSN_FORMAT_ARGS(sk->appendResponse.flushLsn), LSN_FORMAT_ARGS(sk->appendResponse.commitLsn), - sk->host, sk->port))); + sk->host, sk->port); if (sk->appendResponse.term > wp->propTerm) { /* Another compute with higher term is running. */ - elog(PANIC, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT "", + walprop_log(PANIC, "WAL acceptor %s:%s with term " INT64_FORMAT " rejected our request, our term " INT64_FORMAT "", sk->host, sk->port, sk->appendResponse.term, wp->propTerm); } @@ -1462,7 +1474,7 @@ RecvAppendResponses(Safekeeper *sk) /* Parse a PageserverFeedback message, or the PageserverFeedback part of an AppendResponse */ void -ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) +ParsePageserverFeedbackMessage(WalProposer *wp, StringInfo reply_message, PageserverFeedback *rf) { uint8 nkeys; int i; @@ -1480,7 +1492,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->currentClusterSize = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: current_timeline_size %lu", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: current_timeline_size %lu", rf->currentClusterSize); } else if ((strcmp(key, "ps_writelsn") == 0) || (strcmp(key, "last_received_lsn") == 0)) @@ -1488,7 +1500,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->last_received_lsn = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: last_received_lsn %X/%X", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: last_received_lsn %X/%X", LSN_FORMAT_ARGS(rf->last_received_lsn)); } else if ((strcmp(key, "ps_flushlsn") == 0) || (strcmp(key, "disk_consistent_lsn") == 0)) @@ -1496,7 +1508,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->disk_consistent_lsn = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: disk_consistent_lsn %X/%X", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: disk_consistent_lsn %X/%X", LSN_FORMAT_ARGS(rf->disk_consistent_lsn)); } else if ((strcmp(key, "ps_applylsn") == 0) || (strcmp(key, "remote_consistent_lsn") == 0)) @@ -1504,7 +1516,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) pq_getmsgint(reply_message, sizeof(int32)); /* read value length */ rf->remote_consistent_lsn = pq_getmsgint64(reply_message); - elog(DEBUG2, "ParsePageserverFeedbackMessage: remote_consistent_lsn %X/%X", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: remote_consistent_lsn %X/%X", LSN_FORMAT_ARGS(rf->remote_consistent_lsn)); } else if ((strcmp(key, "ps_replytime") == 0) || (strcmp(key, "replytime") == 0)) @@ -1517,7 +1529,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) /* Copy because timestamptz_to_str returns a static buffer */ replyTimeStr = pstrdup(timestamptz_to_str(rf->replytime)); - elog(DEBUG2, "ParsePageserverFeedbackMessage: replytime %lu reply_time: %s", + walprop_log(DEBUG2, "ParsePageserverFeedbackMessage: replytime %lu reply_time: %s", rf->replytime, replyTimeStr); pfree(replyTimeStr); @@ -1532,7 +1544,7 @@ ParsePageserverFeedbackMessage(StringInfo reply_message, PageserverFeedback *rf) * Skip unknown keys to support backward compatibile protocol * changes */ - elog(LOG, "ParsePageserverFeedbackMessage: unknown key: %s len %d", key, len); + walprop_log(LOG, "ParsePageserverFeedbackMessage: unknown key: %s len %d", key, len); pq_getmsgbytes(reply_message, len); }; } @@ -1615,7 +1627,7 @@ HandleSafekeeperResponse(WalProposer *wp) * Advance the replication slot to free up old WAL files. Note that * slot doesn't exist if we are in syncSafekeepers mode. */ - wp->api.confirm_wal_streamed(wp->truncateLsn); + wp->api.confirm_wal_streamed(wp, wp->truncateLsn); } /* @@ -1662,7 +1674,7 @@ HandleSafekeeperResponse(WalProposer *wp) */ BroadcastAppendRequest(wp); - wp->api.finish_sync_safekeepers(wp->propEpochStartLsn); + wp->api.finish_sync_safekeepers(wp, wp->propEpochStartLsn); /* unreachable */ } } @@ -1677,7 +1689,7 @@ AsyncRead(Safekeeper *sk, char **buf, int *buf_size) { WalProposer *wp = sk->wp; - switch (wp->api.conn_async_read(sk->conn, buf, buf_size)) + switch (wp->api.conn_async_read(sk, buf, buf_size)) { case PG_ASYNC_READ_SUCCESS: return true; @@ -1687,9 +1699,9 @@ AsyncRead(Safekeeper *sk, char **buf, int *buf_size) return false; case PG_ASYNC_READ_FAIL: - elog(WARNING, "Failed to read from node %s:%s in %s state: %s", sk->host, + walprop_log(WARNING, "Failed to read from node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; } @@ -1727,12 +1739,12 @@ AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg) tag = pq_getmsgint64_le(&s); if (tag != anymsg->tag) { - elog(WARNING, "unexpected message tag %c from node %s:%s in state %s", (char) tag, sk->host, + walprop_log(WARNING, "unexpected message tag %c from node %s:%s in state %s", (char) tag, sk->host, sk->port, FormatSafekeeperState(sk->state)); ResetConnection(sk); return false; } - sk->latestMsgReceivedAt = wp->api.get_current_timestamp(); + sk->latestMsgReceivedAt = wp->api.get_current_timestamp(wp); switch (tag) { case 'g': @@ -1776,7 +1788,7 @@ AsyncReadMessage(Safekeeper *sk, AcceptorProposerMessage *anymsg) msg->hs.xmin.value = pq_getmsgint64_le(&s); msg->hs.catalog_xmin.value = pq_getmsgint64_le(&s); if (buf_size > APPENDRESPONSE_FIXEDPART_SIZE) - ParsePageserverFeedbackMessage(&s, &msg->rf); + ParsePageserverFeedbackMessage(wp, &s, &msg->rf); pq_getmsgend(&s); return true; } @@ -1801,11 +1813,11 @@ BlockingWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState succes WalProposer *wp = sk->wp; uint32 events; - if (!wp->api.conn_blocking_write(sk->conn, msg, msg_size)) + if (!wp->api.conn_blocking_write(sk, msg, msg_size)) { - elog(WARNING, "Failed to send to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to send to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; } @@ -1835,7 +1847,7 @@ AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_sta { WalProposer *wp = sk->wp; - switch (wp->api.conn_async_write(sk->conn, msg, msg_size)) + switch (wp->api.conn_async_write(sk, msg, msg_size)) { case PG_ASYNC_WRITE_SUCCESS: return true; @@ -1850,9 +1862,9 @@ AsyncWrite(Safekeeper *sk, void *msg, size_t msg_size, SafekeeperState flush_sta wp->api.update_event_set(sk, WL_SOCKET_READABLE | WL_SOCKET_WRITEABLE); return false; case PG_ASYNC_WRITE_FAIL: - elog(WARNING, "Failed to send to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to send to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ShutdownConnection(sk); return false; default: @@ -1880,7 +1892,7 @@ AsyncFlush(Safekeeper *sk) * 1 if unable to send everything yet [call PQflush again] * -1 if it failed [emit an error] */ - switch (wp->api.conn_flush(sk->conn)) + switch (wp->api.conn_flush(sk)) { case 0: /* flush is done */ @@ -1889,9 +1901,9 @@ AsyncFlush(Safekeeper *sk) /* Nothing to do; try again when the socket's ready */ return false; case -1: - elog(WARNING, "Failed to flush write to node %s:%s in %s state: %s", + walprop_log(WARNING, "Failed to flush write to node %s:%s in %s state: %s", sk->host, sk->port, FormatSafekeeperState(sk->state), - wp->api.conn_error_message(sk->conn)); + wp->api.conn_error_message(sk)); ResetConnection(sk); return false; default: @@ -1920,11 +1932,11 @@ CompareLsn(const void *a, const void *b) * * The strings are intended to be used as a prefix to "state", e.g.: * - * elog(LOG, "currently in %s state", FormatSafekeeperState(sk->state)); + * walprop_log(LOG, "currently in %s state", FormatSafekeeperState(sk->state)); * * If this sort of phrasing doesn't fit the message, instead use something like: * - * elog(LOG, "currently in state [%s]", FormatSafekeeperState(sk->state)); + * walprop_log(LOG, "currently in state [%s]", FormatSafekeeperState(sk->state)); */ static char * FormatSafekeeperState(SafekeeperState state) @@ -1972,6 +1984,7 @@ FormatSafekeeperState(SafekeeperState state) static void AssertEventsOkForState(uint32 events, Safekeeper *sk) { + WalProposer *wp = sk->wp; uint32 expected = SafekeeperStateDesiredEvents(sk->state); /* @@ -1994,8 +2007,8 @@ AssertEventsOkForState(uint32 events, Safekeeper *sk) * To give a descriptive message in the case of failure, we use elog * and then an assertion that's guaranteed to fail. */ - elog(WARNING, "events %s mismatched for safekeeper %s:%s in state [%s]", - FormatEvents(events), sk->host, sk->port, FormatSafekeeperState(sk->state)); + walprop_log(WARNING, "events %s mismatched for safekeeper %s:%s in state [%s]", + FormatEvents(wp, events), sk->host, sk->port, FormatSafekeeperState(sk->state)); Assert(events_ok_for_state); } } @@ -2068,7 +2081,7 @@ SafekeeperStateDesiredEvents(SafekeeperState state) * The string should not be freed. It should also not be expected to remain the same between * function calls. */ static char * -FormatEvents(uint32 events) +FormatEvents(WalProposer *wp, uint32 events) { static char return_str[8]; @@ -2097,7 +2110,7 @@ FormatEvents(uint32 events) if (events & (~all_flags)) { - elog(WARNING, "Event formatting found unexpected component %d", + walprop_log(WARNING, "Event formatting found unexpected component %d", events & (~all_flags)); return_str[6] = '*'; return_str[7] = '\0'; diff --git a/pgxn/neon/walproposer.h b/pgxn/neon/walproposer.h index a1a9ccdfdd..664aeedfa7 100644 --- a/pgxn/neon/walproposer.h +++ b/pgxn/neon/walproposer.h @@ -333,24 +333,11 @@ typedef struct Safekeeper */ char conninfo[MAXCONNINFO]; - /* - * postgres protocol connection to the WAL acceptor - * - * Equals NULL only when state = SS_OFFLINE. Nonblocking is set once we - * reach SS_ACTIVE; not before. - */ - WalProposerConn *conn; - /* * Temporary buffer for the message being sent to the safekeeper. */ StringInfoData outbuf; - /* - * WAL reader, allocated for each safekeeper. - */ - XLogReaderState *xlogreader; - /* * Streaming will start here; must be record boundary. */ @@ -361,13 +348,43 @@ typedef struct Safekeeper XLogRecPtr streamingAt; /* current streaming position */ AppendRequestHeader appendRequest; /* request for sending to safekeeper */ - int eventPos; /* position in wait event set. Equal to -1 if* - * no event */ SafekeeperState state; /* safekeeper state machine state */ TimestampTz latestMsgReceivedAt; /* when latest msg is received */ AcceptorGreeting greetResponse; /* acceptor greeting */ VoteResponse voteResponse; /* the vote */ AppendResponse appendResponse; /* feedback for master */ + + + /* postgres-specific fields */ + #ifndef WALPROPOSER_LIB + /* + * postgres protocol connection to the WAL acceptor + * + * Equals NULL only when state = SS_OFFLINE. Nonblocking is set once we + * reach SS_ACTIVE; not before. + */ + WalProposerConn *conn; + + /* + * WAL reader, allocated for each safekeeper. + */ + XLogReaderState *xlogreader; + + /* + * Position in wait event set. Equal to -1 if no event + */ + int eventPos; + #endif + + + /* WalProposer library specifics */ + #ifdef WALPROPOSER_LIB + /* + * Buffer for incoming messages. Usually Rust vector is stored here. + * Caller is responsible for freeing the buffer. + */ + StringInfoData inbuf; + #endif } Safekeeper; /* Re-exported PostgresPollingStatusType */ @@ -433,7 +450,7 @@ typedef struct walproposer_api * Get WalproposerShmemState. This is used to store information about last * elected term. */ - WalproposerShmemState *(*get_shmem_state) (void); + WalproposerShmemState *(*get_shmem_state) (WalProposer *wp); /* * Start receiving notifications about new WAL. This is an infinite loop @@ -443,61 +460,63 @@ typedef struct walproposer_api void (*start_streaming) (WalProposer *wp, XLogRecPtr startpos); /* Get pointer to the latest available WAL. */ - XLogRecPtr (*get_flush_rec_ptr) (void); + XLogRecPtr (*get_flush_rec_ptr) (WalProposer *wp); /* Get current time. */ - TimestampTz (*get_current_timestamp) (void); - - /* Get postgres timeline. */ - TimeLineID (*get_timeline_id) (void); + TimestampTz (*get_current_timestamp) (WalProposer *wp); /* Current error message, aka PQerrorMessage. */ - char *(*conn_error_message) (WalProposerConn *conn); + char *(*conn_error_message) (Safekeeper *sk); /* Connection status, aka PQstatus. */ - WalProposerConnStatusType (*conn_status) (WalProposerConn *conn); + WalProposerConnStatusType (*conn_status) (Safekeeper *sk); /* Start the connection, aka PQconnectStart. */ - WalProposerConn *(*conn_connect_start) (char *conninfo); + void (*conn_connect_start) (Safekeeper *sk); /* Poll an asynchronous connection, aka PQconnectPoll. */ - WalProposerConnectPollStatusType (*conn_connect_poll) (WalProposerConn *conn); + WalProposerConnectPollStatusType (*conn_connect_poll) (Safekeeper *sk); /* Send a blocking SQL query, aka PQsendQuery. */ - bool (*conn_send_query) (WalProposerConn *conn, char *query); + bool (*conn_send_query) (Safekeeper *sk, char *query); /* Read the query result, aka PQgetResult. */ - WalProposerExecStatusType (*conn_get_query_result) (WalProposerConn *conn); + WalProposerExecStatusType (*conn_get_query_result) (Safekeeper *sk); /* Flush buffer to the network, aka PQflush. */ - int (*conn_flush) (WalProposerConn *conn); + int (*conn_flush) (Safekeeper *sk); /* Close the connection, aka PQfinish. */ - void (*conn_finish) (WalProposerConn *conn); + void (*conn_finish) (Safekeeper *sk); - /* Try to read CopyData message, aka PQgetCopyData. */ - PGAsyncReadResult (*conn_async_read) (WalProposerConn *conn, char **buf, int *amount); + /* + * Try to read CopyData message from the safekeeper, aka PQgetCopyData. + * + * On success, the data is placed in *buf. It is valid until the next call + * to this function. + */ + PGAsyncReadResult (*conn_async_read) (Safekeeper *sk, char **buf, int *amount); /* Try to write CopyData message, aka PQputCopyData. */ - PGAsyncWriteResult (*conn_async_write) (WalProposerConn *conn, void const *buf, size_t size); + PGAsyncWriteResult (*conn_async_write) (Safekeeper *sk, void const *buf, size_t size); /* Blocking CopyData write, aka PQputCopyData + PQflush. */ - bool (*conn_blocking_write) (WalProposerConn *conn, void const *buf, size_t size); + bool (*conn_blocking_write) (Safekeeper *sk, void const *buf, size_t size); /* Download WAL from startpos to endpos and make it available locally. */ bool (*recovery_download) (Safekeeper *sk, TimeLineID timeline, XLogRecPtr startpos, XLogRecPtr endpos); /* Read WAL from disk to buf. */ - void (*wal_read) (XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count); + void (*wal_read) (Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count); /* Allocate WAL reader. */ - XLogReaderState *(*wal_reader_allocate) (void); + void (*wal_reader_allocate) (Safekeeper *sk); /* Deallocate event set. */ - void (*free_event_set) (void); + void (*free_event_set) (WalProposer *wp); /* Initialize event set. */ - void (*init_event_set) (int n_safekeepers); + void (*init_event_set) (WalProposer *wp); /* Update events for an existing safekeeper connection. */ void (*update_event_set) (Safekeeper *sk, uint32 events); @@ -513,22 +532,22 @@ typedef struct walproposer_api * events mask to indicate events and sets sk to the safekeeper which has * an event. */ - int (*wait_event_set) (long timeout, Safekeeper **sk, uint32 *events); + int (*wait_event_set) (WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events); /* Read random bytes. */ - bool (*strong_random) (void *buf, size_t len); + bool (*strong_random) (WalProposer *wp, void *buf, size_t len); /* * Get a basebackup LSN. Used to cross-validate with the latest available * LSN on the safekeepers. */ - XLogRecPtr (*get_redo_start_lsn) (void); + XLogRecPtr (*get_redo_start_lsn) (WalProposer *wp); /* * Finish sync safekeepers with the given LSN. This function should not * return and should exit the program. */ - void (*finish_sync_safekeepers) (XLogRecPtr lsn); + void (*finish_sync_safekeepers) (WalProposer *wp, XLogRecPtr lsn); /* * Called after every new message from the safekeeper. Used to propagate @@ -541,7 +560,22 @@ typedef struct walproposer_api * Called on peer_horizon_lsn updates. Used to advance replication slot * and to free up disk space by deleting unnecessary WAL. */ - void (*confirm_wal_streamed) (XLogRecPtr lsn); + void (*confirm_wal_streamed) (WalProposer *wp, XLogRecPtr lsn); + + /* + * Write a log message to the internal log processor. This is used only + * when walproposer is compiled as a library. Otherwise, all logging is + * handled by elog(). + */ + void (*log_internal) (WalProposer *wp, int level, const char *line); + + /* + * Called right after the proposer was elected, but before it started + * recovery and sent ProposerElected message to the safekeepers. + * + * Used by logical replication to update truncateLsn. + */ + void (*after_election) (WalProposer *wp); } walproposer_api; /* @@ -590,6 +624,13 @@ typedef struct WalProposerConfig /* Will be passed to safekeepers in greet request. */ uint64 systemId; + + /* Will be passed to safekeepers in greet request. */ + TimeLineID pgTimeline; + +#ifdef WALPROPOSER_LIB + void *callback_data; +#endif } WalProposerConfig; @@ -666,7 +707,16 @@ extern WalProposer *WalProposerCreate(WalProposerConfig *config, walproposer_api extern void WalProposerStart(WalProposer *wp); extern void WalProposerBroadcast(WalProposer *wp, XLogRecPtr startpos, XLogRecPtr endpos); extern void WalProposerPoll(WalProposer *wp); -extern void ParsePageserverFeedbackMessage(StringInfo reply_message, - PageserverFeedback *rf); +extern void WalProposerFree(WalProposer *wp); + + +#define WPEVENT 1337 /* special log level for walproposer internal events */ + +#ifdef WALPROPOSER_LIB +void WalProposerLibLog(WalProposer *wp, int elevel, char *fmt, ...); +#define walprop_log(elevel, ...) WalProposerLibLog(wp, elevel, __VA_ARGS__) +#else +#define walprop_log(elevel, ...) elog(elevel, __VA_ARGS__) +#endif #endif /* __NEON_WALPROPOSER_H__ */ diff --git a/pgxn/neon/walproposer_compat.c b/pgxn/neon/walproposer_compat.c new file mode 100644 index 0000000000..7617f21a26 --- /dev/null +++ b/pgxn/neon/walproposer_compat.c @@ -0,0 +1,192 @@ +/* + * Contains copied/adapted functions from libpq and some internal postgres functions. + * This is needed to avoid linking to full postgres server installation. This file + * is compiled as a part of libwalproposer static library. + */ + +#include +#include "walproposer.h" +#include "utils/datetime.h" +#include "miscadmin.h" + +void ExceptionalCondition(const char *conditionName, + const char *fileName, int lineNumber) +{ + fprintf(stderr, "ExceptionalCondition: %s:%d: %s\n", + fileName, lineNumber, conditionName); + fprintf(stderr, "aborting...\n"); + exit(1); +} + +void +pq_copymsgbytes(StringInfo msg, char *buf, int datalen) +{ + if (datalen < 0 || datalen > (msg->len - msg->cursor)) + ExceptionalCondition("insufficient data left in message", __FILE__, __LINE__); + memcpy(buf, &msg->data[msg->cursor], datalen); + msg->cursor += datalen; +} + +/* -------------------------------- + * pq_getmsgint - get a binary integer from a message buffer + * + * Values are treated as unsigned. + * -------------------------------- + */ +unsigned int +pq_getmsgint(StringInfo msg, int b) +{ + unsigned int result; + unsigned char n8; + uint16 n16; + uint32 n32; + + switch (b) + { + case 1: + pq_copymsgbytes(msg, (char *) &n8, 1); + result = n8; + break; + case 2: + pq_copymsgbytes(msg, (char *) &n16, 2); + result = pg_ntoh16(n16); + break; + case 4: + pq_copymsgbytes(msg, (char *) &n32, 4); + result = pg_ntoh32(n32); + break; + default: + fprintf(stderr, "unsupported integer size %d\n", b); + ExceptionalCondition("unsupported integer size", __FILE__, __LINE__); + result = 0; /* keep compiler quiet */ + break; + } + return result; +} + +/* -------------------------------- + * pq_getmsgint64 - get a binary 8-byte int from a message buffer + * + * It is tempting to merge this with pq_getmsgint, but we'd have to make the + * result int64 for all data widths --- that could be a big performance + * hit on machines where int64 isn't efficient. + * -------------------------------- + */ +int64 +pq_getmsgint64(StringInfo msg) +{ + uint64 n64; + + pq_copymsgbytes(msg, (char *) &n64, sizeof(n64)); + + return pg_ntoh64(n64); +} + +/* -------------------------------- + * pq_getmsgbyte - get a raw byte from a message buffer + * -------------------------------- + */ +int +pq_getmsgbyte(StringInfo msg) +{ + if (msg->cursor >= msg->len) + ExceptionalCondition("no data left in message", __FILE__, __LINE__); + return (unsigned char) msg->data[msg->cursor++]; +} + +/* -------------------------------- + * pq_getmsgbytes - get raw data from a message buffer + * + * Returns a pointer directly into the message buffer; note this + * may not have any particular alignment. + * -------------------------------- + */ +const char * +pq_getmsgbytes(StringInfo msg, int datalen) +{ + const char *result; + + if (datalen < 0 || datalen > (msg->len - msg->cursor)) + ExceptionalCondition("insufficient data left in message", __FILE__, __LINE__); + result = &msg->data[msg->cursor]; + msg->cursor += datalen; + return result; +} + +/* -------------------------------- + * pq_getmsgstring - get a null-terminated text string (with conversion) + * + * May return a pointer directly into the message buffer, or a pointer + * to a palloc'd conversion result. + * -------------------------------- + */ +const char * +pq_getmsgstring(StringInfo msg) +{ + char *str; + int slen; + + str = &msg->data[msg->cursor]; + + /* + * It's safe to use strlen() here because a StringInfo is guaranteed to + * have a trailing null byte. But check we found a null inside the + * message. + */ + slen = strlen(str); + if (msg->cursor + slen >= msg->len) + ExceptionalCondition("invalid string in message", __FILE__, __LINE__); + msg->cursor += slen + 1; + + return str; +} + +/* -------------------------------- + * pq_getmsgend - verify message fully consumed + * -------------------------------- + */ +void +pq_getmsgend(StringInfo msg) +{ + if (msg->cursor != msg->len) + ExceptionalCondition("invalid msg format", __FILE__, __LINE__); +} + + +/* + * Produce a C-string representation of a TimestampTz. + * + * This is mostly for use in emitting messages. + */ +const char * +timestamptz_to_str(TimestampTz t) +{ + static char buf[MAXDATELEN + 1]; + + snprintf(buf, sizeof(buf), "TimestampTz(%ld)", t); + return buf; +} + +bool +TimestampDifferenceExceeds(TimestampTz start_time, + TimestampTz stop_time, + int msec) +{ + TimestampTz diff = stop_time - start_time; + return (diff >= msec * INT64CONST(1000)); +} + +void +WalProposerLibLog(WalProposer *wp, int elevel, char *fmt, ...) +{ + char buf[1024]; + va_list args; + + fmt = _(fmt); + + va_start(args, fmt); + vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + + wp->api.log_internal(wp, elevel, buf); +} diff --git a/pgxn/neon/walproposer_pg.c b/pgxn/neon/walproposer_pg.c index 654b411e94..865f91165b 100644 --- a/pgxn/neon/walproposer_pg.c +++ b/pgxn/neon/walproposer_pg.c @@ -73,7 +73,8 @@ static void walprop_register_bgworker(void); static void walprop_pg_init_standalone_sync_safekeepers(void); static void walprop_pg_init_walsender(void); static void walprop_pg_init_bgworker(void); -static TimestampTz walprop_pg_get_current_timestamp(void); +static TimestampTz walprop_pg_get_current_timestamp(WalProposer *wp); +static TimeLineID walprop_pg_get_timeline_id(void); static void walprop_pg_load_libpqwalreceiver(void); static process_interrupts_callback_t PrevProcessInterruptsCallback; @@ -104,6 +105,7 @@ init_walprop_config(bool syncSafekeepers) walprop_config.systemId = GetSystemIdentifier(); else walprop_config.systemId = 0; + walprop_config.pgTimeline = walprop_pg_get_timeline_id(); } /* @@ -136,7 +138,7 @@ WalProposerMain(Datum main_arg) walprop_pg_load_libpqwalreceiver(); wp = WalProposerCreate(&walprop_config, walprop_pg); - wp->last_reconnect_attempt = walprop_pg_get_current_timestamp(); + wp->last_reconnect_attempt = walprop_pg_get_current_timestamp(wp); walprop_pg_init_walsender(); WalProposerStart(wp); @@ -379,7 +381,7 @@ nwp_shmem_startup_hook(void) } static WalproposerShmemState * -walprop_pg_get_shmem_state(void) +walprop_pg_get_shmem_state(WalProposer *wp) { Assert(walprop_shared != NULL); return walprop_shared; @@ -505,7 +507,7 @@ walprop_pg_init_bgworker(void) } static XLogRecPtr -walprop_pg_get_flush_rec_ptr(void) +walprop_pg_get_flush_rec_ptr(WalProposer *wp) { #if PG_MAJORVERSION_NUM < 15 return GetFlushRecPtr(); @@ -515,7 +517,7 @@ walprop_pg_get_flush_rec_ptr(void) } static TimestampTz -walprop_pg_get_current_timestamp(void) +walprop_pg_get_current_timestamp(WalProposer *wp) { return GetCurrentTimestamp(); } @@ -565,15 +567,15 @@ ensure_nonblocking_status(WalProposerConn *conn, bool is_nonblocking) /* Exported function definitions */ static char * -walprop_error_message(WalProposerConn *conn) +walprop_error_message(Safekeeper *sk) { - return PQerrorMessage(conn->pg_conn); + return PQerrorMessage(sk->conn->pg_conn); } static WalProposerConnStatusType -walprop_status(WalProposerConn *conn) +walprop_status(Safekeeper *sk) { - switch (PQstatus(conn->pg_conn)) + switch (PQstatus(sk->conn->pg_conn)) { case CONNECTION_OK: return WP_CONNECTION_OK; @@ -584,16 +586,17 @@ walprop_status(WalProposerConn *conn) } } -static WalProposerConn * -walprop_connect_start(char *conninfo) +static void +walprop_connect_start(Safekeeper *sk) { - WalProposerConn *conn; PGconn *pg_conn; const char *keywords[3]; const char *values[3]; int n; char *password = neon_auth_token; + Assert(sk->conn == NULL); + /* * Connect using the given connection string. If the NEON_AUTH_TOKEN * environment variable was set, use that as the password. @@ -611,7 +614,7 @@ walprop_connect_start(char *conninfo) n++; } keywords[n] = "dbname"; - values[n] = conninfo; + values[n] = sk->conninfo; n++; keywords[n] = NULL; values[n] = NULL; @@ -619,11 +622,11 @@ walprop_connect_start(char *conninfo) pg_conn = PQconnectStartParams(keywords, values, 1); /* - * Allocation of a PQconn can fail, and will return NULL. We want to fully - * replicate the behavior of PQconnectStart here. + * "If the result is null, then libpq has been unable to allocate a new + * PGconn structure" */ if (!pg_conn) - return NULL; + elog(FATAL, "failed to allocate new PGconn object"); /* * And in theory this allocation can fail as well, but it's incredibly @@ -632,20 +635,19 @@ walprop_connect_start(char *conninfo) * palloc will exit on failure though, so there's not much we could do if * it *did* fail. */ - conn = palloc(sizeof(WalProposerConn)); - conn->pg_conn = pg_conn; - conn->is_nonblocking = false; /* connections always start in blocking + sk->conn = palloc(sizeof(WalProposerConn)); + sk->conn->pg_conn = pg_conn; + sk->conn->is_nonblocking = false; /* connections always start in blocking * mode */ - conn->recvbuf = NULL; - return conn; + sk->conn->recvbuf = NULL; } static WalProposerConnectPollStatusType -walprop_connect_poll(WalProposerConn *conn) +walprop_connect_poll(Safekeeper *sk) { WalProposerConnectPollStatusType return_val; - switch (PQconnectPoll(conn->pg_conn)) + switch (PQconnectPoll(sk->conn->pg_conn)) { case PGRES_POLLING_FAILED: return_val = WP_CONN_POLLING_FAILED; @@ -682,24 +684,24 @@ walprop_connect_poll(WalProposerConn *conn) } static bool -walprop_send_query(WalProposerConn *conn, char *query) +walprop_send_query(Safekeeper *sk, char *query) { /* * We need to be in blocking mode for sending the query to run without * requiring a call to PQflush */ - if (!ensure_nonblocking_status(conn, false)) + if (!ensure_nonblocking_status(sk->conn, false)) return false; /* PQsendQuery returns 1 on success, 0 on failure */ - if (!PQsendQuery(conn->pg_conn, query)) + if (!PQsendQuery(sk->conn->pg_conn, query)) return false; return true; } static WalProposerExecStatusType -walprop_get_query_result(WalProposerConn *conn) +walprop_get_query_result(Safekeeper *sk) { PGresult *result; WalProposerExecStatusType return_val; @@ -708,14 +710,14 @@ walprop_get_query_result(WalProposerConn *conn) char *unexpected_success = NULL; /* Consume any input that we might be missing */ - if (!PQconsumeInput(conn->pg_conn)) + if (!PQconsumeInput(sk->conn->pg_conn)) return WP_EXEC_FAILED; - if (PQisBusy(conn->pg_conn)) + if (PQisBusy(sk->conn->pg_conn)) return WP_EXEC_NEEDS_INPUT; - result = PQgetResult(conn->pg_conn); + result = PQgetResult(sk->conn->pg_conn); /* * PQgetResult returns NULL only if getting the result was successful & @@ -777,24 +779,28 @@ walprop_get_query_result(WalProposerConn *conn) } static pgsocket -walprop_socket(WalProposerConn *conn) +walprop_socket(Safekeeper *sk) { - return PQsocket(conn->pg_conn); + return PQsocket(sk->conn->pg_conn); } static int -walprop_flush(WalProposerConn *conn) +walprop_flush(Safekeeper *sk) { - return (PQflush(conn->pg_conn)); + return (PQflush(sk->conn->pg_conn)); } static void -walprop_finish(WalProposerConn *conn) +walprop_finish(Safekeeper *sk) { - if (conn->recvbuf != NULL) - PQfreemem(conn->recvbuf); - PQfinish(conn->pg_conn); - pfree(conn); + if (!sk->conn) + return; + + if (sk->conn->recvbuf != NULL) + PQfreemem(sk->conn->recvbuf); + PQfinish(sk->conn->pg_conn); + pfree(sk->conn); + sk->conn = NULL; } /* @@ -804,18 +810,18 @@ walprop_finish(WalProposerConn *conn) * to this function. */ static PGAsyncReadResult -walprop_async_read(WalProposerConn *conn, char **buf, int *amount) +walprop_async_read(Safekeeper *sk, char **buf, int *amount) { int result; - if (conn->recvbuf != NULL) + if (sk->conn->recvbuf != NULL) { - PQfreemem(conn->recvbuf); - conn->recvbuf = NULL; + PQfreemem(sk->conn->recvbuf); + sk->conn->recvbuf = NULL; } /* Call PQconsumeInput so that we have the data we need */ - if (!PQconsumeInput(conn->pg_conn)) + if (!PQconsumeInput(sk->conn->pg_conn)) { *amount = 0; *buf = NULL; @@ -833,7 +839,7 @@ walprop_async_read(WalProposerConn *conn, char **buf, int *amount) * sometimes be triggered by the server returning an ErrorResponse (which * also happens to have the effect that the copy is done). */ - switch (result = PQgetCopyData(conn->pg_conn, &conn->recvbuf, true)) + switch (result = PQgetCopyData(sk->conn->pg_conn, &sk->conn->recvbuf, true)) { case 0: *amount = 0; @@ -848,7 +854,7 @@ walprop_async_read(WalProposerConn *conn, char **buf, int *amount) * We can check PQgetResult to make sure that the server * failed; it'll always result in PGRES_FATAL_ERROR */ - ExecStatusType status = PQresultStatus(PQgetResult(conn->pg_conn)); + ExecStatusType status = PQresultStatus(PQgetResult(sk->conn->pg_conn)); if (status != PGRES_FATAL_ERROR) elog(FATAL, "unexpected result status %d after failed PQgetCopyData", status); @@ -869,18 +875,18 @@ walprop_async_read(WalProposerConn *conn, char **buf, int *amount) default: /* Positive values indicate the size of the returned result */ *amount = result; - *buf = conn->recvbuf; + *buf = sk->conn->recvbuf; return PG_ASYNC_READ_SUCCESS; } } static PGAsyncWriteResult -walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) +walprop_async_write(Safekeeper *sk, void const *buf, size_t size) { int result; /* If we aren't in non-blocking mode, switch to it. */ - if (!ensure_nonblocking_status(conn, true)) + if (!ensure_nonblocking_status(sk->conn, true)) return PG_ASYNC_WRITE_FAIL; /* @@ -888,7 +894,7 @@ walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) * queued, 0 if it was not queued because of full buffers, or -1 if an * error occurred */ - result = PQputCopyData(conn->pg_conn, buf, size); + result = PQputCopyData(sk->conn->pg_conn, buf, size); /* * We won't get a result of zero because walproposer always empties the @@ -916,7 +922,7 @@ walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) * sucessful, 1 if it was unable to send all the data in the send queue * yet -1 if it failed for some reason */ - switch (result = PQflush(conn->pg_conn)) + switch (result = PQflush(sk->conn->pg_conn)) { case 0: return PG_ASYNC_WRITE_SUCCESS; @@ -934,22 +940,22 @@ walprop_async_write(WalProposerConn *conn, void const *buf, size_t size) * information, refer to the comments there. */ static bool -walprop_blocking_write(WalProposerConn *conn, void const *buf, size_t size) +walprop_blocking_write(Safekeeper *sk, void const *buf, size_t size) { int result; /* If we are in non-blocking mode, switch out of it. */ - if (!ensure_nonblocking_status(conn, false)) + if (!ensure_nonblocking_status(sk->conn, false)) return false; - if ((result = PQputCopyData(conn->pg_conn, buf, size)) == -1) + if ((result = PQputCopyData(sk->conn->pg_conn, buf, size)) == -1) return false; Assert(result == 1); /* Because the connection is non-blocking, flushing returns 0 or -1 */ - if ((result = PQflush(conn->pg_conn)) == -1) + if ((result = PQflush(sk->conn->pg_conn)) == -1) return false; Assert(result == 0); @@ -1381,11 +1387,11 @@ XLogWalPropClose(XLogRecPtr recptr) } static void -walprop_pg_wal_read(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size count) +walprop_pg_wal_read(Safekeeper *sk, char *buf, XLogRecPtr startptr, Size count) { WALReadError errinfo; - if (!WALRead(state, + if (!WALRead(sk->xlogreader, buf, startptr, count, @@ -1396,31 +1402,38 @@ walprop_pg_wal_read(XLogReaderState *state, char *buf, XLogRecPtr startptr, Size } } -static XLogReaderState * -walprop_pg_wal_reader_allocate(void) +static void +walprop_pg_wal_reader_allocate(Safekeeper *sk) { - return XLogReaderAllocate(wal_segment_size, NULL, XL_ROUTINE(.segment_open = wal_segment_open,.segment_close = wal_segment_close), NULL); + sk->xlogreader = XLogReaderAllocate(wal_segment_size, NULL, XL_ROUTINE(.segment_open = wal_segment_open,.segment_close = wal_segment_close), NULL); + if (sk->xlogreader == NULL) + elog(FATAL, "Failed to allocate xlog reader"); } static WaitEventSet *waitEvents; static void -walprop_pg_free_event_set(void) +walprop_pg_free_event_set(WalProposer *wp) { if (waitEvents) { FreeWaitEventSet(waitEvents); waitEvents = NULL; } + + for (int i = 0; i < wp->n_safekeepers; i++) + { + wp->safekeeper[i].eventPos = -1; + } } static void -walprop_pg_init_event_set(int n_safekeepers) +walprop_pg_init_event_set(WalProposer *wp) { if (waitEvents) elog(FATAL, "double-initialization of event set"); - waitEvents = CreateWaitEventSet(TopMemoryContext, 2 + n_safekeepers); + waitEvents = CreateWaitEventSet(TopMemoryContext, 2 + wp->n_safekeepers); AddWaitEventToSet(waitEvents, WL_LATCH_SET, PGINVALID_SOCKET, MyLatch, NULL); AddWaitEventToSet(waitEvents, WL_EXIT_ON_PM_DEATH, PGINVALID_SOCKET, @@ -1439,11 +1452,11 @@ walprop_pg_update_event_set(Safekeeper *sk, uint32 events) static void walprop_pg_add_safekeeper_event_set(Safekeeper *sk, uint32 events) { - sk->eventPos = AddWaitEventToSet(waitEvents, events, walprop_socket(sk->conn), NULL, sk); + sk->eventPos = AddWaitEventToSet(waitEvents, events, walprop_socket(sk), NULL, sk); } static int -walprop_pg_wait_event_set(long timeout, Safekeeper **sk, uint32 *events) +walprop_pg_wait_event_set(WalProposer *wp, long timeout, Safekeeper **sk, uint32 *events) { WaitEvent event = {0}; int rc = 0; @@ -1499,7 +1512,7 @@ walprop_pg_wait_event_set(long timeout, Safekeeper **sk, uint32 *events) } static void -walprop_pg_finish_sync_safekeepers(XLogRecPtr lsn) +walprop_pg_finish_sync_safekeepers(WalProposer *wp, XLogRecPtr lsn) { fprintf(stdout, "%X/%X\n", LSN_FORMAT_ARGS(lsn)); exit(0); @@ -1611,7 +1624,7 @@ walprop_pg_process_safekeeper_feedback(WalProposer *wp, XLogRecPtr commitLsn) * pageserver. */ quorumFeedback.rf.disk_consistent_lsn, - walprop_pg_get_current_timestamp(), false); + walprop_pg_get_current_timestamp(wp), false); } CombineHotStanbyFeedbacks(&hsFeedback, wp); @@ -1628,18 +1641,65 @@ walprop_pg_process_safekeeper_feedback(WalProposer *wp, XLogRecPtr commitLsn) } static void -walprop_pg_confirm_wal_streamed(XLogRecPtr lsn) +walprop_pg_confirm_wal_streamed(WalProposer *wp, XLogRecPtr lsn) { if (MyReplicationSlot) PhysicalConfirmReceivedLocation(lsn); } +static XLogRecPtr +walprop_pg_get_redo_start_lsn(WalProposer *wp) +{ + return GetRedoStartLsn(); +} + +static bool +walprop_pg_strong_random(WalProposer *wp, void *buf, size_t len) +{ + return pg_strong_random(buf, len); +} + +static void +walprop_pg_log_internal(WalProposer *wp, int level, const char *line) +{ + elog(FATAL, "unexpected log_internal message at level %d: %s", level, line); +} + +static void +walprop_pg_after_election(WalProposer *wp) +{ + FILE* f; + XLogRecPtr lrRestartLsn; + + /* We don't need to do anything in syncSafekeepers mode.*/ + if (wp->config->syncSafekeepers) + return; + + /* + * If there are active logical replication subscription we need + * to provide enough WAL for their WAL senders based on th position + * of their replication slots. + */ + f = fopen("restart.lsn", "rb"); + if (f != NULL && !wp->config->syncSafekeepers) + { + fread(&lrRestartLsn, sizeof(lrRestartLsn), 1, f); + fclose(f); + if (lrRestartLsn != InvalidXLogRecPtr) + { + elog(LOG, "Logical replication restart LSN %X/%X", LSN_FORMAT_ARGS(lrRestartLsn)); + /* start from the beginning of the segment to fetch page headers verifed by XLogReader */ + lrRestartLsn = lrRestartLsn - XLogSegmentOffset(lrRestartLsn, wal_segment_size); + wp->truncateLsn = Min(wp->truncateLsn, lrRestartLsn); + } + } +} + static const walproposer_api walprop_pg = { .get_shmem_state = walprop_pg_get_shmem_state, .start_streaming = walprop_pg_start_streaming, .get_flush_rec_ptr = walprop_pg_get_flush_rec_ptr, .get_current_timestamp = walprop_pg_get_current_timestamp, - .get_timeline_id = walprop_pg_get_timeline_id, .conn_error_message = walprop_error_message, .conn_status = walprop_status, .conn_connect_start = walprop_connect_start, @@ -1659,9 +1719,11 @@ static const walproposer_api walprop_pg = { .update_event_set = walprop_pg_update_event_set, .add_safekeeper_event_set = walprop_pg_add_safekeeper_event_set, .wait_event_set = walprop_pg_wait_event_set, - .strong_random = pg_strong_random, - .get_redo_start_lsn = GetRedoStartLsn, + .strong_random = walprop_pg_strong_random, + .get_redo_start_lsn = walprop_pg_get_redo_start_lsn, .finish_sync_safekeepers = walprop_pg_finish_sync_safekeepers, .process_safekeeper_feedback = walprop_pg_process_safekeeper_feedback, .confirm_wal_streamed = walprop_pg_confirm_wal_streamed, + .log_internal = walprop_pg_log_internal, + .after_election = walprop_pg_after_election, }; diff --git a/poetry.lock b/poetry.lock index efc13f7c87..421e22b523 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2415,13 +2415,13 @@ files = [ [[package]] name = "urllib3" -version = "1.26.17" +version = "1.26.18" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ - {file = "urllib3-1.26.17-py2.py3-none-any.whl", hash = "sha256:94a757d178c9be92ef5539b8840d48dc9cf1b2709c9d6b588232a055c524458b"}, - {file = "urllib3-1.26.17.tar.gz", hash = "sha256:24d6a242c28d29af46c3fae832c36db3bbebcc533dd1bb549172cd739c82df21"}, + {file = "urllib3-1.26.18-py2.py3-none-any.whl", hash = "sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07"}, + {file = "urllib3-1.26.18.tar.gz", hash = "sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0"}, ] [package.extras] @@ -2488,6 +2488,16 @@ files = [ {file = "wrapt-1.14.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:8ad85f7f4e20964db4daadcab70b47ab05c7c1cf2a7c1e51087bfaa83831854c"}, {file = "wrapt-1.14.1-cp310-cp310-win32.whl", hash = "sha256:a9a52172be0b5aae932bef82a79ec0a0ce87288c7d132946d645eba03f0ad8a8"}, {file = "wrapt-1.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:6d323e1554b3d22cfc03cd3243b5bb815a51f5249fdcbb86fda4bf62bab9e164"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ecee4132c6cd2ce5308e21672015ddfed1ff975ad0ac8d27168ea82e71413f55"}, + {file = "wrapt-1.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2020f391008ef874c6d9e208b24f28e31bcb85ccff4f335f15a3251d222b92d9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2feecf86e1f7a86517cab34ae6c2f081fd2d0dac860cb0c0ded96d799d20b335"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:240b1686f38ae665d1b15475966fe0472f78e71b1b4903c143a842659c8e4cb9"}, + {file = "wrapt-1.14.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9008dad07d71f68487c91e96579c8567c98ca4c3881b9b113bc7b33e9fd78b8"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:6447e9f3ba72f8e2b985a1da758767698efa72723d5b59accefd716e9e8272bf"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:acae32e13a4153809db37405f5eba5bac5fbe2e2ba61ab227926a22901051c0a"}, + {file = "wrapt-1.14.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:49ef582b7a1152ae2766557f0550a9fcbf7bbd76f43fbdc94dd3bf07cc7168be"}, + {file = "wrapt-1.14.1-cp311-cp311-win32.whl", hash = "sha256:358fe87cc899c6bb0ddc185bf3dbfa4ba646f05b1b0b9b5a27c2cb92c2cea204"}, + {file = "wrapt-1.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:26046cd03936ae745a502abf44dac702a5e6880b2b01c29aea8ddf3353b68224"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:43ca3bbbe97af00f49efb06e352eae40434ca9d915906f77def219b88e85d907"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:6b1a564e6cb69922c7fe3a678b9f9a3c54e72b469875aa8018f18b4d1dd1adf3"}, {file = "wrapt-1.14.1-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:00b6d4ea20a906c0ca56d84f93065b398ab74b927a7a3dbd470f6fc503f95dc3"}, diff --git a/proxy/src/auth/backend.rs b/proxy/src/auth/backend.rs index 03c9029862..aa19acf65a 100644 --- a/proxy/src/auth/backend.rs +++ b/proxy/src/auth/backend.rs @@ -6,6 +6,7 @@ pub use link::LinkAuthError; use crate::{ auth::{self, ClientCredentials}, + config::AuthenticationConfig, console::{ self, provider::{CachedNodeInfo, ConsoleReqExtra}, @@ -124,6 +125,7 @@ async fn auth_quirks( creds: &mut ClientCredentials<'_>, client: &mut stream::PqStream, allow_cleartext: bool, + config: &'static AuthenticationConfig, ) -> auth::Result> { // If there's no project so far, that entails that client doesn't // support SNI or other means of passing the endpoint (project) name. @@ -145,7 +147,7 @@ async fn auth_quirks( } // Finally, proceed with the main auth flow (SCRAM-based). - classic::authenticate(api, extra, creds, client).await + classic::authenticate(api, extra, creds, client, config).await } impl BackendType<'_, ClientCredentials<'_>> { @@ -180,6 +182,7 @@ impl BackendType<'_, ClientCredentials<'_>> { extra: &ConsoleReqExtra<'_>, client: &mut stream::PqStream, allow_cleartext: bool, + config: &'static AuthenticationConfig, ) -> auth::Result> { use BackendType::*; @@ -192,7 +195,7 @@ impl BackendType<'_, ClientCredentials<'_>> { ); let api = api.as_ref(); - auth_quirks(api, extra, creds, client, allow_cleartext).await? + auth_quirks(api, extra, creds, client, allow_cleartext, config).await? } Postgres(api, creds) => { info!( @@ -202,7 +205,7 @@ impl BackendType<'_, ClientCredentials<'_>> { ); let api = api.as_ref(); - auth_quirks(api, extra, creds, client, allow_cleartext).await? + auth_quirks(api, extra, creds, client, allow_cleartext, config).await? } // NOTE: this auth backend doesn't use client credentials. Link(url) => { diff --git a/proxy/src/auth/backend/classic.rs b/proxy/src/auth/backend/classic.rs index 9a056f1445..af628efc5e 100644 --- a/proxy/src/auth/backend/classic.rs +++ b/proxy/src/auth/backend/classic.rs @@ -4,6 +4,7 @@ use super::AuthSuccess; use crate::{ auth::{self, AuthFlow, ClientCredentials}, compute, + config::AuthenticationConfig, console::{self, AuthInfo, CachedNodeInfo, ConsoleReqExtra}, proxy::{handle_try_wake, retry_after}, sasl, scram, @@ -17,6 +18,7 @@ pub(super) async fn authenticate( extra: &ConsoleReqExtra<'_>, creds: &ClientCredentials<'_>, client: &mut PqStream, + config: &'static AuthenticationConfig, ) -> auth::Result> { info!("fetching user's authentication info"); let info = api.get_auth_info(extra, creds).await?.unwrap_or_else(|| { @@ -42,7 +44,16 @@ pub(super) async fn authenticate( error })?; - let auth_outcome = auth_flow.authenticate().await.map_err(|error| { + let auth_outcome = tokio::time::timeout( + config.scram_protocol_timeout, + auth_flow.authenticate(), + ) + .await + .map_err(|error| { + warn!("error processing scram messages error = authentication timed out, execution time exeeded {} seconds", config.scram_protocol_timeout.as_secs()); + auth::io::Error::new(auth::io::ErrorKind::TimedOut, error) + })? + .map_err(|error| { warn!(?error, "error processing scram messages"); error })?; diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 6bccdf6e5a..9b54f98402 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -1,5 +1,6 @@ use futures::future::Either; use proxy::auth; +use proxy::config::AuthenticationConfig; use proxy::config::HttpConfig; use proxy::console; use proxy::http; @@ -83,6 +84,12 @@ struct ProxyCliArgs { /// timeout for http connections #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] sql_over_http_timeout: tokio::time::Duration, + /// timeout for scram authentication protocol + #[clap(long, default_value = "15s", value_parser = humantime::parse_duration)] + scram_protocol_timeout: tokio::time::Duration, + /// Require that all incoming requests have a Proxy Protocol V2 packet **and** have an IP address associated. + #[clap(long, default_value_t = false, value_parser = clap::builder::BoolishValueParser::new(), action = clap::ArgAction::Set)] + require_client_ip: bool, } #[tokio::main] @@ -227,12 +234,17 @@ fn build_config(args: &ProxyCliArgs) -> anyhow::Result<&'static ProxyConfig> { let http_config = HttpConfig { sql_over_http_timeout: args.sql_over_http_timeout, }; + let authentication_config = AuthenticationConfig { + scram_protocol_timeout: args.scram_protocol_timeout, + }; let config = Box::leak(Box::new(ProxyConfig { tls_config, auth_backend, metric_collection, allow_self_signed_compute: args.allow_self_signed_compute, http_config, + authentication_config, + require_client_ip: args.require_client_ip, })); Ok(config) diff --git a/proxy/src/cancellation.rs b/proxy/src/cancellation.rs index 8d16f202e9..a5eb3544b4 100644 --- a/proxy/src/cancellation.rs +++ b/proxy/src/cancellation.rs @@ -1,5 +1,5 @@ -use anyhow::{anyhow, Context}; -use hashbrown::HashMap; +use anyhow::{bail, Context}; +use dashmap::DashMap; use pq_proto::CancelKeyData; use std::net::SocketAddr; use tokio::net::TcpStream; @@ -8,7 +8,7 @@ use tracing::info; /// Enables serving `CancelRequest`s. #[derive(Default)] -pub struct CancelMap(parking_lot::RwLock>>); +pub struct CancelMap(DashMap>); impl CancelMap { /// Cancel a running query for the corresponding connection. @@ -16,7 +16,6 @@ impl CancelMap { // NB: we should immediately release the lock after cloning the token. let cancel_closure = self .0 - .read() .get(&key) .and_then(|x| x.clone()) .with_context(|| format!("query cancellation key not found: {key}"))?; @@ -40,15 +39,19 @@ impl CancelMap { // Random key collisions are unlikely to happen here, but they're still possible, // which is why we have to take care not to rewrite an existing key. - self.0 - .write() - .try_insert(key, None) - .map_err(|_| anyhow!("query cancellation key already exists: {key}"))?; + match self.0.entry(key) { + dashmap::mapref::entry::Entry::Occupied(_) => { + bail!("query cancellation key already exists: {key}") + } + dashmap::mapref::entry::Entry::Vacant(e) => { + e.insert(None); + } + } // This will guarantee that the session gets dropped // as soon as the future is finished. scopeguard::defer! { - self.0.write().remove(&key); + self.0.remove(&key); info!("dropped query cancellation key {key}"); } @@ -59,12 +62,12 @@ impl CancelMap { #[cfg(test)] fn contains(&self, session: &Session) -> bool { - self.0.read().contains_key(&session.key) + self.0.contains_key(&session.key) } #[cfg(test)] fn is_empty(&self) -> bool { - self.0.read().is_empty() + self.0.is_empty() } } @@ -113,10 +116,7 @@ impl Session<'_> { /// This enables query cancellation in `crate::proxy::prepare_client_connection`. pub fn enable_query_cancellation(self, cancel_closure: CancelClosure) -> CancelKeyData { info!("enabling query cancellation for this session"); - self.cancel_map - .0 - .write() - .insert(self.key, Some(cancel_closure)); + self.cancel_map.0.insert(self.key, Some(cancel_closure)); self.key } diff --git a/proxy/src/config.rs b/proxy/src/config.rs index b37c1736bd..96856ba051 100644 --- a/proxy/src/config.rs +++ b/proxy/src/config.rs @@ -14,6 +14,8 @@ pub struct ProxyConfig { pub metric_collection: Option, pub allow_self_signed_compute: bool, pub http_config: HttpConfig, + pub authentication_config: AuthenticationConfig, + pub require_client_ip: bool, } #[derive(Debug)] @@ -31,6 +33,10 @@ pub struct HttpConfig { pub sql_over_http_timeout: tokio::time::Duration, } +pub struct AuthenticationConfig { + pub scram_protocol_timeout: tokio::time::Duration, +} + impl TlsConfig { pub fn to_server_config(&self) -> Arc { self.config.clone() diff --git a/proxy/src/http/conn_pool.rs b/proxy/src/http/conn_pool.rs index b268c4073e..7a5f00d627 100644 --- a/proxy/src/http/conn_pool.rs +++ b/proxy/src/http/conn_pool.rs @@ -8,14 +8,17 @@ use pbkdf2::{ Params, Pbkdf2, }; use pq_proto::StartupMessageParams; -use std::sync::atomic::{self, AtomicUsize}; use std::{collections::HashMap, sync::Arc}; use std::{ fmt, task::{ready, Poll}, }; +use std::{ + ops::Deref, + sync::atomic::{self, AtomicUsize}, +}; use tokio::time; -use tokio_postgres::AsyncMessage; +use tokio_postgres::{AsyncMessage, ReadyForQueryStatus}; use crate::{ auth, console, @@ -26,13 +29,13 @@ use crate::{compute, config}; use crate::proxy::ConnectMechanism; -use tracing::{error, warn}; +use tracing::{error, warn, Span}; use tracing::{info, info_span, Instrument}; pub const APP_NAME: &str = "sql_over_http"; const MAX_CONNS_PER_ENDPOINT: usize = 20; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ConnInfo { pub username: String, pub dbname: String, @@ -55,7 +58,7 @@ impl fmt::Display for ConnInfo { } struct ConnPoolEntry { - conn: Client, + conn: ClientInner, _last_access: std::time::Instant, } @@ -133,14 +136,20 @@ impl GlobalConnPool { } pub async fn get( - &self, + self: &Arc, conn_info: &ConnInfo, force_new: bool, session_id: uuid::Uuid, ) -> anyhow::Result { - let mut client: Option = None; + let mut client: Option = None; let mut latency_timer = LatencyTimer::new("http"); + let pool = if force_new { + None + } else { + Some((conn_info.clone(), self.clone())) + }; + let mut hash_valid = false; if !force_new { let pool = self.get_or_create_endpoint_pool(&conn_info.hostname); @@ -185,10 +194,15 @@ impl GlobalConnPool { info!("pool: cached connection '{conn_info}' is closed, opening a new one"); connect_to_compute(self.proxy_config, conn_info, session_id, latency_timer).await } else { - latency_timer.pool_hit(); info!("pool: reusing connection '{conn_info}'"); client.session.send(session_id)?; - return Ok(client); + latency_timer.pool_hit(); + latency_timer.success(); + return Ok(Client { + inner: Some(client), + span: Span::current(), + pool, + }); } } else { info!("pool: opening a new connection '{conn_info}'"); @@ -228,10 +242,14 @@ impl GlobalConnPool { _ => {} } - new_client + new_client.map(|inner| Client { + inner: Some(inner), + span: Span::current(), + pool, + }) } - pub fn put(&self, conn_info: &ConnInfo, client: Client) -> anyhow::Result<()> { + fn put(&self, conn_info: &ConnInfo, client: ClientInner) -> anyhow::Result<()> { // We want to hold this open while we return. This ensures that the pool can't close // while we are in the middle of returning the connection. let closed = self.closed.read(); @@ -326,7 +344,7 @@ struct TokioMechanism<'a> { #[async_trait] impl ConnectMechanism for TokioMechanism<'_> { - type Connection = Client; + type Connection = ClientInner; type ConnectError = tokio_postgres::Error; type Error = anyhow::Error; @@ -350,7 +368,7 @@ async fn connect_to_compute( conn_info: &ConnInfo, session_id: uuid::Uuid, latency_timer: LatencyTimer, -) -> anyhow::Result { +) -> anyhow::Result { let tls = config.tls_config.as_ref(); let common_names = tls.and_then(|tls| tls.common_names.clone()); @@ -399,7 +417,7 @@ async fn connect_to_compute_once( conn_info: &ConnInfo, timeout: time::Duration, mut session: uuid::Uuid, -) -> Result { +) -> Result { let mut config = (*node_info.config).clone(); let (client, mut connection) = config @@ -462,21 +480,99 @@ async fn connect_to_compute_once( .instrument(span) ); - Ok(Client { + Ok(ClientInner { inner: client, session: tx, ids, }) } -pub struct Client { - pub inner: tokio_postgres::Client, +struct ClientInner { + inner: tokio_postgres::Client, session: tokio::sync::watch::Sender, ids: Ids, } impl Client { pub fn metrics(&self) -> Arc { - USAGE_METRICS.register(self.ids.clone()) + USAGE_METRICS.register(self.inner.as_ref().unwrap().ids.clone()) + } +} + +pub struct Client { + span: Span, + inner: Option, + pool: Option<(ConnInfo, Arc)>, +} + +pub struct Discard<'a> { + pool: &'a mut Option<(ConnInfo, Arc)>, +} + +impl Client { + pub fn inner(&mut self) -> (&mut tokio_postgres::Client, Discard<'_>) { + let Self { + inner, + pool, + span: _, + } = self; + ( + &mut inner + .as_mut() + .expect("client inner should not be removed") + .inner, + Discard { pool }, + ) + } + + pub fn check_idle(&mut self, status: ReadyForQueryStatus) { + self.inner().1.check_idle(status) + } + pub fn discard(&mut self) { + self.inner().1.discard() + } +} + +impl Discard<'_> { + pub fn check_idle(&mut self, status: ReadyForQueryStatus) { + if status != ReadyForQueryStatus::Idle { + if let Some((conn_info, _)) = self.pool.take() { + info!("pool: throwing away connection '{conn_info}' because connection is not idle") + } + } + } + pub fn discard(&mut self) { + if let Some((conn_info, _)) = self.pool.take() { + info!("pool: throwing away connection '{conn_info}' because connection is potentially in a broken state") + } + } +} + +impl Deref for Client { + type Target = tokio_postgres::Client; + + fn deref(&self) -> &Self::Target { + &self + .inner + .as_ref() + .expect("client inner should not be removed") + .inner + } +} + +impl Drop for Client { + fn drop(&mut self) { + let client = self + .inner + .take() + .expect("client inner should not be removed"); + if let Some((conn_info, conn_pool)) = self.pool.take() { + let current_span = self.span.clone(); + // return connection to the pool + tokio::task::spawn_blocking(move || { + let _span = current_span.enter(); + let _ = conn_pool.put(&conn_info, client); + }); + } } } diff --git a/proxy/src/http/sql_over_http.rs b/proxy/src/http/sql_over_http.rs index 02ce3aa9dc..93df86cfc4 100644 --- a/proxy/src/http/sql_over_http.rs +++ b/proxy/src/http/sql_over_http.rs @@ -17,7 +17,9 @@ use tokio_postgres::types::Kind; use tokio_postgres::types::Type; use tokio_postgres::GenericClient; use tokio_postgres::IsolationLevel; +use tokio_postgres::ReadyForQueryStatus; use tokio_postgres::Row; +use tokio_postgres::Transaction; use tracing::error; use tracing::instrument; use url::Url; @@ -64,20 +66,18 @@ static HEADER_VALUE_TRUE: HeaderValue = HeaderValue::from_static("true"); // Convert json non-string types to strings, so that they can be passed to Postgres // as parameters. // -fn json_to_pg_text(json: Vec) -> Result>, serde_json::Error> { +fn json_to_pg_text(json: Vec) -> Vec> { json.iter() .map(|value| { match value { // special care for nulls - Value::Null => Ok(None), + Value::Null => None, // convert to text with escaping - Value::Bool(_) => serde_json::to_string(value).map(Some), - Value::Number(_) => serde_json::to_string(value).map(Some), - Value::Object(_) => serde_json::to_string(value).map(Some), + v @ (Value::Bool(_) | Value::Number(_) | Value::Object(_)) => Some(v.to_string()), // avoid escaping here, as we pass this as a parameter - Value::String(s) => Ok(Some(s.to_string())), + Value::String(s) => Some(s.to_string()), // special care for arrays Value::Array(_) => json_array_to_pg_array(value), @@ -94,29 +94,26 @@ fn json_to_pg_text(json: Vec) -> Result>, serde_json:: // // Example of the same escaping in node-postgres: packages/pg/lib/utils.js // -fn json_array_to_pg_array(value: &Value) -> Result, serde_json::Error> { +fn json_array_to_pg_array(value: &Value) -> Option { match value { // special care for nulls - Value::Null => Ok(None), + Value::Null => None, // convert to text with escaping - Value::Bool(_) => serde_json::to_string(value).map(Some), - Value::Number(_) => serde_json::to_string(value).map(Some), - // here string needs to be escaped, as it is part of the array - Value::Object(_) => json_array_to_pg_array(&Value::String(serde_json::to_string(value)?)), - Value::String(_) => serde_json::to_string(value).map(Some), + v @ (Value::Bool(_) | Value::Number(_) | Value::String(_)) => Some(v.to_string()), + v @ Value::Object(_) => json_array_to_pg_array(&Value::String(v.to_string())), // recurse into array Value::Array(arr) => { let vals = arr .iter() .map(json_array_to_pg_array) - .map(|r| r.map(|v| v.unwrap_or_else(|| "NULL".to_string()))) - .collect::, _>>()? + .map(|v| v.unwrap_or_else(|| "NULL".to_string())) + .collect::>() .join(","); - Ok(Some(format!("{{{}}}", vals))) + Some(format!("{{{}}}", vals)) } } } @@ -315,83 +312,119 @@ async fn handle_inner( // Now execute the query and return the result // let mut size = 0; - let result = match payload { - Payload::Single(query) => { - query_to_json(&client.inner, query, &mut size, raw_output, array_mode).await - } - Payload::Batch(batch_query) => { - let mut results = Vec::new(); - let mut builder = client.inner.build_transaction(); - if let Some(isolation_level) = txn_isolation_level { - builder = builder.isolation_level(isolation_level); + let result = + match payload { + Payload::Single(stmt) => { + let (status, results) = + query_to_json(&*client, stmt, &mut 0, raw_output, array_mode) + .await + .map_err(|e| { + client.discard(); + e + })?; + client.check_idle(status); + results } - if txn_read_only { - builder = builder.read_only(true); - } - if txn_deferrable { - builder = builder.deferrable(true); - } - let transaction = builder.start().await?; - for query in batch_query.queries { - let result = - query_to_json(&transaction, query, &mut size, raw_output, array_mode).await; - match result { - Ok(r) => results.push(r), - Err(e) => { - transaction.rollback().await?; - return Err(e); - } + Payload::Batch(statements) => { + let (inner, mut discard) = client.inner(); + let mut builder = inner.build_transaction(); + if let Some(isolation_level) = txn_isolation_level { + builder = builder.isolation_level(isolation_level); } + if txn_read_only { + builder = builder.read_only(true); + } + if txn_deferrable { + builder = builder.deferrable(true); + } + + let transaction = builder.start().await.map_err(|e| { + // if we cannot start a transaction, we should return immediately + // and not return to the pool. connection is clearly broken + discard.discard(); + e + })?; + + let results = + match query_batch(&transaction, statements, &mut size, raw_output, array_mode) + .await + { + Ok(results) => { + let status = transaction.commit().await.map_err(|e| { + // if we cannot commit - for now don't return connection to pool + // TODO: get a query status from the error + discard.discard(); + e + })?; + discard.check_idle(status); + results + } + Err(err) => { + let status = transaction.rollback().await.map_err(|e| { + // if we cannot rollback - for now don't return connection to pool + // TODO: get a query status from the error + discard.discard(); + e + })?; + discard.check_idle(status); + return Err(err); + } + }; + + if txn_read_only { + response = response.header( + TXN_READ_ONLY.clone(), + HeaderValue::try_from(txn_read_only.to_string())?, + ); + } + if txn_deferrable { + response = response.header( + TXN_DEFERRABLE.clone(), + HeaderValue::try_from(txn_deferrable.to_string())?, + ); + } + if let Some(txn_isolation_level) = txn_isolation_level_raw { + response = response.header(TXN_ISOLATION_LEVEL.clone(), txn_isolation_level); + } + json!({ "results": results }) } - transaction.commit().await?; - if txn_read_only { - response = response.header( - TXN_READ_ONLY.clone(), - HeaderValue::try_from(txn_read_only.to_string())?, - ); - } - if txn_deferrable { - response = response.header( - TXN_DEFERRABLE.clone(), - HeaderValue::try_from(txn_deferrable.to_string())?, - ); - } - if let Some(txn_isolation_level) = txn_isolation_level_raw { - response = response.header(TXN_ISOLATION_LEVEL.clone(), txn_isolation_level); - } - Ok(json!({ "results": results })) - } - }; + }; let metrics = client.metrics(); - if allow_pool { - let current_span = tracing::Span::current(); - // return connection to the pool - tokio::task::spawn_blocking(move || { - let _span = current_span.enter(); - let _ = conn_pool.put(&conn_info, client); - }); - } + // how could this possibly fail + let body = serde_json::to_string(&result).expect("json serialization should not fail"); + let len = body.len(); + let response = response + .body(Body::from(body)) + // only fails if invalid status code or invalid header/values are given. + // these are not user configurable so it cannot fail dynamically + .expect("building response payload should not fail"); - match result { - Ok(value) => { - // how could this possibly fail - let body = serde_json::to_string(&value).expect("json serialization should not fail"); - let len = body.len(); - let response = response - .body(Body::from(body)) - // only fails if invalid status code or invalid header/values are given. - // these are not user configurable so it cannot fail dynamically - .expect("building response payload should not fail"); + // count the egress bytes - we miss the TLS and header overhead but oh well... + // moving this later in the stack is going to be a lot of effort and ehhhh + metrics.record_egress(len as u64); - // count the egress bytes - we miss the TLS and header overhead but oh well... - // moving this later in the stack is going to be a lot of effort and ehhhh - metrics.record_egress(len as u64); - Ok(response) - } - Err(e) => Err(e), + Ok(response) +} + +async fn query_batch( + transaction: &Transaction<'_>, + queries: BatchQueryData, + total_size: &mut usize, + raw_output: bool, + array_mode: bool, +) -> anyhow::Result> { + let mut results = Vec::with_capacity(queries.queries.len()); + let mut current_size = 0; + for stmt in queries.queries { + // TODO: maybe we should check that the transaction bit is set here + let (_, values) = + query_to_json(transaction, stmt, &mut current_size, raw_output, array_mode).await?; + results.push(values); } + *total_size += current_size; + Ok(results) } async fn query_to_json( @@ -400,11 +433,9 @@ async fn query_to_json( current_size: &mut usize, raw_output: bool, array_mode: bool, -) -> anyhow::Result { - let query_params = json_to_pg_text(data.params)?; - let row_stream = client - .query_raw_txt::(data.query, query_params) - .await?; +) -> anyhow::Result<(ReadyForQueryStatus, Value)> { + let query_params = json_to_pg_text(data.params); + let row_stream = client.query_raw_txt(&data.query, query_params).await?; // Manually drain the stream into a vector to leave row_stream hanging // around to get a command tag. Also check that the response is not too @@ -424,6 +455,8 @@ async fn query_to_json( } } + let ready = row_stream.ready_status(); + // grab the command tag and number of rows affected let command_tag = row_stream.command_tag().unwrap_or_default(); let mut command_tag_split = command_tag.split(' '); @@ -464,13 +497,16 @@ async fn query_to_json( .collect::, _>>()?; // resulting JSON format is based on the format of node-postgres result - Ok(json!({ - "command": command_tag_name, - "rowCount": command_tag_count, - "rows": rows, - "fields": fields, - "rowAsArray": array_mode, - })) + Ok(( + ready, + json!({ + "command": command_tag_name, + "rowCount": command_tag_count, + "rows": rows, + "fields": fields, + "rowAsArray": array_mode, + }), + )) } // @@ -655,22 +691,22 @@ mod tests { #[test] fn test_atomic_types_to_pg_params() { let json = vec![Value::Bool(true), Value::Bool(false)]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!( pg_params, vec![Some("true".to_owned()), Some("false".to_owned())] ); let json = vec![Value::Number(serde_json::Number::from(42))]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!(pg_params, vec![Some("42".to_owned())]); let json = vec![Value::String("foo\"".to_string())]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!(pg_params, vec![Some("foo\"".to_owned())]); let json = vec![Value::Null]; - let pg_params = json_to_pg_text(json).unwrap(); + let pg_params = json_to_pg_text(json); assert_eq!(pg_params, vec![None]); } @@ -679,7 +715,7 @@ mod tests { // atoms and escaping let json = "[true, false, null, \"NULL\", 42, \"foo\", \"bar\\\"-\\\\\"]"; let json: Value = serde_json::from_str(json).unwrap(); - let pg_params = json_to_pg_text(vec![json]).unwrap(); + let pg_params = json_to_pg_text(vec![json]); assert_eq!( pg_params, vec![Some( @@ -690,7 +726,7 @@ mod tests { // nested arrays let json = "[[true, false], [null, 42], [\"foo\", \"bar\\\"-\\\\\"]]"; let json: Value = serde_json::from_str(json).unwrap(); - let pg_params = json_to_pg_text(vec![json]).unwrap(); + let pg_params = json_to_pg_text(vec![json]); assert_eq!( pg_params, vec![Some( @@ -700,7 +736,7 @@ mod tests { // array of objects let json = r#"[{"foo": 1},{"bar": 2}]"#; let json: Value = serde_json::from_str(json).unwrap(); - let pg_params = json_to_pg_text(vec![json]).unwrap(); + let pg_params = json_to_pg_text(vec![json]); assert_eq!( pg_params, vec![Some(r#"{"{\"foo\":1}","{\"bar\":2}"}"#.to_owned())] diff --git a/proxy/src/http/websocket.rs b/proxy/src/http/websocket.rs index 656635b299..689a84969c 100644 --- a/proxy/src/http/websocket.rs +++ b/proxy/src/http/websocket.rs @@ -8,6 +8,7 @@ use crate::{ NUM_CLIENT_CONNECTION_OPENED_COUNTER, }, }; +use anyhow::bail; use bytes::{Buf, Bytes}; use futures::{Sink, Stream, StreamExt}; use hyper::{ @@ -22,7 +23,6 @@ use hyper_tungstenite::{tungstenite::Message, HyperWebsocket, WebSocketStream}; use pin_project_lite::pin_project; use std::{ - convert::Infallible, future::ready, pin::Pin, sync::Arc, @@ -280,12 +280,18 @@ pub async fn task_main( let make_svc = hyper::service::make_service_fn( |stream: &tokio_rustls::server::TlsStream>| { let (io, tls) = stream.get_ref(); - let peer_addr = io.client_addr().unwrap_or(io.inner.remote_addr()); + let client_addr = io.client_addr(); + let remote_addr = io.inner.remote_addr(); let sni_name = tls.server_name().map(|s| s.to_string()); let conn_pool = conn_pool.clone(); async move { - Ok::<_, Infallible>(MetricService::new(hyper::service::service_fn( + let peer_addr = match client_addr { + Some(addr) => addr, + None if config.require_client_ip => bail!("missing required client ip"), + None => remote_addr, + }; + Ok(MetricService::new(hyper::service::service_fn( move |req: Request| { let sni_name = sni_name.clone(); let conn_pool = conn_pool.clone(); diff --git a/proxy/src/proxy.rs b/proxy/src/proxy.rs index 6a928547a9..ec878b696e 100644 --- a/proxy/src/proxy.rs +++ b/proxy/src/proxy.rs @@ -5,7 +5,7 @@ use crate::{ auth::{self, backend::AuthSuccess}, cancellation::{self, CancelMap}, compute::{self, PostgresConnection}, - config::{ProxyConfig, TlsConfig}, + config::{AuthenticationConfig, ProxyConfig, TlsConfig}, console::{self, errors::WakeComputeError, messages::MetricsAuxInfo, Api}, http::StatusCode, metrics::{Ids, USAGE_METRICS}, @@ -96,7 +96,9 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { register_histogram_vec!( "proxy_compute_connection_latency_seconds", "Time it took for proxy to establish a connection to the compute endpoint", - &["protocol", "cache_miss", "pool_miss"], + // http/ws/tcp, true/false, true/false, success/failure + // 3 * 2 * 2 * 2 = 24 counters + &["protocol", "cache_miss", "pool_miss", "outcome"], // largest bucket = 2^16 * 0.5ms = 32s exponential_buckets(0.0005, 2.0, 16).unwrap(), ) @@ -105,19 +107,22 @@ static COMPUTE_CONNECTION_LATENCY: Lazy = Lazy::new(|| { pub struct LatencyTimer { start: Instant, - pool_miss: bool, - cache_miss: bool, protocol: &'static str, + cache_miss: bool, + pool_miss: bool, + outcome: &'static str, } impl LatencyTimer { pub fn new(protocol: &'static str) -> Self { Self { start: Instant::now(), + protocol, cache_miss: false, // by default we don't do pooling pool_miss: true, - protocol, + // assume failed unless otherwise specified + outcome: "failed", } } @@ -128,6 +133,10 @@ impl LatencyTimer { pub fn pool_hit(&mut self) { self.pool_miss = false; } + + pub fn success(mut self) { + self.outcome = "success"; + } } impl Drop for LatencyTimer { @@ -138,6 +147,7 @@ impl Drop for LatencyTimer { self.protocol, bool_to_str(self.cache_miss), bool_to_str(self.pool_miss), + self.outcome, ]) .observe(duration) } @@ -200,6 +210,8 @@ pub async fn task_main( let mut socket = WithClientIp::new(socket); if let Some(ip) = socket.wait_for_addr().await? { tracing::Span::current().record("peer_addr", &tracing::field::display(ip)); + } else if config.require_client_ip { + bail!("missing required client IP"); } socket @@ -338,7 +350,7 @@ pub async fn handle_client( mode.allow_self_signed_compute(config), ); cancel_map - .with_session(|session| client.connect_to_db(session, mode)) + .with_session(|session| client.connect_to_db(session, mode, &config.authentication_config)) .await } @@ -545,7 +557,10 @@ where // try once let (config, err) = match mechanism.connect_once(&node_info, CONNECT_TIMEOUT).await { - Ok(res) => return Ok(res), + Ok(res) => { + latency_timer.success(); + return Ok(res); + } Err(e) => { error!(error = ?e, "could not connect to compute node"); (invalidate_cache(node_info), e) @@ -599,7 +614,10 @@ where info!("wake_compute success. attempting to connect"); loop { match mechanism.connect_once(&node_info, CONNECT_TIMEOUT).await { - Ok(res) => return Ok(res), + Ok(res) => { + latency_timer.success(); + return Ok(res); + } Err(e) => { let retriable = e.should_retry(num_retries); if !retriable { @@ -816,6 +834,7 @@ impl Client<'_, S> { self, session: cancellation::Session<'_>, mode: ClientMode, + config: &'static AuthenticationConfig, ) -> anyhow::Result<()> { let Self { mut stream, @@ -833,7 +852,7 @@ impl Client<'_, S> { let latency_timer = LatencyTimer::new(mode.protocol_label()); let auth_result = match creds - .authenticate(&extra, &mut stream, mode.allow_cleartext()) + .authenticate(&extra, &mut stream, mode.allow_cleartext(), config) .await { Ok(auth_result) => auth_result, diff --git a/proxy/src/sasl/messages.rs b/proxy/src/sasl/messages.rs index fb3833c8b6..b9208f6f1f 100644 --- a/proxy/src/sasl/messages.rs +++ b/proxy/src/sasl/messages.rs @@ -31,7 +31,7 @@ impl<'a> FirstMessage<'a> { /// A single SASL message. /// This struct is deliberately decoupled from lower-level -/// [`BeAuthenticationSaslMessage`](pq_proto::BeAuthenticationSaslMessage). +/// [`BeAuthenticationSaslMessage`]. #[derive(Debug)] pub(super) enum ServerMessage { /// We expect to see more steps. diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 9cc47ec039..ca12f0dee5 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "1.72.1" +channel = "1.73.0" profile = "default" # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy. # https://rust-lang.github.io/rustup/concepts/profiles.html diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 763dcd9eb8..202343fca1 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -3,7 +3,7 @@ // use anyhow::{bail, Context, Result}; use camino::{Utf8Path, Utf8PathBuf}; -use clap::Parser; +use clap::{ArgAction, Parser}; use futures::future::BoxFuture; use futures::stream::FuturesUnordered; use futures::{FutureExt, StreamExt}; @@ -105,6 +105,9 @@ struct Args { /// it during this period passed as a human readable duration. #[arg(long, value_parser= humantime::parse_duration, default_value = DEFAULT_HEARTBEAT_TIMEOUT, verbatim_doc_comment)] heartbeat_timeout: Duration, + /// Enable/disable peer recovery. + #[arg(long, default_value = "false", action=ArgAction::Set)] + peer_recovery: bool, /// Remote storage configuration for WAL backup (offloading to s3) as TOML /// inline table, e.g. /// {"max_concurrent_syncs" = 17, "max_sync_errors": 13, "bucket_name": "", "bucket_region":"", "concurrency_limit": 119} @@ -265,6 +268,7 @@ async fn main() -> anyhow::Result<()> { broker_endpoint: args.broker_endpoint, broker_keepalive_interval: args.broker_keepalive_interval, heartbeat_timeout: args.heartbeat_timeout, + peer_recovery_enabled: args.peer_recovery, remote_storage: args.remote_storage, max_offloader_lag_bytes: args.max_offloader_lag, wal_backup_enabled: !args.disable_wal_backup, diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index 134331c673..71b99ab1d8 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -372,6 +372,13 @@ impl SafekeeperPostgresHandler { /// from a walproposer recovery function. This connection gets a special handling: /// safekeeper must stream all local WAL till the flush_lsn, whether committed or not. pub fn is_walproposer_recovery(&self) -> bool { - self.appname == Some("wal_proposer_recovery".to_string()) + match &self.appname { + None => false, + Some(appname) => { + appname == "wal_proposer_recovery" || + // set by safekeeper peer recovery + appname.starts_with("safekeeper") + } + } } } diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index f4c3a4aa03..940ac82df6 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -16,8 +16,8 @@ use tokio::io::AsyncReadExt; use utils::http::endpoint::request_span; use crate::receive_wal::WalReceiverState; -use crate::safekeeper::ServerInfo; use crate::safekeeper::Term; +use crate::safekeeper::{ServerInfo, TermLsn}; use crate::send_wal::WalSenderState; use crate::timeline::PeerInfo; use crate::{debug_dump, pull_timeline}; @@ -60,16 +60,25 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { .as_ref() } -/// Same as TermSwitchEntry, but serializes LSN using display serializer +/// Same as TermLsn, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. #[serde_as] -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] pub struct TermSwitchApiEntry { pub term: Term, #[serde_as(as = "DisplayFromStr")] pub lsn: Lsn, } +impl From for TermLsn { + fn from(api_val: TermSwitchApiEntry) -> Self { + TermLsn { + term: api_val.term, + lsn: api_val.lsn, + } + } +} + /// Augment AcceptorState with epoch for convenience #[derive(Debug, Serialize, Deserialize)] pub struct AcceptorStateStatus { diff --git a/safekeeper/src/lib.rs b/safekeeper/src/lib.rs index 00aa405047..aa70cee591 100644 --- a/safekeeper/src/lib.rs +++ b/safekeeper/src/lib.rs @@ -62,6 +62,7 @@ pub struct SafeKeeperConf { pub broker_endpoint: Uri, pub broker_keepalive_interval: Duration, pub heartbeat_timeout: Duration, + pub peer_recovery_enabled: bool, pub remote_storage: Option, pub max_offloader_lag_bytes: u64, pub backup_parallel_jobs: usize, @@ -100,6 +101,7 @@ impl SafeKeeperConf { .parse() .expect("failed to parse default broker endpoint"), broker_keepalive_interval: Duration::from_secs(5), + peer_recovery_enabled: true, wal_backup_enabled: true, backup_parallel_jobs: 1, pg_auth: None, diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index cf63eace21..934dda10b2 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -55,9 +55,12 @@ impl WalReceivers { /// Register new walreceiver. Returned guard provides access to the slot and /// automatically deregisters in Drop. - pub fn register(self: &Arc) -> WalReceiverGuard { + pub fn register(self: &Arc, conn_id: Option) -> WalReceiverGuard { let slots = &mut self.mutex.lock().slots; - let walreceiver = WalReceiverState::Voting; + let walreceiver = WalReceiverState { + conn_id, + status: WalReceiverStatus::Voting, + }; // find empty slot or create new one let pos = if let Some(pos) = slots.iter().position(|s| s.is_none()) { slots[pos] = Some(walreceiver); @@ -96,6 +99,18 @@ impl WalReceivers { self.mutex.lock().slots.iter().flatten().cloned().collect() } + /// Get number of streaming walreceivers (normally 0 or 1) from compute. + pub fn get_num_streaming(self: &Arc) -> usize { + self.mutex + .lock() + .slots + .iter() + .flatten() + // conn_id.is_none skips recovery which also registers here + .filter(|s| s.conn_id.is_some() && matches!(s.status, WalReceiverStatus::Streaming)) + .count() + } + /// Unregister walsender. fn unregister(self: &Arc, id: WalReceiverId) { let mut shared = self.mutex.lock(); @@ -108,10 +123,17 @@ struct WalReceiversShared { slots: Vec>, } +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct WalReceiverState { + /// None means it is recovery initiated by us (this safekeeper). + pub conn_id: Option, + pub status: WalReceiverStatus, +} + /// Walreceiver status. Currently only whether it passed voting stage and /// started receiving the stream, but it is easy to add more if needed. #[derive(Debug, Clone, Serialize, Deserialize)] -pub enum WalReceiverState { +pub enum WalReceiverStatus { Voting, Streaming, } @@ -136,8 +158,8 @@ impl Drop for WalReceiverGuard { } } -const MSG_QUEUE_SIZE: usize = 256; -const REPLY_QUEUE_SIZE: usize = 16; +pub const MSG_QUEUE_SIZE: usize = 256; +pub const REPLY_QUEUE_SIZE: usize = 16; impl SafekeeperPostgresHandler { /// Wrapper around handle_start_wal_push_guts handling result. Error is @@ -261,7 +283,7 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { tli.clone(), msg_rx, reply_tx, - self.conn_id, + Some(self.conn_id), )); // Forward all messages to WalAcceptor @@ -317,31 +339,41 @@ async fn network_write( // even when it writes a steady stream of messages. const KEEPALIVE_INTERVAL: Duration = Duration::from_secs(1); -/// Takes messages from msg_rx, processes and pushes replies to reply_tx. -struct WalAcceptor { +/// Encapsulates a task which takes messages from msg_rx, processes and pushes +/// replies to reply_tx; reading from socket and writing to disk in parallel is +/// beneficial for performance, this struct provides writing to disk part. +pub struct WalAcceptor { tli: Arc, msg_rx: Receiver, reply_tx: Sender, + conn_id: Option, } impl WalAcceptor { - /// Spawn thread with WalAcceptor running, return handle to it. - fn spawn( + /// Spawn task with WalAcceptor running, return handle to it. Task returns + /// Ok(()) if either of channels has closed, and Err if any error during + /// message processing is encountered. + /// + /// conn_id None means WalAcceptor is used by recovery initiated at this safekeeper. + pub fn spawn( tli: Arc, msg_rx: Receiver, reply_tx: Sender, - conn_id: ConnectionId, + conn_id: Option, ) -> JoinHandle> { task::spawn(async move { let mut wa = WalAcceptor { tli, msg_rx, reply_tx, + conn_id, }; let span_ttid = wa.tli.ttid; // satisfy borrow checker wa.run() - .instrument(info_span!("WAL acceptor", cid = %conn_id, ttid = %span_ttid)) + .instrument( + info_span!("WAL acceptor", cid = %conn_id.unwrap_or(0), ttid = %span_ttid), + ) .await }) } @@ -355,7 +387,7 @@ impl WalAcceptor { let _compute_conn_guard = ComputeConnectionGuard { timeline: Arc::clone(&self.tli), }; - let walreceiver_guard = self.tli.get_walreceivers().register(); + let walreceiver_guard = self.tli.get_walreceivers().register(self.conn_id); self.tli.update_status_notify().await?; // After this timestamp we will stop processing AppendRequests and send a response @@ -372,7 +404,7 @@ impl WalAcceptor { // Update walreceiver state in shmem for reporting. if let ProposerAcceptorMessage::Elected(_) = &next_msg { - *walreceiver_guard.get() = WalReceiverState::Streaming; + walreceiver_guard.get().status = WalReceiverStatus::Streaming; } let reply_msg = if matches!(next_msg, ProposerAcceptorMessage::AppendRequest(_)) { diff --git a/safekeeper/src/recovery.rs b/safekeeper/src/recovery.rs index 90ba3d2e16..e8fa6c55f4 100644 --- a/safekeeper/src/recovery.rs +++ b/safekeeper/src/recovery.rs @@ -1,17 +1,41 @@ //! This module implements pulling WAL from peer safekeepers if compute can't //! provide it, i.e. safekeeper lags too much. -use std::sync::Arc; +use std::time::SystemTime; +use std::{fmt, pin::pin, sync::Arc}; -use tokio::{select, time::sleep, time::Duration}; -use tracing::{info, instrument}; +use anyhow::{bail, Context}; +use futures::StreamExt; +use postgres_protocol::message::backend::ReplicationMessage; +use tokio::sync::mpsc::{channel, Receiver, Sender}; +use tokio::time::timeout; +use tokio::{ + select, + time::sleep, + time::{self, Duration}, +}; +use tokio_postgres::replication::ReplicationStream; +use tokio_postgres::types::PgLsn; +use tracing::*; +use utils::{id::NodeId, lsn::Lsn, postgres_client::wal_stream_connection_config}; -use crate::{timeline::Timeline, SafeKeeperConf}; +use crate::receive_wal::{WalAcceptor, REPLY_QUEUE_SIZE}; +use crate::safekeeper::{AppendRequest, AppendRequestHeader}; +use crate::{ + http::routes::TimelineStatus, + receive_wal::MSG_QUEUE_SIZE, + safekeeper::{ + AcceptorProposerMessage, ProposerAcceptorMessage, ProposerElected, Term, TermHistory, + TermLsn, VoteRequest, + }, + timeline::{PeerInfo, Timeline}, + SafeKeeperConf, +}; /// Entrypoint for per timeline task which always runs, checking whether /// recovery for this safekeeper is needed and starting it if so. #[instrument(name = "recovery task", skip_all, fields(ttid = %tli.ttid))] -pub async fn recovery_main(tli: Arc, _conf: SafeKeeperConf) { +pub async fn recovery_main(tli: Arc, conf: SafeKeeperConf) { info!("started"); let mut cancellation_rx = match tli.get_cancellation_rx() { Ok(rx) => rx, @@ -22,19 +46,387 @@ pub async fn recovery_main(tli: Arc, _conf: SafeKeeperConf) { }; select! { - _ = recovery_main_loop(tli) => { unreachable!() } + _ = recovery_main_loop(tli, conf) => { unreachable!() } _ = cancellation_rx.changed() => { info!("stopped"); } } } +/// Result of Timeline::recovery_needed, contains donor(s) if recovery needed and +/// fields to explain the choice. +#[derive(Debug)] +pub struct RecoveryNeededInfo { + /// my term + pub term: Term, + /// my last_log_term + pub last_log_term: Term, + /// my flush_lsn + pub flush_lsn: Lsn, + /// peers from which we can fetch WAL, for observability. + pub peers: Vec, + /// for observability + pub num_streaming_computes: usize, + pub donors: Vec, +} + +// Custom to omit not important fields from PeerInfo. +impl fmt::Display for RecoveryNeededInfo { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{{")?; + write!( + f, + "term: {}, last_log_term: {}, flush_lsn: {}, peers: {{", + self.term, self.last_log_term, self.flush_lsn + )?; + for p in self.peers.iter() { + write!( + f, + "PeerInfo {{ sk_id: {}, term: {}, last_log_term: {}, flush_lsn: {} }}, ", + p.sk_id, p.term, p.last_log_term, p.flush_lsn + )?; + } + write!( + f, + "}} num_streaming_computes: {}, donors: {:?}", + self.num_streaming_computes, self.donors + ) + } +} + +#[derive(Clone, Debug)] +pub struct Donor { + pub sk_id: NodeId, + /// equals to last_log_term + pub term: Term, + pub flush_lsn: Lsn, + pub pg_connstr: String, + pub http_connstr: String, +} + +impl From<&PeerInfo> for Donor { + fn from(p: &PeerInfo) -> Self { + Donor { + sk_id: p.sk_id, + term: p.term, + flush_lsn: p.flush_lsn, + pg_connstr: p.pg_connstr.clone(), + http_connstr: p.http_connstr.clone(), + } + } +} + const CHECK_INTERVAL_MS: u64 = 2000; /// Check regularly whether we need to start recovery. -async fn recovery_main_loop(_tli: Arc) { +async fn recovery_main_loop(tli: Arc, conf: SafeKeeperConf) { let check_duration = Duration::from_millis(CHECK_INTERVAL_MS); loop { + let recovery_needed_info = tli.recovery_needed(conf.heartbeat_timeout).await; + match recovery_needed_info.donors.first() { + Some(donor) => { + info!( + "starting recovery from donor {}: {}", + donor.sk_id, recovery_needed_info + ); + match recover(tli.clone(), donor, &conf).await { + // Note: 'write_wal rewrites WAL written before' error is + // expected here and might happen if compute and recovery + // concurrently write the same data. Eventually compute + // should win. + Err(e) => warn!("recovery failed: {:#}", e), + Ok(msg) => info!("recovery finished: {}", msg), + } + } + None => { + trace!( + "recovery not needed or not possible: {}", + recovery_needed_info + ); + } + } sleep(check_duration).await; } } + +/// Recover from the specified donor. Returns message explaining normal finish +/// reason or error. +async fn recover( + tli: Arc, + donor: &Donor, + conf: &SafeKeeperConf, +) -> anyhow::Result { + // Learn donor term switch history to figure out starting point. + let client = reqwest::Client::new(); + let timeline_info: TimelineStatus = client + .get(format!( + "http://{}/v1/tenant/{}/timeline/{}", + donor.http_connstr, tli.ttid.tenant_id, tli.ttid.timeline_id + )) + .send() + .await? + .json() + .await?; + if timeline_info.acceptor_state.term != donor.term { + bail!( + "donor term changed from {} to {}", + donor.term, + timeline_info.acceptor_state.term + ); + } + // convert from API TermSwitchApiEntry into TermLsn. + let donor_th = TermHistory( + timeline_info + .acceptor_state + .term_history + .iter() + .map(|tl| Into::::into(*tl)) + .collect(), + ); + + // Now understand our term history. + let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: donor.term }); + let vote_response = match tli + .process_msg(&vote_request) + .await + .context("VoteRequest handling")? + { + Some(AcceptorProposerMessage::VoteResponse(vr)) => vr, + _ => { + bail!("unexpected VoteRequest response"); // unreachable + } + }; + if vote_response.term != donor.term { + bail!( + "our term changed from {} to {}", + donor.term, + vote_response.term + ); + } + + let last_common_point = match TermHistory::find_highest_common_point( + &donor_th, + &vote_response.term_history, + vote_response.flush_lsn, + ) { + None => bail!( + "couldn't find common point in histories, donor {:?}, sk {:?}", + donor_th, + vote_response.term_history, + ), + Some(lcp) => lcp, + }; + info!("found last common point at {:?}", last_common_point); + + // truncate WAL locally + let pe = ProposerAcceptorMessage::Elected(ProposerElected { + term: donor.term, + start_streaming_at: last_common_point.lsn, + term_history: donor_th, + timeline_start_lsn: Lsn::INVALID, + }); + // Successful ProposerElected handling always returns None. If term changed, + // we'll find out that during the streaming. Note: it is expected to get + // 'refusing to overwrite correct WAL' here if walproposer reconnected + // concurrently, restart helps here. + tli.process_msg(&pe) + .await + .context("ProposerElected handling")?; + + recovery_stream(tli, donor, last_common_point.lsn, conf).await +} + +// Pull WAL from donor, assuming handshake is already done. +async fn recovery_stream( + tli: Arc, + donor: &Donor, + start_streaming_at: Lsn, + conf: &SafeKeeperConf, +) -> anyhow::Result { + // TODO: pass auth token + let cfg = wal_stream_connection_config(tli.ttid, &donor.pg_connstr, None, None)?; + let mut cfg = cfg.to_tokio_postgres_config(); + // It will make safekeeper give out not committed WAL (up to flush_lsn). + cfg.application_name(&format!("safekeeper_{}", conf.my_id)); + cfg.replication_mode(tokio_postgres::config::ReplicationMode::Physical); + + let connect_timeout = Duration::from_millis(10000); + let (client, connection) = match time::timeout(connect_timeout, cfg.connect(postgres::NoTls)) + .await + { + Ok(client_and_conn) => client_and_conn?, + Err(_elapsed) => { + bail!("timed out while waiting {connect_timeout:?} for connection to peer safekeeper to open"); + } + }; + trace!("connected to {:?}", donor); + + // The connection object performs the actual communication with the + // server, spawn it off to run on its own. + let ttid = tli.ttid; + tokio::spawn(async move { + if let Err(e) = connection + .instrument(info_span!("recovery task connection poll", ttid = %ttid)) + .await + { + // This logging isn't very useful as error is anyway forwarded to client. + trace!( + "tokio_postgres connection object finished with error: {}", + e + ); + } + }); + + let query = format!( + "START_REPLICATION PHYSICAL {} (term='{}')", + start_streaming_at, donor.term + ); + + let copy_stream = client.copy_both_simple(&query).await?; + let physical_stream = ReplicationStream::new(copy_stream); + + // As in normal walreceiver, do networking and writing to disk in parallel. + let (msg_tx, msg_rx) = channel(MSG_QUEUE_SIZE); + let (reply_tx, reply_rx) = channel(REPLY_QUEUE_SIZE); + let wa = WalAcceptor::spawn(tli.clone(), msg_rx, reply_tx, None); + + let res = tokio::select! { + r = network_io(physical_stream, msg_tx, donor.clone(), tli.clone(), conf.clone()) => r, + r = read_replies(reply_rx, donor.term) => r.map(|()| None), + }; + + // Join the spawned WalAcceptor. At this point chans to/from it passed to + // network routines are dropped, so it will exit as soon as it touches them. + match wa.await { + Ok(Ok(())) => { + // WalAcceptor finished normally, termination reason is different + match res { + Ok(Some(success_desc)) => Ok(success_desc), + Ok(None) => bail!("unexpected recovery end without error/success"), // can't happen + Err(e) => Err(e), // network error or term change + } + } + Ok(Err(e)) => Err(e), // error while processing message + Err(e) => bail!("WalAcceptor panicked: {}", e), + } +} + +// Perform network part of streaming: read data and push it to msg_tx, send KA +// to make sender hear from us. If there is nothing coming for a while, check +// for termination. +// Returns +// - Ok(None) if channel to WalAcceptor closed -- its task should return error. +// - Ok(Some(String)) if recovery successfully completed. +// - Err if error happened while reading/writing to socket. +async fn network_io( + physical_stream: ReplicationStream, + msg_tx: Sender, + donor: Donor, + tli: Arc, + conf: SafeKeeperConf, +) -> anyhow::Result> { + let mut physical_stream = pin!(physical_stream); + let mut last_received_lsn = Lsn::INVALID; + // tear down connection if no data arrives withing this period + let no_data_timeout = Duration::from_millis(30000); + + loop { + let msg = match timeout(no_data_timeout, physical_stream.next()).await { + Ok(next) => match next { + None => bail!("unexpected end of replication stream"), + Some(msg) => msg.context("get replication message")?, + }, + Err(_) => bail!("no message received within {:?}", no_data_timeout), + }; + + match msg { + ReplicationMessage::XLogData(xlog_data) => { + let ar_hdr = AppendRequestHeader { + term: donor.term, + epoch_start_lsn: Lsn::INVALID, // unused + begin_lsn: Lsn(xlog_data.wal_start()), + end_lsn: Lsn(xlog_data.wal_start()) + xlog_data.data().len() as u64, + commit_lsn: Lsn::INVALID, // do not attempt to advance, peer communication anyway does it + truncate_lsn: Lsn::INVALID, // do not attempt to advance + proposer_uuid: [0; 16], + }; + let ar = AppendRequest { + h: ar_hdr, + wal_data: xlog_data.into_data(), + }; + trace!( + "processing AppendRequest {}-{}, len {}", + ar.h.begin_lsn, + ar.h.end_lsn, + ar.wal_data.len() + ); + last_received_lsn = ar.h.end_lsn; + if msg_tx + .send(ProposerAcceptorMessage::AppendRequest(ar)) + .await + .is_err() + { + return Ok(None); // chan closed, WalAcceptor terminated + } + } + ReplicationMessage::PrimaryKeepAlive(_) => { + // keepalive means nothing is being streamed for a while. Check whether we need to stop. + let recovery_needed_info = tli.recovery_needed(conf.heartbeat_timeout).await; + // do current donors still contain one we currently connected to? + if !recovery_needed_info + .donors + .iter() + .any(|d| d.sk_id == donor.sk_id) + { + // Most likely it means we are caughtup. + // note: just exiting makes tokio_postgres send CopyFail to the far end. + return Ok(Some(format!( + "terminating at {} as connected safekeeper {} with term {} is not a donor anymore: {}", + last_received_lsn, donor.sk_id, donor.term, recovery_needed_info + ))); + } + } + _ => {} + } + // Send reply to each message to keep connection alive. Ideally we + // should do that once in a while instead, but this again requires + // stream split or similar workaround, and recovery is anyway not that + // performance critical. + // + // We do not know here real write/flush LSNs (need to take mutex again + // or check replies which are read in different future), but neither + // sender much cares about them, so just send last received. + physical_stream + .as_mut() + .standby_status_update( + PgLsn::from(last_received_lsn.0), + PgLsn::from(last_received_lsn.0), + PgLsn::from(last_received_lsn.0), + SystemTime::now(), + 0, + ) + .await?; + } +} + +// Read replies from WalAcceptor. We are not interested much in sending them to +// donor safekeeper, so don't route them anywhere. However, we should check if +// term changes and exit if it does. +// Returns Ok(()) if channel closed, Err in case of term change. +async fn read_replies( + mut reply_rx: Receiver, + donor_term: Term, +) -> anyhow::Result<()> { + loop { + match reply_rx.recv().await { + Some(msg) => { + if let AcceptorProposerMessage::AppendResponse(ar) = msg { + if ar.term != donor_term { + bail!("donor term changed from {} to {}", donor_term, ar.term); + } + } + } + None => return Ok(()), // chan closed, WalAcceptor terminated + } + } +} diff --git a/safekeeper/src/safekeeper.rs b/safekeeper/src/safekeeper.rs index 85e556aff2..1437bdb50e 100644 --- a/safekeeper/src/safekeeper.rs +++ b/safekeeper/src/safekeeper.rs @@ -91,6 +91,69 @@ impl TermHistory { } TermHistory(res) } + + /// Find point of divergence between leader (walproposer) term history and + /// safekeeper. Arguments are not symmetrics as proposer history ends at + /// +infinity while safekeeper at flush_lsn. + /// C version is at walproposer SendProposerElected. + pub fn find_highest_common_point( + prop_th: &TermHistory, + sk_th: &TermHistory, + sk_wal_end: Lsn, + ) -> Option { + let (prop_th, sk_th) = (&prop_th.0, &sk_th.0); // avoid .0 below + + if let Some(sk_th_last) = sk_th.last() { + assert!( + sk_th_last.lsn <= sk_wal_end, + "safekeeper term history end {:?} LSN is higher than WAL end {:?}", + sk_th_last, + sk_wal_end + ); + } + + // find last common term, if any... + let mut last_common_idx = None; + for i in 0..min(sk_th.len(), prop_th.len()) { + if prop_th[i].term != sk_th[i].term { + break; + } + // If term is the same, LSN must be equal as well. + assert!( + prop_th[i].lsn == sk_th[i].lsn, + "same term {} has different start LSNs: prop {}, sk {}", + prop_th[i].term, + prop_th[i].lsn, + sk_th[i].lsn + ); + last_common_idx = Some(i); + } + let last_common_idx = match last_common_idx { + None => return None, // no common point + Some(lci) => lci, + }; + // Now find where it ends at both prop and sk and take min. End of + // (common) term is the start of the next except it is the last one; + // there it is flush_lsn in case of safekeeper or, in case of proposer + // +infinity, so we just take flush_lsn then. + if last_common_idx == prop_th.len() - 1 { + Some(TermLsn { + term: prop_th[last_common_idx].term, + lsn: sk_wal_end, + }) + } else { + let prop_common_term_end = prop_th[last_common_idx + 1].lsn; + let sk_common_term_end = if last_common_idx + 1 < sk_th.len() { + sk_th[last_common_idx + 1].lsn + } else { + sk_wal_end + }; + Some(TermLsn { + term: prop_th[last_common_idx].term, + lsn: min(prop_common_term_end, sk_common_term_end), + }) + } + } } /// Display only latest entries for Debug. @@ -305,19 +368,19 @@ pub struct AcceptorGreeting { /// Vote request sent from proposer to safekeepers #[derive(Debug, Deserialize)] pub struct VoteRequest { - term: Term, + pub term: Term, } /// Vote itself, sent from safekeeper to proposer #[derive(Debug, Serialize)] pub struct VoteResponse { - term: Term, // safekeeper's current term; if it is higher than proposer's, the compute is out of date. + pub term: Term, // safekeeper's current term; if it is higher than proposer's, the compute is out of date. vote_given: u64, // fixme u64 due to padding // Safekeeper flush_lsn (end of WAL) + history of term switches allow // proposer to choose the most advanced one. - flush_lsn: Lsn, + pub flush_lsn: Lsn, truncate_lsn: Lsn, - term_history: TermHistory, + pub term_history: TermHistory, timeline_start_lsn: Lsn, } @@ -344,7 +407,8 @@ pub struct AppendRequest { pub struct AppendRequestHeader { // safekeeper's current term; if it is higher than proposer's, the compute is out of date. pub term: Term, - // LSN since the proposer appends WAL; determines epoch switch point. + // TODO: remove this field, it in unused -- LSN of term switch can be taken + // from ProposerElected (as well as from term history). pub epoch_start_lsn: Lsn, /// start position of message in WAL pub begin_lsn: Lsn, @@ -759,7 +823,7 @@ where bail!("refusing ProposerElected which is going to overwrite correct WAL: term={}, flush_lsn={}, start_streaming_at={}; restarting the handshake should help", msg.term, self.flush_lsn(), msg.start_streaming_at) } - // Otherwise this shouldn't happen. + // Otherwise we must never attempt to truncate committed data. assert!( msg.start_streaming_at >= self.inmem.commit_lsn, "attempt to truncate committed data: start_streaming_at={}, commit_lsn={}", @@ -810,6 +874,14 @@ where info!("start receiving WAL since {:?}", msg.start_streaming_at); + // Cache LSN where term starts to immediately fsync control file with + // commit_lsn once we reach it -- sync-safekeepers finishes when + // persisted commit_lsn on majority of safekeepers aligns. + self.epoch_start_lsn = match msg.term_history.0.last() { + None => bail!("proposer elected with empty term history"), + Some(term_lsn_start) => term_lsn_start.lsn, + }; + Ok(None) } @@ -835,10 +907,7 @@ where // file: walproposer in sync mode is very interested when this // happens. Note: this is for sync-safekeepers mode only, as // otherwise commit_lsn might jump over epoch_start_lsn. - // Also note that commit_lsn can reach epoch_start_lsn earlier - // that we receive new epoch_start_lsn, and we still need to sync - // control file in this case. - if commit_lsn == self.epoch_start_lsn && self.state.commit_lsn != commit_lsn { + if commit_lsn >= self.epoch_start_lsn && self.state.commit_lsn < self.epoch_start_lsn { self.persist_control_file(self.state.clone()).await?; } @@ -902,7 +971,6 @@ where // Now we know that we are in the same term as the proposer, // processing the message. - self.epoch_start_lsn = msg.h.epoch_start_lsn; self.inmem.proposer_uuid = msg.h.proposer_uuid; // do the job @@ -1185,4 +1253,65 @@ mod tests { sk.wal_store.truncate_wal(Lsn(3)).await.unwrap(); // imitate the complete record at 3 %) assert_eq!(sk.get_epoch(), 1); } + + #[test] + fn test_find_highest_common_point_none() { + let prop_th = TermHistory(vec![(0, Lsn(1)).into()]); + let sk_th = TermHistory(vec![(1, Lsn(1)).into(), (2, Lsn(2)).into()]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(3),), + None + ); + } + + #[test] + fn test_find_highest_common_point_middle() { + let prop_th = TermHistory(vec![ + (1, Lsn(10)).into(), + (2, Lsn(20)).into(), + (4, Lsn(40)).into(), + ]); + let sk_th = TermHistory(vec![ + (1, Lsn(10)).into(), + (2, Lsn(20)).into(), + (3, Lsn(30)).into(), // sk ends last common term 2 at 30 + ]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(40),), + Some(TermLsn { + term: 2, + lsn: Lsn(30), + }) + ); + } + + #[test] + fn test_find_highest_common_point_sk_end() { + let prop_th = TermHistory(vec![ + (1, Lsn(10)).into(), + (2, Lsn(20)).into(), // last common term 2, sk will end it at 32 sk_end_lsn + (4, Lsn(40)).into(), + ]); + let sk_th = TermHistory(vec![(1, Lsn(10)).into(), (2, Lsn(20)).into()]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(32),), + Some(TermLsn { + term: 2, + lsn: Lsn(32), + }) + ); + } + + #[test] + fn test_find_highest_common_point_walprop() { + let prop_th = TermHistory(vec![(1, Lsn(10)).into(), (2, Lsn(20)).into()]); + let sk_th = TermHistory(vec![(1, Lsn(10)).into(), (2, Lsn(20)).into()]); + assert_eq!( + TermHistory::find_highest_common_point(&prop_th, &sk_th, Lsn(32),), + Some(TermLsn { + term: 2, + lsn: Lsn(32), + }) + ); + } } diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index b684083446..cd765dcbce 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -418,10 +418,11 @@ impl SafekeeperPostgresHandler { } info!( - "starting streaming from {:?}, available WAL ends at {}, recovery={}", + "starting streaming from {:?}, available WAL ends at {}, recovery={}, appname={:?}", start_pos, end_pos, - matches!(end_watch, EndWatch::Flush(_)) + matches!(end_watch, EndWatch::Flush(_)), + appname ); // switch to copy diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index 37821eae23..96d844fa7a 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -11,6 +11,7 @@ use tokio::fs; use serde_with::DisplayFromStr; use std::cmp::max; use std::sync::Arc; +use std::time::Duration; use tokio::sync::{Mutex, MutexGuard}; use tokio::{ sync::{mpsc::Sender, watch}, @@ -27,7 +28,7 @@ use storage_broker::proto::SafekeeperTimelineInfo; use storage_broker::proto::TenantTimelineId as ProtoTenantTimelineId; use crate::receive_wal::WalReceivers; -use crate::recovery::recovery_main; +use crate::recovery::{recovery_main, Donor, RecoveryNeededInfo}; use crate::safekeeper::{ AcceptorProposerMessage, ProposerAcceptorMessage, SafeKeeper, SafeKeeperState, SafekeeperMemState, ServerInfo, Term, TermLsn, INVALID_TERM, @@ -45,11 +46,12 @@ use crate::{debug_dump, wal_storage}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PeerInfo { pub sk_id: NodeId, + pub term: Term, /// Term of the last entry. - _last_log_term: Term, + pub last_log_term: Term, /// LSN of the last record. #[serde_as(as = "DisplayFromStr")] - _flush_lsn: Lsn, + pub flush_lsn: Lsn, #[serde_as(as = "DisplayFromStr")] pub commit_lsn: Lsn, /// Since which LSN safekeeper has WAL. TODO: remove this once we fill new @@ -61,16 +63,21 @@ pub struct PeerInfo { #[serde(skip)] #[serde(default = "Instant::now")] ts: Instant, + pub pg_connstr: String, + pub http_connstr: String, } impl PeerInfo { fn from_sk_info(sk_info: &SafekeeperTimelineInfo, ts: Instant) -> PeerInfo { PeerInfo { sk_id: NodeId(sk_info.safekeeper_id), - _last_log_term: sk_info.last_log_term, - _flush_lsn: Lsn(sk_info.flush_lsn), + term: sk_info.term, + last_log_term: sk_info.last_log_term, + flush_lsn: Lsn(sk_info.flush_lsn), commit_lsn: Lsn(sk_info.commit_lsn), local_start_lsn: Lsn(sk_info.local_start_lsn), + pg_connstr: sk_info.safekeeper_connstr.clone(), + http_connstr: sk_info.http_connstr.clone(), ts, } } @@ -112,7 +119,6 @@ pub struct SharedState { /// TODO: it might be better to remove tli completely from GlobalTimelines /// when tli is inactive instead of having this flag. active: bool, - num_computes: u32, last_removed_segno: XLogSegNo, } @@ -151,7 +157,6 @@ impl SharedState { peers_info: PeersInfo(vec![]), wal_backup_active: false, active: false, - num_computes: 0, last_removed_segno: 0, }) } @@ -171,7 +176,6 @@ impl SharedState { peers_info: PeersInfo(vec![]), wal_backup_active: false, active: false, - num_computes: 0, last_removed_segno: 0, }) } @@ -219,7 +223,7 @@ impl SharedState { }; trace!( "timeline {} s3 offloading action {} pending: num_computes={}, commit_lsn={}, backup_lsn={}", - self.sk.state.timeline_id, action_pending, self.num_computes, self.sk.inmem.commit_lsn, self.sk.inmem.backup_lsn + self.sk.state.timeline_id, action_pending, num_computes, self.sk.inmem.commit_lsn, self.sk.inmem.backup_lsn ); } res @@ -265,6 +269,20 @@ impl SharedState { availability_zone: conf.availability_zone.clone(), } } + + /// Get our latest view of alive peers status on the timeline. + /// We pass our own info through the broker as well, so when we don't have connection + /// to the broker returned vec is empty. + fn get_peers(&self, heartbeat_timeout: Duration) -> Vec { + let now = Instant::now(); + self.peers_info + .0 + .iter() + // Regard peer as absent if we haven't heard from it within heartbeat_timeout. + .filter(|p| now.duration_since(p.ts) <= heartbeat_timeout) + .cloned() + .collect() + } } #[derive(Debug, thiserror::Error)] @@ -446,7 +464,9 @@ impl Timeline { /// Bootstrap new or existing timeline starting background stasks. pub fn bootstrap(self: &Arc, conf: &SafeKeeperConf) { // Start recovery task which always runs on the timeline. - tokio::spawn(recovery_main(self.clone(), conf.clone())); + if conf.peer_recovery_enabled { + tokio::spawn(recovery_main(self.clone(), conf.clone())); + } } /// Delete timeline from disk completely, by removing timeline directory. Background @@ -531,7 +551,7 @@ impl Timeline { return true; } let shared_state = self.write_shared_state().await; - if shared_state.num_computes == 0 { + if self.walreceivers.get_num() == 0 { return shared_state.sk.inmem.commit_lsn == Lsn(0) || // no data at all yet reported_remote_consistent_lsn >= shared_state.sk.inmem.commit_lsn; } @@ -680,20 +700,88 @@ impl Timeline { Ok(()) } - /// Get our latest view of alive peers status on the timeline. - /// We pass our own info through the broker as well, so when we don't have connection - /// to the broker returned vec is empty. pub async fn get_peers(&self, conf: &SafeKeeperConf) -> Vec { let shared_state = self.write_shared_state().await; - let now = Instant::now(); - shared_state - .peers_info - .0 - .iter() - // Regard peer as absent if we haven't heard from it within heartbeat_timeout. - .filter(|p| now.duration_since(p.ts) <= conf.heartbeat_timeout) - .cloned() - .collect() + shared_state.get_peers(conf.heartbeat_timeout) + } + + /// Should we start fetching WAL from a peer safekeeper, and if yes, from + /// which? Answer is yes, i.e. .donors is not empty if 1) there is something + /// to fetch, and we can do that without running elections; 2) there is no + /// actively streaming compute, as we don't want to compete with it. + /// + /// If donor(s) are choosen, theirs last_log_term is guaranteed to be equal + /// to its last_log_term so we are sure such a leader ever had been elected. + /// + /// All possible donors are returned so that we could keep connection to the + /// current one if it is good even if it slightly lags behind. + /// + /// Note that term conditions above might be not met, but safekeepers are + /// still not aligned on last flush_lsn. Generally in this case until + /// elections are run it is not possible to say which safekeeper should + /// recover from which one -- history which would be committed is different + /// depending on assembled quorum (e.g. classic picture 8 from Raft paper). + /// Thus we don't try to predict it here. + pub async fn recovery_needed(&self, heartbeat_timeout: Duration) -> RecoveryNeededInfo { + let ss = self.write_shared_state().await; + let term = ss.sk.state.acceptor_state.term; + let last_log_term = ss.sk.get_epoch(); + let flush_lsn = ss.sk.flush_lsn(); + // note that peers contain myself, but that's ok -- we are interested only in peers which are strictly ahead of us. + let mut peers = ss.get_peers(heartbeat_timeout); + // Sort by pairs. + peers.sort_by(|p1, p2| { + let tl1 = TermLsn { + term: p1.last_log_term, + lsn: p1.flush_lsn, + }; + let tl2 = TermLsn { + term: p2.last_log_term, + lsn: p2.flush_lsn, + }; + tl2.cmp(&tl1) // desc + }); + let num_streaming_computes = self.walreceivers.get_num_streaming(); + let donors = if num_streaming_computes > 0 { + vec![] // If there is a streaming compute, don't try to recover to not intervene. + } else { + peers + .iter() + .filter_map(|candidate| { + // Are we interested in this candidate? + let candidate_tl = TermLsn { + term: candidate.last_log_term, + lsn: candidate.flush_lsn, + }; + let my_tl = TermLsn { + term: last_log_term, + lsn: flush_lsn, + }; + if my_tl < candidate_tl { + // Yes, we are interested. Can we pull from it without + // (re)running elections? It is possible if 1) his term + // is equal to his last_log_term so we could act on + // behalf of leader of this term (we must be sure he was + // ever elected) and 2) our term is not higher, or we'll refuse data. + if candidate.term == candidate.last_log_term && candidate.term >= term { + Some(Donor::from(candidate)) + } else { + None + } + } else { + None + } + }) + .collect() + }; + RecoveryNeededInfo { + term, + last_log_term, + flush_lsn, + peers, + num_streaming_computes, + donors, + } } pub fn get_walsenders(&self) -> &Arc { @@ -765,7 +853,7 @@ impl Timeline { ps_feedback, wal_backup_active: state.wal_backup_active, timeline_is_active: state.active, - num_computes: state.num_computes, + num_computes: self.walreceivers.get_num() as u32, last_removed_segno: state.last_removed_segno, epoch_start_lsn: state.sk.epoch_start_lsn, mem_state: state.sk.inmem.clone(), @@ -792,7 +880,7 @@ impl Timeline { walsenders: self.walsenders.get_all(), wal_backup_active: state.wal_backup_active, active: state.active, - num_computes: state.num_computes, + num_computes: self.walreceivers.get_num() as u32, last_removed_segno: state.last_removed_segno, epoch_start_lsn: state.sk.epoch_start_lsn, mem_state: state.sk.inmem.clone(), diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index d9a75637b8..38e17985ac 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -1631,7 +1631,10 @@ class NeonPageserver(PgProtocol): ".*took more than expected to complete.*", # these can happen during shutdown, but it should not be a reason to fail a test ".*completed, took longer than expected.*", - '.*registered custom resource manager "neon".*', + '.*registered custom resource manager \\\\"neon\\\\".*', + # AWS S3 may emit 500 errors for keys in a DeleteObjects response: we retry these + # and it is not a failure of our code when it happens. + ".*DeleteObjects.*We encountered an internal error. Please try again.*", ] def timeline_dir(self, tenant_id: TenantId, timeline_id: Optional[TimelineId] = None) -> Path: @@ -2755,6 +2758,20 @@ class Safekeeper: def data_dir(self) -> str: return os.path.join(self.env.repo_dir, "safekeepers", f"sk{self.id}") + def timeline_dir(self, tenant_id, timeline_id) -> str: + return os.path.join(self.data_dir(), str(tenant_id), str(timeline_id)) + + def list_segments(self, tenant_id, timeline_id) -> List[str]: + """ + Get list of segment names of the given timeline. + """ + tli_dir = self.timeline_dir(tenant_id, timeline_id) + segments = [] + for _, _, filenames in os.walk(tli_dir): + segments.extend([f for f in filenames if f != "safekeeper.control"]) + segments.sort() + return segments + @dataclass class SafekeeperTimelineStatus: @@ -3119,6 +3136,22 @@ def check_restored_datadir_content( assert (mismatch, error) == ([], []) +def logical_replication_sync(subscriber: VanillaPostgres, publisher: Endpoint) -> Lsn: + """Wait logical replication subscriber to sync with publisher.""" + publisher_lsn = Lsn(publisher.safe_psql("SELECT pg_current_wal_flush_lsn()")[0][0]) + while True: + res = subscriber.safe_psql("select latest_end_lsn from pg_catalog.pg_stat_subscription")[0][ + 0 + ] + if res: + log.info(f"subscriber_lsn={res}") + subscriber_lsn = Lsn(res) + log.info(f"Subscriber LSN={subscriber_lsn}, publisher LSN={ publisher_lsn}") + if subscriber_lsn >= publisher_lsn: + return subscriber_lsn + time.sleep(0.5) + + def wait_for_last_flush_lsn( env: NeonEnv, endpoint: Endpoint, diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 460a30ad56..598b48b56f 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -453,6 +453,15 @@ class PageserverHttpClient(requests.Session): res_json = res.json() return res_json + def timeline_get_timestamp_of_lsn(self, tenant_id: TenantId, timeline_id: TimelineId, lsn: Lsn): + log.info(f"Requesting time range of lsn {lsn}, tenant {tenant_id}, timeline {timeline_id}") + res = self.get( + f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/get_timestamp_of_lsn?lsn={lsn}", + ) + self.verbose_error(res) + res_json = res.json() + return res_json + def timeline_checkpoint(self, tenant_id: TenantId, timeline_id: TimelineId): self.is_testing_enabled_or_skip() diff --git a/test_runner/performance/test_logical_replication.py b/test_runner/performance/test_logical_replication.py new file mode 100644 index 0000000000..b799f7248f --- /dev/null +++ b/test_runner/performance/test_logical_replication.py @@ -0,0 +1,43 @@ +import time + +import pytest +from fixtures.log_helper import log +from fixtures.neon_fixtures import NeonEnv, PgBin, logical_replication_sync + + +@pytest.mark.timeout(1000) +def test_logical_replication(neon_simple_env: NeonEnv, pg_bin: PgBin, vanilla_pg): + env = neon_simple_env + + env.neon_cli.create_branch("test_logical_replication", "empty") + endpoint = env.endpoints.create_start("test_logical_replication") + + log.info("postgres is running on 'test_logical_replication' branch") + pg_bin.run_capture(["pgbench", "-i", "-s10", endpoint.connstr()]) + + endpoint.safe_psql("create publication pub1 for table pgbench_accounts, pgbench_history") + + # now start subscriber + vanilla_pg.start() + pg_bin.run_capture(["pgbench", "-i", "-s10", vanilla_pg.connstr()]) + + vanilla_pg.safe_psql("truncate table pgbench_accounts") + vanilla_pg.safe_psql("truncate table pgbench_history") + + connstr = endpoint.connstr().replace("'", "''") + print(f"connstr='{connstr}'") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + + # Wait logical replication channel to be established + logical_replication_sync(vanilla_pg, endpoint) + + pg_bin.run_capture(["pgbench", "-c10", "-T100", "-Mprepared", endpoint.connstr()]) + + # Wait logical replication to sync + start = time.time() + logical_replication_sync(vanilla_pg, endpoint) + log.info(f"Sync with master took {time.time() - start} seconds") + + sum_master = endpoint.safe_psql("select sum(abalance) from pgbench_accounts")[0][0] + sum_replica = vanilla_pg.safe_psql("select sum(abalance) from pgbench_accounts")[0][0] + assert sum_master == sum_replica diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock index bdbbe0ad69..4ea21eb378 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.lock @@ -3,10 +3,25 @@ version = 3 [[package]] -name = "async-trait" -version = "0.1.68" +name = "addr2line" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" dependencies = [ "proc-macro2", "quote", @@ -20,10 +35,25 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] -name = "base64" -version = "0.21.0" +name = "backtrace" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + +[[package]] +name = "base64" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" [[package]] name = "bitflags" @@ -31,6 +61,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.10.4" @@ -41,22 +77,31 @@ dependencies = [ ] [[package]] -name = "byteorder" -version = "1.4.3" +name = "bumpalo" +version = "3.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" +checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.4.0" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" [[package]] name = "cc" -version = "1.0.79" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" +dependencies = [ + "libc", +] [[package]] name = "cfg-if" @@ -82,9 +127,9 @@ checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" [[package]] name = "cpufeatures" -version = "0.2.7" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c1eaa2012c47becbbad2ab175484c2a84d1185b566fb2cc5b8707343dfe58" +checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" dependencies = [ "libc", ] @@ -101,9 +146,9 @@ dependencies = [ [[package]] name = "digest" -version = "0.10.6" +version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8168378f4e5023e7218c89c891c0fd8ecdb5e5e4f18cb78f38cf245dd021e76f" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", @@ -112,23 +157,12 @@ dependencies = [ [[package]] name = "errno" -version = "0.3.1" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys 0.48.0", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys", ] [[package]] @@ -139,12 +173,15 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "fastrand" -version = "1.9.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" + +[[package]] +name = "finl_unicode" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdc7a0362c9f4444381a9e697c79d435fe65b52a37466fc2c1184cee9edc6" [[package]] name = "foreign-types" @@ -262,9 +299,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.9" +version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" dependencies = [ "cfg-if", "libc", @@ -272,10 +309,10 @@ dependencies = [ ] [[package]] -name = "hermit-abi" -version = "0.3.1" +name = "gimli" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "hmac" @@ -287,23 +324,12 @@ dependencies = [ ] [[package]] -name = "instant" -version = "0.1.12" +name = "js-sys" +version = "0.3.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +checksum = "c5f195fe497f702db0f318b07fdd68edb16955aed830df8363d837542f8f935a" dependencies = [ - "cfg-if", -] - -[[package]] -name = "io-lifetimes" -version = "1.0.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" -dependencies = [ - "hermit-abi", - "libc", - "windows-sys 0.48.0", + "wasm-bindgen", ] [[package]] @@ -314,21 +340,21 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.142" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "linux-raw-sys" -version = "0.3.4" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36eb31c1778188ae1e64398743890d0877fef36d11521ac60406b42016e8c2cf" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "lock_api" -version = "0.4.9" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "435011366fe56583b16cf956f9df0095b405b82d76425bc8981c0e22e60ec4df" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" dependencies = [ "autocfg", "scopeguard", @@ -336,38 +362,44 @@ dependencies = [ [[package]] name = "log" -version = "0.4.17" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" -dependencies = [ - "cfg-if", -] +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "md-5" -version = "0.10.5" +version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6365506850d44bff6e2fbcb5176cf63650e48bd45ef2fe2665ae1570e0f4b9ca" +checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ + "cfg-if", "digest", ] [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" + +[[package]] +name = "miniz_oxide" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +dependencies = [ + "adler", +] [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -389,18 +421,27 @@ dependencies = [ ] [[package]] -name = "once_cell" -version = "1.17.1" +name = "object" +version = "0.32.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" +checksum = "9cf5f9dd3933bd50a9e1f149ec995f39ae2c496d31fd772c1fd45ebc27e902b0" +dependencies = [ + "memchr", +] + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "openssl" -version = "0.10.55" +version = "0.10.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "345df152bc43501c5eb9e4654ff05f794effb78d4efe3d53abc158baddc0703d" +checksum = "bac25ee399abb46215765b1cb35bc0212377e58a061560d8b29b024fd0430e7c" dependencies = [ - "bitflags", + "bitflags 2.4.1", "cfg-if", "foreign-types", "libc", @@ -428,9 +469,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-sys" -version = "0.9.90" +version = "0.9.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "374533b0e45f3a7ced10fcaeccca020e66656bc03dac384f852e4e5a7a8104a6" +checksum = "db4d56a4c0478783083cfafcc42493dd4a981d41669da64b4572a2a089b51b1d" dependencies = [ "cc", "libc", @@ -450,46 +491,46 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.7" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" +checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" dependencies = [ "cfg-if", "libc", - "redox_syscall 0.2.16", + "redox_syscall 0.4.1", "smallvec", - "windows-sys 0.45.0", + "windows-targets", ] [[package]] name = "percent-encoding" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "phf" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_shared", ] [[package]] name = "phf_shared" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" dependencies = [ "siphasher", ] [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -499,9 +540,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "pkg-config" -version = "0.3.26" +version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" +checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" [[package]] name = "postgres-native-tls" @@ -518,9 +559,9 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" +checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ "base64", "byteorder", @@ -536,9 +577,9 @@ dependencies = [ [[package]] name = "postgres-types" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f028f05971fe20f512bcc679e2c10227e57809a3af86a7606304435bc8896cd6" +checksum = "8d2234cdee9408b523530a9b6d2d6b373d1db34f6a8e51dc03ded1828d7fb67c" dependencies = [ "bytes", "fallible-iterator", @@ -553,18 +594,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.56" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" +checksum = "134c189feb4956b20f6f547d2cf727d4c0fe06722b20a0eec87ed445a97f92da" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.26" +version = "1.0.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" +checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae" dependencies = [ "proc-macro2", ] @@ -599,22 +640,22 @@ dependencies = [ "getrandom", ] -[[package]] -name = "redox_syscall" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" -dependencies = [ - "bitflags", -] - [[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +dependencies = [ + "bitflags 1.3.2", ] [[package]] @@ -628,41 +669,46 @@ dependencies = [ ] [[package]] -name = "rustix" -version = "0.37.15" +name = "rustc-demangle" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0661814f891c57c930a610266415528da53c4933e6dea5fb350cbfe048a9ece" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + +[[package]] +name = "rustix" +version = "0.38.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags", + "bitflags 2.4.1", "errno", - "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "schannel" -version = "0.1.21" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" +checksum = "0c3733bf4cf7ea0880754e19cb5a462007c4a8c1914bff372ccc95b464f1df88" dependencies = [ - "windows-sys 0.42.0", + "windows-sys", ] [[package]] name = "scopeguard" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" [[package]] name = "security-framework" -version = "2.8.2" +version = "2.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" +checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" dependencies = [ - "bitflags", + "bitflags 1.3.2", "core-foundation", "core-foundation-sys", "libc", @@ -671,9 +717,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.8.0" +version = "2.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" +checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" dependencies = [ "core-foundation-sys", "libc", @@ -681,9 +727,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.10.6" +version = "0.10.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82e6b795fe2e3b1e845bafcb27aa35405c4d47cdfc92af5fc8d3002f76cebdc0" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" dependencies = [ "cfg-if", "cpufeatures", @@ -692,66 +738,57 @@ dependencies = [ [[package]] name = "siphasher" -version = "0.3.10" +version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" +checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" [[package]] name = "slab" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" dependencies = [ "autocfg", ] [[package]] name = "smallvec" -version = "1.10.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" [[package]] name = "socket2" -version = "0.4.9" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d283f86695ae989d1e18440a943880967156325ba025f05049946bff47bcc2b" -dependencies = [ - "libc", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] name = "stringprep" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ee348cb74b87454fff4b551cbf727025810a004f88aeacae7f85b87f4e9a1c1" +checksum = "bb41d74e231a107a1b4ee36bd1214b11285b77768d2e3824aedafa988fd36ee6" dependencies = [ + "finl_unicode", "unicode-bidi", "unicode-normalization", ] [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "syn" -version = "2.0.15" +version = "2.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" +checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b" dependencies = [ "proc-macro2", "quote", @@ -760,15 +797,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.5.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.3.5", "rustix", - "windows-sys 0.45.0", + "windows-sys", ] [[package]] @@ -788,18 +825,18 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.0" +version = "1.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "4f38200e3ef7995e5ef13baec2f432a6da0aa9ac495b2c0e8f3b7eec2c92d653" dependencies = [ - "autocfg", + "backtrace", "bytes", "libc", "mio", "pin-project-lite", - "socket2 0.4.9", + "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys", ] [[package]] @@ -825,9 +862,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.8" +version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" +checksum = "d340244b32d920260ae7448cb72b6e238bddc3d4f7603394e7dd46ed8e48f5b8" dependencies = [ "async-trait", "byteorder", @@ -842,16 +879,18 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2 0.5.2", + "rand", + "socket2", "tokio", "tokio-util", + "whoami", ] [[package]] name = "tokio-util" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +checksum = "1d68074620f57a0b21594d9735eb2e98ab38b17f80d3fcb189fca266771ca60d" dependencies = [ "bytes", "futures-core", @@ -863,9 +902,9 @@ dependencies = [ [[package]] name = "tracing" -version = "0.1.38" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9cf6a813d3f40c88b0b6b6f29a5c95c6cdbf97c1f9cc53fb820200f5ad814d" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", "tracing-core", @@ -873,18 +912,18 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", ] [[package]] name = "typenum" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "497961ef93d974e23eb6f433eb5fe1b7930b659f06d12dec6fc44a8f554c0bba" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" [[package]] name = "unicode-bidi" @@ -894,9 +933,9 @@ checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" [[package]] name = "unicode-ident" -version = "1.0.8" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "unicode-normalization" @@ -926,49 +965,77 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] -name = "winapi" -version = "0.3.9" +name = "wasm-bindgen" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "7706a72ab36d8cb1f80ffbf0e071533974a60d0a308d01a5d0375bf60499a342" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "cfg-if", + "wasm-bindgen-macro", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" +name = "wasm-bindgen-backend" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-sys" -version = "0.42.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" +checksum = "5ef2b6d3c510e9625e5fe6f509ab07d66a760f0885d858736483c32ed7809abd" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", ] [[package]] -name = "windows-sys" -version = "0.45.0" +name = "wasm-bindgen-macro" +version = "0.2.87" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "dee495e55982a3bd48105a7b947fd2a9b4a8ae3010041b9e0faab3f9cd028f1d" dependencies = [ - "windows-targets 0.42.2", + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.87" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca6ad05a4870b2bf5fe995117d3728437bd27d7cd5f06f13c17443ef369775a1" + +[[package]] +name = "web-sys" +version = "0.3.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b85cbef8c220a6abc02aefd892dfc0fc23afb1c6a426316ec33253a3877249b" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "whoami" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +dependencies = [ + "wasm-bindgen", + "web-sys", ] [[package]] @@ -977,119 +1044,62 @@ version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "windows-targets 0.48.0", + "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows_aarch64_gnullvm 0.42.2", - "windows_aarch64_msvc 0.42.2", - "windows_i686_gnu 0.42.2", - "windows_i686_msvc 0.42.2", - "windows_x86_64_gnu 0.42.2", - "windows_x86_64_gnullvm 0.42.2", - "windows_x86_64_msvc 0.42.2", -] - -[[package]] -name = "windows-targets" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" -dependencies = [ - "windows_aarch64_gnullvm 0.48.0", - "windows_aarch64_msvc 0.48.0", - "windows_i686_gnu 0.48.0", - "windows_i686_msvc 0.48.0", - "windows_x86_64_gnu 0.48.0", - "windows_x86_64_gnullvm 0.48.0", - "windows_x86_64_msvc 0.48.0", + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" - -[[package]] -name = "windows_i686_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" - -[[package]] -name = "windows_i686_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" diff --git a/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml b/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml index 4675ac8a3f..6f100aafd5 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml +++ b/test_runner/pg_clients/rust/tokio-postgres/Cargo.toml @@ -9,8 +9,8 @@ publish = false [dependencies] native-tls = "0.2.11" postgres-native-tls = "0.5.0" -tokio = { version = "1.28", features=["rt", "macros"] } -tokio-postgres = "0.7.8" +tokio = { version = "1.33", features=["rt", "macros"] } +tokio-postgres = "0.7.10" # This is not part of the main 'neon' workspace diff --git a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile index 35ae25a470..1d3709803e 100644 --- a/test_runner/pg_clients/rust/tokio-postgres/Dockerfile +++ b/test_runner/pg_clients/rust/tokio-postgres/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.70 +FROM rust:1.73 WORKDIR /source COPY . . diff --git a/test_runner/regress/test_branching.py b/test_runner/regress/test_branching.py index 2541d5d475..32b1466c90 100644 --- a/test_runner/regress/test_branching.py +++ b/test_runner/regress/test_branching.py @@ -311,8 +311,8 @@ def test_competing_branchings_from_loading_race_to_ok_or_err(neon_env_builder: N assert isinstance(failed, Exception) assert isinstance(succeeded, Dict) - # FIXME: there's probably multiple valid status codes: - # - Timeline 62505b9a9f6b1d29117b1b74eaf07b12/56cd19d3b2dbcc65e9d53ec6ca304f24 already exists + # there's multiple valid status codes: + # - Timeline x/y already exists # - whatever 409 response says, but that is a subclass of PageserverApiException assert isinstance(failed, PageserverApiException) assert succeeded["state"] == "Active" @@ -320,17 +320,14 @@ def test_competing_branchings_from_loading_race_to_ok_or_err(neon_env_builder: N # we might still have the failpoint active env.pageserver.stop(immediate=True) - # pytest should nag if we leave threads unjoined for t in threads: t.join() create_root.join() -def test_non_uploaded_branch_availability_after_restart(neon_env_builder: NeonEnvBuilder): +def test_non_uploaded_root_timeline_is_deleted_after_restart(neon_env_builder: NeonEnvBuilder): """ - Currently before RFC#27 we keep and continue uploading branches which were not successfully uploaded before shutdown. - - This test likely duplicates some other test, but it's easier to write one than to make sure there will be a failing test when the rfc is implemented. + Check that a timeline is deleted locally on subsequent restart if it never successfully uploaded during creation. """ env = neon_env_builder.init_configs() @@ -366,9 +363,59 @@ def test_non_uploaded_branch_availability_after_restart(neon_env_builder: NeonEn wait_until_tenant_active(ps_http, env.initial_tenant) - # currently it lives on and will get eventually uploaded, but this will change - detail = ps_http.timeline_detail(env.initial_tenant, env.initial_timeline) - assert detail["state"] == "Active" + with pytest.raises(PageserverApiException, match="not found"): + ps_http.timeline_detail(env.initial_tenant, env.initial_timeline) + + +def test_non_uploaded_branch_is_deleted_after_restart(neon_env_builder: NeonEnvBuilder): + """ + Check that a timeline is deleted locally on subsequent restart if it never successfully uploaded during creation. + """ + + env = neon_env_builder.init_configs() + env.start() + + env.pageserver.allowed_errors.append( + ".*request{method=POST path=/v1/tenant/.*/timeline request_id=.*}: request was dropped before completing.*" + ) + ps_http = env.pageserver.http_client() + + ps_http.tenant_create(env.initial_tenant) + ps_http.timeline_create(env.pg_version, env.initial_tenant, env.initial_timeline) + + # pause all uploads + ps_http.configure_failpoints(("before-upload-index-pausable", "pause")) + branch_id = TimelineId.generate() + + def start_creating_timeline(): + with pytest.raises(RequestException): + ps_http.timeline_create( + env.pg_version, + env.initial_tenant, + branch_id, + ancestor_timeline_id=env.initial_timeline, + timeout=60, + ) + + t = threading.Thread(target=start_creating_timeline) + try: + t.start() + + wait_until_paused(env, "before-upload-index-pausable") + finally: + # FIXME: paused uploads bother shutdown + env.pageserver.stop(immediate=True) + t.join() + + # now without a failpoint + env.pageserver.start() + + wait_until_tenant_active(ps_http, env.initial_tenant) + + ps_http.timeline_detail(env.initial_tenant, env.initial_timeline) + + with pytest.raises(PageserverApiException, match="not found"): + ps_http.timeline_detail(env.initial_tenant, branch_id) def wait_until_paused(env: NeonEnv, failpoint: str): diff --git a/test_runner/regress/test_duplicate_layers.py b/test_runner/regress/test_duplicate_layers.py index ec5a2f7473..bcf99cae7c 100644 --- a/test_runner/regress/test_duplicate_layers.py +++ b/test_runner/regress/test_duplicate_layers.py @@ -1,9 +1,9 @@ import time import pytest -from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder, PgBin, wait_for_last_flush_lsn from fixtures.pageserver.utils import ( + wait_for_last_record_lsn, wait_for_upload_queue_empty, wait_until_tenant_active, ) @@ -41,9 +41,12 @@ def test_duplicate_layers(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): """ - This test sets fail point at the end of first compaction phase: - after flushing new L1 layers but before deletion of L0 layers - it should cause generation of duplicate L1 layer by compaction after restart. + Test sets fail point at the end of first compaction phase: after + flushing new L1 layer but before deletion of L0 layers. + + The L1 used to be overwritten, but with crash-consistency via remote + index_part.json, we end up deleting the not yet uploaded L1 layer on + startup. """ neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) @@ -65,7 +68,8 @@ def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) connstr = endpoint.connstr(options="-csynchronous_commit=off") pg_bin.run_capture(["pgbench", "-i", "-s1", connstr]) - wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + lsn = wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop() # make sure we receive no new wal after this, so that we'll write over the same L1 file. endpoint.stop() @@ -74,7 +78,7 @@ def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) # hit the exit failpoint with pytest.raises(ConnectionError, match="Remote end closed connection without response"): - pageserver_http.timeline_compact(tenant_id, timeline_id) + pageserver_http.timeline_checkpoint(tenant_id, timeline_id) env.pageserver.stop() # now the duplicate L1 has been created, but is not yet uploaded @@ -107,33 +111,32 @@ def test_actually_duplicated_l1(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin) l1_found = path assert l1_found is not None, "failed to find L1 locally" - original_created_at = l1_found.stat()[8] uploaded = env.pageserver_remote_storage.timeline_path(tenant_id, timeline_id) / l1_found.name assert not uploaded.exists(), "to-be-overwritten should not yet be uploaded" - # give room for fs timestamps - time.sleep(1) - env.pageserver.start() wait_until_tenant_active(pageserver_http, tenant_id) - message = f".*duplicated L1 layer layer={l1_found.name}" - env.pageserver.allowed_errors.append(message) + assert not l1_found.exists(), "partial compaction result should had been removed during startup" + + # wait for us to catch up again + wait_for_last_record_lsn(pageserver_http, tenant_id, timeline_id, lsn) pageserver_http.timeline_compact(tenant_id, timeline_id) + # give time for log flush time.sleep(1) + message = f".*duplicated L1 layer layer={l1_found.name}" found_msg = env.pageserver.log_contains(message) - assert found_msg is not None, "no layer was duplicated, has this been fixed already?" + # resident or evicted, it should not be overwritten, however it should had been non-existing at startup + assert ( + found_msg is None + ), "layer should had been removed during startup, did it live on as evicted?" - log.info(f"found log line: {found_msg}") - - overwritten_at = l1_found.stat()[8] - assert original_created_at < overwritten_at, "expected the L1 to be overwritten" + assert l1_found.exists(), "the L1 reappears" wait_for_upload_queue_empty(pageserver_http, tenant_id, timeline_id) - uploaded_at = uploaded.stat()[8] - assert overwritten_at <= uploaded_at, "expected the L1 to finally be uploaded" + assert uploaded.exists(), "the L1 is uploaded" diff --git a/test_runner/regress/test_gc_aggressive.py b/test_runner/regress/test_gc_aggressive.py index 017a38f85c..ef68049ee7 100644 --- a/test_runner/regress/test_gc_aggressive.py +++ b/test_runner/regress/test_gc_aggressive.py @@ -2,7 +2,6 @@ import asyncio import concurrent.futures import random -import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( Endpoint, @@ -95,13 +94,12 @@ def test_gc_aggressive(neon_env_builder: NeonEnvBuilder): # -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_gc_index_upload(neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind): +def test_gc_index_upload(neon_env_builder: NeonEnvBuilder): # Disable time-based pitr, we will use LSN-based thresholds in the manual GC calls neon_env_builder.pageserver_config_override = "tenant_config={pitr_interval = '0 sec'}" num_index_uploads = 0 - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() tenant_id = env.initial_tenant diff --git a/test_runner/regress/test_layer_eviction.py b/test_runner/regress/test_layer_eviction.py index 3e23a8e165..3ab3700866 100644 --- a/test_runner/regress/test_layer_eviction.py +++ b/test_runner/regress/test_layer_eviction.py @@ -1,6 +1,5 @@ import time -import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import ( NeonEnvBuilder, @@ -14,12 +13,10 @@ from fixtures.utils import query_scalar # Crates a few layers, ensures that we can evict them (removing locally but keeping track of them anyway) # and then download them back. -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_basic_eviction( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start( initial_tenant_conf={ diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py new file mode 100644 index 0000000000..726e5e5def --- /dev/null +++ b/test_runner/regress/test_logical_replication.py @@ -0,0 +1,149 @@ +import time + +from fixtures.log_helper import log +from fixtures.neon_fixtures import ( + NeonEnv, + logical_replication_sync, + wait_for_last_flush_lsn, +) + + +def test_logical_replication(neon_simple_env: NeonEnv, vanilla_pg): + env = neon_simple_env + + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_logical_replication", "empty") + endpoint = env.endpoints.create_start( + "test_logical_replication", config_lines=["log_statement=all"] + ) + + log.info("postgres is running on 'test_logical_replication' branch") + pg_conn = endpoint.connect() + cur = pg_conn.cursor() + + cur.execute("create table t(pk integer primary key, payload integer)") + cur.execute( + "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120));" + ) + cur.execute("create publication pub1 for table t, replication_example") + + # now start subscriber + vanilla_pg.start() + vanilla_pg.safe_psql("create table t(pk integer primary key, payload integer)") + vanilla_pg.safe_psql( + "CREATE TABLE replication_example(id SERIAL PRIMARY KEY, somedata int, text varchar(120), testcolumn1 int, testcolumn2 int, testcolumn3 int);" + ) + connstr = endpoint.connstr().replace("'", "''") + log.info(f"ep connstr is {endpoint.connstr()}, subscriber connstr {vanilla_pg.connstr()}") + vanilla_pg.safe_psql(f"create subscription sub1 connection '{connstr}' publication pub1") + + # Wait logical replication channel to be established + logical_replication_sync(vanilla_pg, endpoint) + + # insert some data + cur.execute("insert into t values (generate_series(1,1000), 0)") + + # Wait logical replication to sync + logical_replication_sync(vanilla_pg, endpoint) + assert vanilla_pg.safe_psql("select count(*) from t")[0][0] == 1000 + + # now stop subscriber... + vanilla_pg.stop() + + # ... and insert some more data which should be delivered to subscriber after restart + cur.execute("insert into t values (generate_series(1001,2000), 0)") + + # Restart compute + endpoint.stop() + endpoint.start() + + # start subscriber + vanilla_pg.start() + + # Wait logical replication to sync + logical_replication_sync(vanilla_pg, endpoint) + + # Check that subscribers receives all data + assert vanilla_pg.safe_psql("select count(*) from t")[0][0] == 2000 + + # Test that save/restore of RewriteMappingFile works. Partial copy of + # rewrite.sql test. + log.info("checking rewriteheap") + vanilla_pg.stop() + cmds = """ +INSERT INTO replication_example(somedata) VALUES (1); + +BEGIN; +INSERT INTO replication_example(somedata) VALUES (2); +ALTER TABLE replication_example ADD COLUMN testcolumn1 int; +INSERT INTO replication_example(somedata, testcolumn1) VALUES (3, 1); +COMMIT; + +BEGIN; +INSERT INTO replication_example(somedata) VALUES (3); +ALTER TABLE replication_example ADD COLUMN testcolumn2 int; +INSERT INTO replication_example(somedata, testcolumn1, testcolumn2) VALUES (4, 2, 1); +COMMIT; + +VACUUM FULL pg_am; +VACUUM FULL pg_amop; +VACUUM FULL pg_proc; +VACUUM FULL pg_opclass; +VACUUM FULL pg_type; +VACUUM FULL pg_index; +VACUUM FULL pg_database; + + +-- repeated rewrites that fail +BEGIN; +CLUSTER pg_class USING pg_class_oid_index; +CLUSTER pg_class USING pg_class_oid_index; +ROLLBACK; + +-- repeated rewrites that succeed +BEGIN; +CLUSTER pg_class USING pg_class_oid_index; +CLUSTER pg_class USING pg_class_oid_index; +CLUSTER pg_class USING pg_class_oid_index; +COMMIT; + +-- repeated rewrites in different transactions +VACUUM FULL pg_class; +VACUUM FULL pg_class; + +-- reindexing of important relations / indexes +REINDEX TABLE pg_class; +REINDEX INDEX pg_class_oid_index; +REINDEX INDEX pg_class_tblspc_relfilenode_index; + +INSERT INTO replication_example(somedata, testcolumn1) VALUES (5, 3); + +BEGIN; +INSERT INTO replication_example(somedata, testcolumn1) VALUES (6, 4); +ALTER TABLE replication_example ADD COLUMN testcolumn3 int; +INSERT INTO replication_example(somedata, testcolumn1, testcolumn3) VALUES (7, 5, 1); +COMMIT; +""" + endpoint.safe_psql_many([q for q in cmds.splitlines() if q != "" and not q.startswith("-")]) + + # refetch rewrite files from pageserver + endpoint.stop() + endpoint.start() + + vanilla_pg.start() + logical_replication_sync(vanilla_pg, endpoint) + eq_q = "select testcolumn1, testcolumn2, testcolumn3 from replication_example order by 1, 2, 3" + assert vanilla_pg.safe_psql(eq_q) == endpoint.safe_psql(eq_q) + log.info("rewriteheap synced") + + # test that removal of repl slots works across restart + vanilla_pg.stop() + time.sleep(1) # wait for conn termination; active slots can't be dropped + endpoint.safe_psql("select pg_drop_replication_slot('sub1');") + endpoint.safe_psql("insert into t values (2001, 1);") # forces WAL flush + # wait for drop message to reach safekeepers (it is not transactional) + wait_for_last_flush_lsn(env, endpoint, tenant_id, timeline_id) + endpoint.stop() + endpoint.start() + # it must be gone (but walproposer slot still exists, hence 1) + assert endpoint.safe_psql("select count(*) from pg_replication_slots")[0][0] == 1 diff --git a/test_runner/regress/test_lsn_mapping.py b/test_runner/regress/test_lsn_mapping.py index 8ccfc21cf7..03606e3c1c 100644 --- a/test_runner/regress/test_lsn_mapping.py +++ b/test_runner/regress/test_lsn_mapping.py @@ -1,7 +1,10 @@ -from datetime import timedelta +import time +from datetime import datetime, timedelta, timezone from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnvBuilder, wait_for_last_flush_lsn +from fixtures.pageserver.http import PageserverApiException +from fixtures.types import Lsn from fixtures.utils import query_scalar @@ -25,13 +28,14 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): cur.execute("CREATE TABLE foo (x integer)") tbl = [] for i in range(1000): - cur.execute(f"INSERT INTO foo VALUES({i})") + cur.execute("INSERT INTO foo VALUES(%s)", (i,)) # Get the timestamp at UTC after_timestamp = query_scalar(cur, "SELECT clock_timestamp()").replace(tzinfo=None) tbl.append([i, after_timestamp]) # Execute one more transaction with synchronous_commit enabled, to flush # all the previous transactions + cur.execute("SET synchronous_commit=on") cur.execute("INSERT INTO foo VALUES (-1)") # Wait until WAL is received by pageserver @@ -67,3 +71,100 @@ def test_lsn_mapping(neon_env_builder: NeonEnvBuilder): assert endpoint_here.safe_psql("SELECT max(x) FROM foo")[0][0] == i endpoint_here.stop_and_destroy() + + +# Test pageserver get_timestamp_of_lsn API +def test_ts_of_lsn_api(neon_env_builder: NeonEnvBuilder): + env = neon_env_builder.init_start() + + new_timeline_id = env.neon_cli.create_branch("test_ts_of_lsn_api") + endpoint_main = env.endpoints.create_start("test_ts_of_lsn_api") + log.info("postgres is running on 'test_ts_of_lsn_api' branch") + + cur = endpoint_main.connect().cursor() + # Create table, and insert rows, each in a separate transaction + # Disable synchronous_commit to make this initialization go faster. + # + # Each row contains current insert LSN and the current timestamp, when + # the row was inserted. + cur.execute("SET synchronous_commit=off") + cur.execute("CREATE TABLE foo (x integer)") + tbl = [] + for i in range(1000): + cur.execute("INSERT INTO foo VALUES(%s)", (i,)) + # Get the timestamp at UTC + after_timestamp = query_scalar(cur, "SELECT clock_timestamp()").replace(tzinfo=timezone.utc) + after_lsn = query_scalar(cur, "SELECT pg_current_wal_lsn()") + tbl.append([i, after_timestamp, after_lsn]) + time.sleep(0.005) + + # Execute one more transaction with synchronous_commit enabled, to flush + # all the previous transactions + cur.execute("SET synchronous_commit=on") + cur.execute("INSERT INTO foo VALUES (-1)") + + # Wait until WAL is received by pageserver + last_flush_lsn = wait_for_last_flush_lsn( + env, endpoint_main, env.initial_tenant, new_timeline_id + ) + + with env.pageserver.http_client() as client: + # Check edge cases: lsn larger than the last flush lsn + probe_lsn = Lsn(int(last_flush_lsn) * 20 + 80_000) + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + probe_lsn, + ) + + # lsn of zero + try: + probe_lsn = Lsn(0) + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + probe_lsn, + ) + # There should always be an error here. + raise RuntimeError("there should have been an 'Invalid LSN' error") + except PageserverApiException as error: + assert error.status_code == 500 + assert str(error) == "Invalid LSN" + env.pageserver.allowed_errors.append(".*Invalid LSN.*") + + # small lsn before initdb_lsn + try: + probe_lsn = Lsn(64) + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + probe_lsn, + ) + # There should always be an error here. + raise RuntimeError("there should have been an 'could not find data for key' error") + except PageserverApiException as error: + assert error.status_code == 500 + assert str(error).startswith("could not find data for key") + env.pageserver.allowed_errors.append(".*could not find data for key.*") + + # Probe a bunch of timestamps in the valid range + step_size = 100 + for i in range(step_size, len(tbl), step_size): + after_timestamp = tbl[i][1] + after_lsn = tbl[i][2] + result = client.timeline_get_timestamp_of_lsn( + env.initial_tenant, + new_timeline_id, + after_lsn, + ) + log.info("result: %s, after_ts: %s", result, after_timestamp) + + # TODO use fromisoformat once we have Python 3.11+ + # which has https://github.com/python/cpython/pull/92177 + timestamp = datetime.strptime(result, "%Y-%m-%dT%H:%M:%S.%f000Z").replace( + tzinfo=timezone.utc + ) + assert timestamp < after_timestamp, "after_timestamp after timestamp" + if i > 1: + before_timestamp = tbl[i - step_size][1] + assert timestamp >= before_timestamp, "before_timestamp before timestamp" diff --git a/test_runner/regress/test_ondemand_download.py b/test_runner/regress/test_ondemand_download.py index a38a517100..33de7445a4 100644 --- a/test_runner/regress/test_ondemand_download.py +++ b/test_runner/regress/test_ondemand_download.py @@ -306,12 +306,10 @@ def test_ondemand_download_timetravel( # # Ensure that the `download_remote_layers` API works # -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_download_remote_layers_api( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) ##### First start, insert data and upload it to the remote storage env = neon_env_builder.init_start( @@ -465,14 +463,11 @@ def test_download_remote_layers_api( assert query_scalar(cur, "select count(*) from testtab") == table_len -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.MOCK_S3]) -def test_compaction_downloads_on_demand_without_image_creation( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): +def test_compaction_downloads_on_demand_without_image_creation(neon_env_builder: NeonEnvBuilder): """ Create a few layers, then evict, then make sure compaction runs successfully. """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) conf = { # Disable background GC & compaction @@ -547,17 +542,14 @@ def test_compaction_downloads_on_demand_without_image_creation( assert post_compact[1] >= 3, "should had downloaded the three layers" -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.MOCK_S3]) -def test_compaction_downloads_on_demand_with_image_creation( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): +def test_compaction_downloads_on_demand_with_image_creation(neon_env_builder: NeonEnvBuilder): """ Create layers, compact with high image_creation_threshold, then run final compaction with all layers evicted. Due to current implementation, this will make image creation on-demand download layers, but we cannot really directly test for it. """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.MOCK_S3) conf = { # Disable background GC & compaction @@ -645,17 +637,14 @@ def test_compaction_downloads_on_demand_with_image_creation( assert dict(kinds_after) == {"Delta": 4, "Image": 1} -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_ondemand_download_failure_to_replace( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): +def test_ondemand_download_failure_to_replace(neon_env_builder: NeonEnvBuilder): """ Make sure that we fail on being unable to replace a RemoteLayer instead of for example livelocking. See: https://github.com/neondatabase/neon/issues/3533 """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) # disable gc and compaction via default tenant config because config is lost while detaching # so that compaction will not be the one to download the layer but the http handler is diff --git a/test_runner/regress/test_pageserver_api.py b/test_runner/regress/test_pageserver_api.py index f5bcfd52f9..2d83788193 100644 --- a/test_runner/regress/test_pageserver_api.py +++ b/test_runner/regress/test_pageserver_api.py @@ -157,6 +157,8 @@ def test_pageserver_http_get_wal_receiver_success(neon_simple_env: NeonEnv): tenant_id, timeline_id = env.neon_cli.create_tenant() endpoint = env.endpoints.create_start(DEFAULT_BRANCH_NAME, tenant_id=tenant_id) + # insert something to force sk -> ps message + endpoint.safe_psql("CREATE TABLE t(key int primary key, value text)") # Wait to make sure that we get a latest WAL receiver data. # We need to wait here because it's possible that we don't have access to # the latest WAL yet, when the `timeline_detail` API is first called. @@ -168,7 +170,7 @@ def test_pageserver_http_get_wal_receiver_success(neon_simple_env: NeonEnv): ) # Make a DB modification then expect getting a new WAL receiver's data. - endpoint.safe_psql("CREATE TABLE t(key int primary key, value text)") + endpoint.safe_psql("INSERT INTO t VALUES (1, 'hey')") wait_until( number_of_iterations=5, interval=1, diff --git a/test_runner/regress/test_pageserver_generations.py b/test_runner/regress/test_pageserver_generations.py index 6dc8582755..994d09665c 100644 --- a/test_runner/regress/test_pageserver_generations.py +++ b/test_runner/regress/test_pageserver_generations.py @@ -492,7 +492,7 @@ def evict_all_layers(env: NeonEnv, tenant_id: TenantId, timeline_id: TimelineId) ) client = env.pageserver.http_client() for layer in initial_local_layers: - if "ephemeral" in layer.name: + if "ephemeral" in layer.name or "temp" in layer.name: continue log.info(f"Evicting layer {tenant_id}/{timeline_id} {layer.name}") client.evict_layer(tenant_id=tenant_id, timeline_id=timeline_id, layer_name=layer.name) diff --git a/test_runner/regress/test_pageserver_restarts_under_workload.py b/test_runner/regress/test_pageserver_restarts_under_workload.py index 71058268a6..65569f3bac 100644 --- a/test_runner/regress/test_pageserver_restarts_under_workload.py +++ b/test_runner/regress/test_pageserver_restarts_under_workload.py @@ -17,8 +17,6 @@ def test_pageserver_restarts_under_worload(neon_simple_env: NeonEnv, pg_bin: PgB n_restarts = 10 scale = 10 - env.pageserver.allowed_errors.append(".*query handler.*failed.*Shutting down") - def run_pgbench(connstr: str): log.info(f"Start a pgbench workload on pg {connstr}") pg_bin.run_capture(["pgbench", "-i", f"-s{scale}", connstr]) diff --git a/test_runner/regress/test_proxy.py b/test_runner/regress/test_proxy.py index f57b15c9b9..a33b29549c 100644 --- a/test_runner/regress/test_proxy.py +++ b/test_runner/regress/test_proxy.py @@ -7,6 +7,8 @@ import pytest import requests from fixtures.neon_fixtures import PSQL, NeonProxy, VanillaPostgres +GET_CONNECTION_PID_QUERY = "SELECT pid FROM pg_stat_activity WHERE state = 'active'" + def test_proxy_select_1(static_proxy: NeonProxy): """ @@ -353,7 +355,7 @@ def test_sql_over_http_pool(static_proxy: NeonProxy): def get_pid(status: int, pw: str) -> Any: return static_proxy.http_query( - "SELECT pid FROM pg_stat_activity WHERE state = 'active'", + GET_CONNECTION_PID_QUERY, [], user="http_auth", password=pw, @@ -387,7 +389,6 @@ def test_sql_over_http_pool(static_proxy: NeonProxy): # Beginning a transaction should not impact the next query, # which might come from a completely different client. -@pytest.mark.xfail(reason="not implemented") def test_http_pool_begin(static_proxy: NeonProxy): static_proxy.safe_psql("create user http_auth with password 'http' superuser") @@ -403,3 +404,21 @@ def test_http_pool_begin(static_proxy: NeonProxy): query(200, "BEGIN;") query(400, "garbage-lol(&(&(&(&") # Intentional error to break the transaction query(200, "SELECT 1;") # Query that should succeed regardless of the transaction + + +def test_sql_over_http_pool_idle(static_proxy: NeonProxy): + static_proxy.safe_psql("create user http_auth2 with password 'http' superuser") + + def query(status: int, query: str) -> Any: + return static_proxy.http_query( + query, + [], + user="http_auth2", + password="http", + expected_code=status, + ) + + pid1 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] + query(200, "BEGIN") + pid2 = query(200, GET_CONNECTION_PID_QUERY)["rows"][0]["pid"] + assert pid1 != pid2 diff --git a/test_runner/regress/test_remote_storage.py b/test_runner/regress/test_remote_storage.py index f316b42d1c..ecd02c4c16 100644 --- a/test_runner/regress/test_remote_storage.py +++ b/test_runner/regress/test_remote_storage.py @@ -216,12 +216,10 @@ def test_remote_storage_backup_and_restore( # - Disable failpoints # - Wait for all uploads to finish # - Verify that remote is consistent and up-to-date (=all retries were done and succeeded) -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_remote_storage_upload_queue_retries( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() @@ -369,12 +367,10 @@ def test_remote_storage_upload_queue_retries( assert query_scalar(cur, "SELECT COUNT(*) FROM foo WHERE val = 'd'") == 20000 -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_remote_timeline_client_calls_started_metric( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) # thinking about using a shared environment? the test assumes that global # metrics are for single tenant. @@ -509,12 +505,10 @@ def test_remote_timeline_client_calls_started_metric( # Test that we correctly handle timeline with layers stuck in upload queue -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_timeline_deletion_with_files_stuck_in_upload_queue( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start( initial_tenant_conf={ @@ -624,12 +618,8 @@ def test_timeline_deletion_with_files_stuck_in_upload_queue( # Branches off a root branch, but does not write anything to the new branch, so it has a metadata file only. # Ensures that such branch is still persisted on the remote storage, and can be restored during tenant (re)attach. -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_empty_branch_remote_storage_upload( - neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_empty_branch_remote_storage_upload(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() client = env.pageserver.http_client() @@ -665,11 +655,7 @@ def test_empty_branch_remote_storage_upload( ), f"Expected to have same timelines after reattach, but got {timelines_after_detach}" -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_empty_branch_remote_storage_upload_on_restart( - neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, -): +def test_empty_branch_remote_storage_upload_on_restart(neon_env_builder: NeonEnvBuilder): """ Branches off a root branch, but does not write anything to the new branch, so it has a metadata file only. @@ -678,7 +664,7 @@ def test_empty_branch_remote_storage_upload_on_restart( — the upload should be scheduled by load, and create_timeline should await for it even though it gets 409 Conflict. """ - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() client = env.pageserver.http_client() @@ -731,6 +717,10 @@ def test_empty_branch_remote_storage_upload_on_restart( def create_in_background(): barrier.wait() try: + # retrying this kind of query makes no sense in real life as we do + # not lock in the lsn. with the immediate stop, we could in real + # life revert back the ancestor in startup, but most likely the lsn + # would still be branchable. client.timeline_create( tenant_id=env.initial_tenant, ancestor_timeline_id=env.initial_timeline, @@ -751,13 +741,13 @@ def test_empty_branch_remote_storage_upload_on_restart( assert not new_branch_on_remote_storage.exists(), "failpoint should had stopped uploading" client.configure_failpoints(("before-upload-index", "off")) - conflict = q.get() + exception = q.get() - assert conflict, "create_timeline should not have succeeded" assert ( - conflict.status_code == 409 - ), "timeline was created before restart, and uploads scheduled during initial load, so we expect 409 conflict" + exception is None + ), "create_timeline should have succeeded, because we deleted unuploaded local state" + # this is because creating a timeline always awaits for the uploads to complete assert_nothing_to_upload(client, env.initial_tenant, new_branch_timeline_id) assert ( @@ -767,15 +757,13 @@ def test_empty_branch_remote_storage_upload_on_restart( create_thread.join() -# Regression test for a race condition where files are compactified before the upload, +# Regression test for a race condition where L0 layers are compacted before the upload, # resulting in the uploading complaining about the file not being found # https://github.com/neondatabase/neon/issues/4526 -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_compaction_delete_before_upload( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start( initial_tenant_conf={ diff --git a/test_runner/regress/test_tenant_detach.py b/test_runner/regress/test_tenant_detach.py index 519af1cbde..e92a906fab 100644 --- a/test_runner/regress/test_tenant_detach.py +++ b/test_runner/regress/test_tenant_detach.py @@ -18,7 +18,10 @@ from fixtures.pageserver.utils import ( wait_for_upload, wait_until_tenant_state, ) -from fixtures.remote_storage import RemoteStorageKind, available_remote_storages +from fixtures.remote_storage import ( + RemoteStorageKind, + available_remote_storages, +) from fixtures.types import Lsn, TenantId, TimelineId from fixtures.utils import query_scalar, wait_until from prometheus_client.samples import Sample @@ -585,11 +588,8 @@ def test_ignored_tenant_reattach(neon_env_builder: NeonEnvBuilder): # * `load` the same tenant # * ensure that it's status is `Active` # * check that timeline data is restored -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_ignored_tenant_download_missing_layers( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_ignored_tenant_download_missing_layers(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() endpoint = env.endpoints.create_start("main") @@ -648,11 +648,8 @@ def test_ignored_tenant_download_missing_layers( # * removes its `metadata` file locally # * `load` the same tenant # * ensure that it's status is `Broken` -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_ignored_tenant_stays_broken_without_metadata( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_ignored_tenant_stays_broken_without_metadata(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() env.endpoints.create_start("main") @@ -689,11 +686,8 @@ def test_ignored_tenant_stays_broken_without_metadata( # Tests that attach is never working on a tenant, ignored or not, as long as it's not absent locally # Similarly, tests that it's not possible to schedule a `load` for tenat that's not ignored. -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_load_attach_negatives( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_load_attach_negatives(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() env.endpoints.create_start("main") @@ -730,12 +724,10 @@ def test_load_attach_negatives( pageserver_http.tenant_attach(tenant_id) -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_ignore_while_attaching( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() pageserver_http = env.pageserver.http_client() @@ -752,9 +744,6 @@ def test_ignore_while_attaching( env.pageserver.allowed_errors.append( f".*Tenant {tenant_id} will not become active\\. Current state: Stopping.*" ) - # An endpoint is starting up concurrently with our detach, it can - # experience RPC failure due to shutdown. - env.pageserver.allowed_errors.append(".*query handler.*failed.*Shutting down") data_id = 1 data_secret = "very secret secret" @@ -834,12 +823,10 @@ def ensure_test_data(data_id: int, data: str, endpoint: Endpoint): ), "Should have timeline data back" -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) def test_metrics_while_ignoring_broken_tenant_and_reloading( neon_env_builder: NeonEnvBuilder, - remote_storage_kind: RemoteStorageKind, ): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) env = neon_env_builder.init_start() diff --git a/test_runner/regress/test_tenants_with_remote_storage.py b/test_runner/regress/test_tenants_with_remote_storage.py index d42e566c36..30a36d0eca 100644 --- a/test_runner/regress/test_tenants_with_remote_storage.py +++ b/test_runner/regress/test_tenants_with_remote_storage.py @@ -108,11 +108,8 @@ def test_tenants_many(neon_env_builder: NeonEnvBuilder, remote_storage_kind: Rem wait_for_upload(pageserver_http, tenant_id, timeline_id, current_lsn) -@pytest.mark.parametrize("remote_storage_kind", [RemoteStorageKind.LOCAL_FS]) -def test_tenants_attached_after_download( - neon_env_builder: NeonEnvBuilder, remote_storage_kind: RemoteStorageKind -): - neon_env_builder.enable_pageserver_remote_storage(remote_storage_kind) +def test_tenants_attached_after_download(neon_env_builder: NeonEnvBuilder): + neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS) data_id = 1 data_secret = "very secret secret" diff --git a/test_runner/regress/test_wal_acceptor.py b/test_runner/regress/test_wal_acceptor.py index 631798d643..05c60eb102 100644 --- a/test_runner/regress/test_wal_acceptor.py +++ b/test_runner/regress/test_wal_acceptor.py @@ -1,3 +1,4 @@ +import filecmp import os import pathlib import random @@ -11,7 +12,7 @@ from contextlib import closing from dataclasses import dataclass, field from functools import partial from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional import psycopg2 import psycopg2.errors @@ -19,6 +20,7 @@ import psycopg2.extras import pytest from fixtures.broker import NeonBroker from fixtures.log_helper import log +from fixtures.metrics import parse_metrics from fixtures.neon_fixtures import ( Endpoint, NeonEnv, @@ -399,8 +401,11 @@ def test_wal_removal(neon_env_builder: NeonEnvBuilder, auth_enabled: bool): def wait(f, desc, timeout=30, wait_f=None): started_at = time.time() while True: - if f(): - break + try: + if f(): + break + except Exception: + pass elapsed = time.time() - started_at if elapsed > timeout: raise RuntimeError(f"timed out waiting {elapsed:.0f}s for {desc}") @@ -983,6 +988,141 @@ def test_restart_endpoint(neon_env_builder: NeonEnvBuilder): endpoint.start() +# Test that we can create timeline with one safekeeper down and initialize it +# later when some data already had been written. +def test_late_init(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + + sk1 = env.safekeepers[0] + sk1.stop() + + # create and insert smth while safekeeper is down... + env.neon_cli.create_branch("test_late_init") + endpoint = env.endpoints.create_start("test_late_init") + endpoint.safe_psql("create table t(key int, value text)") + endpoint.safe_psql("insert into t select generate_series(1, 1000), 'payload'") + log.info("insert with safekeeper down done") + endpoint.stop() # stop compute + + # stop another safekeeper, and start one which missed timeline creation + sk2 = env.safekeepers[1] + sk2.stop() + sk1.start() + + # insert some more + endpoint = env.endpoints.create_start("test_late_init") + endpoint.safe_psql("insert into t select generate_series(1,100), 'payload'") + + +# is timeline flush_lsn equal on provided safekeepers? +def is_flush_lsn_aligned(sk1_http_cli, sk2_http_cli, tenant_id, timeline_id): + status1 = sk1_http_cli.timeline_status(tenant_id, timeline_id) + status2 = sk2_http_cli.timeline_status(tenant_id, timeline_id) + log.info( + f"waiting for flush_lsn alignment, sk1.flush_lsn={status1.flush_lsn}, sk2.flush_lsn={status2.flush_lsn}" + ) + return status1.flush_lsn == status2.flush_lsn + + +# Test behaviour with one safekeeper down and missing a lot of WAL. Namely, that +# 1) walproposer can't recover node if it misses WAL written by previous computes, but +# still starts up and functions normally if two other sks are ok. +# 2) walproposer doesn't keep WAL after some threshold (pg_wal bloat is limited), but functions +# normally if two other sks are ok. +# 3) Lagged safekeeper can still recover by peer recovery. +def test_one_sk_down(neon_env_builder: NeonEnvBuilder): + pass + + +# Smaller version of test_one_sk_down testing peer recovery in isolation: that +# it works without compute at all. +def test_peer_recovery(neon_env_builder: NeonEnvBuilder): + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_peer_recovery") + endpoint = env.endpoints.create_start("test_peer_recovery") + + endpoint.safe_psql("create table t(key int, value text)") + sk1 = env.safekeepers[0] + sk2 = env.safekeepers[1] + sk1_http_cli = sk1.http_client() + sk2_http_cli = sk2.http_client() + # ensure tli gets created on sk1, peer recovery won't do that + wait( + partial(is_flush_lsn_aligned, sk1_http_cli, sk2_http_cli, tenant_id, timeline_id), + "flush_lsn to get aligned", + ) + + sk1 = env.safekeepers[0] + sk1.stop() + + # roughly fills one segment + endpoint.safe_psql("insert into t select generate_series(1,250000), 'payload'") + + endpoint.stop() # stop compute + + # now start safekeeper, but with peer recovery disabled; it should lag for about a segment + sk1.start(extra_opts=["--peer-recovery=false"]) + sk1_tli_status = sk1_http_cli.timeline_status(tenant_id, timeline_id) + sk2_tli_status = sk2_http_cli.timeline_status(tenant_id, timeline_id) + log.info( + f"flush_lsns after insertion: sk1={sk1_tli_status.flush_lsn}, sk2={sk2_tli_status.flush_lsn}" + ) + assert sk2_tli_status.flush_lsn - sk1_tli_status.flush_lsn >= 16 * 1024 * 1024 + + # wait a bit, lsns shouldn't change + # time.sleep(5) + sk1_tli_status = sk1_http_cli.timeline_status(tenant_id, timeline_id) + sk2_tli_status = sk2_http_cli.timeline_status(tenant_id, timeline_id) + log.info( + f"flush_lsns after waiting: sk1={sk1_tli_status.flush_lsn}, sk2={sk2_tli_status.flush_lsn}" + ) + assert sk2_tli_status.flush_lsn - sk1_tli_status.flush_lsn >= 16 * 1024 * 1024 + + # now restart safekeeper with peer recovery enabled and wait for recovery + sk1.stop().start(extra_opts=["--peer-recovery=true"]) + wait( + partial(is_flush_lsn_aligned, sk1_http_cli, sk2_http_cli, tenant_id, timeline_id), + "flush_lsn to get aligned", + ) + + # check that WALs are identic after recovery + segs = sk1.list_segments(tenant_id, timeline_id) + log.info(f"segs are {segs}") + + (_, mismatch, not_regular) = filecmp.cmpfiles( + sk1.timeline_dir(tenant_id, timeline_id), + sk2.timeline_dir(tenant_id, timeline_id), + segs, + shallow=False, + ) + log.info( + f"filecmp result mismatch and not regular files:\n\t mismatch={mismatch}\n\t not_regular={not_regular}" + ) + + for f in mismatch: + f1 = os.path.join(sk1.timeline_dir(tenant_id, timeline_id), f) + f2 = os.path.join(sk2.timeline_dir(tenant_id, timeline_id), f) + stdout_filename = "{}.filediff".format(f2) + + with open(stdout_filename, "w") as stdout_f: + subprocess.run("xxd {} > {}.hex ".format(f1, f1), shell=True) + subprocess.run("xxd {} > {}.hex ".format(f2, f2), shell=True) + + cmd = "diff {}.hex {}.hex".format(f1, f2) + subprocess.run([cmd], stdout=stdout_f, shell=True) + + assert (mismatch, not_regular) == ([], []) + + # stop one of safekeepers which weren't recovering and insert a bit more to check we can commit + env.safekeepers[2].stop() + endpoint = env.endpoints.create_start("test_peer_recovery") + endpoint.safe_psql("insert into t select generate_series(1,100), 'payload'") + + class SafekeeperEnv: def __init__( self, @@ -1477,3 +1617,72 @@ def test_pull_timeline(neon_env_builder: NeonEnvBuilder): execute_payload(endpoint) show_statuses(env.safekeepers, tenant_id, timeline_id) + + +# In this test we check for excessive START_REPLICATION and START_WAL_PUSH queries +# when compute is active, but there are no writes to the timeline. In that case +# pageserver should maintain a single connection to safekeeper and don't attempt +# to reconnect extra times. +# +# The only way to verify this without manipulating time is to sleep for a while. +# In this test we sleep for 60 seconds, so this test takes at least 1 minute to run. +# This is longer than most other tests, we run it only for v16 to save CI resources. +def test_idle_reconnections(neon_env_builder: NeonEnvBuilder): + if os.environ.get("PYTEST_CURRENT_TEST", "").find("[debug-pg16]") == -1: + pytest.skip("run only on debug postgres v16 to save CI resources") + + neon_env_builder.num_safekeepers = 3 + env = neon_env_builder.init_start() + + tenant_id = env.initial_tenant + timeline_id = env.neon_cli.create_branch("test_sk_auth_restart_endpoint") + + def collect_stats() -> Dict[str, float]: + # we need to collect safekeeper_pg_queries_received_total metric from all safekeepers + sk_metrics = [ + parse_metrics(sk.http_client().get_metrics_str(), f"safekeeper_{sk.id}") + for sk in env.safekeepers + ] + + total: Dict[str, float] = {} + + for sk in sk_metrics: + queries_received = sk.query_all("safekeeper_pg_queries_received_total") + log.info(f"{sk.name} queries received: {queries_received}") + for sample in queries_received: + total[sample.labels["query"]] = total.get(sample.labels["query"], 0) + sample.value + + log.info(f"Total queries received: {total}") + + # in the perfect world, we should see only one START_REPLICATION query, + # here we check for 5 to prevent flakiness + assert total.get("START_REPLICATION", 0) <= 5 + + # in the perfect world, we should see ~6 START_WAL_PUSH queries, + # here we check for 15 to prevent flakiness + assert total.get("START_WAL_PUSH", 0) <= 15 + + return total + + collect_stats() + + endpoint = env.endpoints.create_start("test_sk_auth_restart_endpoint") + # just write something to the timeline + endpoint.safe_psql("create table t(i int)") + collect_stats() + + # sleep a bit + time.sleep(30) + + # force checkpoint in pageserver to advance remote_consistent_lsn + wait_lsn_force_checkpoint(tenant_id, timeline_id, endpoint, env.pageserver) + + collect_stats() + + time.sleep(30) + + final_stats = collect_stats() + # pageserver should connect to safekeepers at least once + assert final_stats.get("START_REPLICATION", 0) >= 1 + # walproposer should connect to each safekeeper at least once + assert final_stats.get("START_WAL_PUSH", 0) >= 3 diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 5d5cfee127..6669a672ee 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 5d5cfee12783f0989a9c9fe13bb40b5585812568 +Subproject commit 6669a672ee14ab2c09d44c4552f9a13fad3afc10 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 74cfe3e681..ab67ab9635 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 74cfe3e681836747a31fdbd47bdd14b3d81b0772 +Subproject commit ab67ab96355d61e9d0218630be4aa7db53bf83e7 diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index 389ce36b4b..550ffa6495 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit 389ce36b4b3da7aa654a25e1b3f10b641319a87f +Subproject commit 550ffa6495a5dc62fccc3a8b449386633758680b diff --git a/vendor/revisions.json b/vendor/revisions.json index d08cb25f43..012fb14035 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,5 +1,5 @@ { - "postgres-v16": "389ce36b4b3da7aa654a25e1b3f10b641319a87f", - "postgres-v15": "74cfe3e681836747a31fdbd47bdd14b3d81b0772", - "postgres-v14": "5d5cfee12783f0989a9c9fe13bb40b5585812568" + "postgres-v16": "550ffa6495a5dc62fccc3a8b449386633758680b", + "postgres-v15": "ab67ab96355d61e9d0218630be4aa7db53bf83e7", + "postgres-v14": "6669a672ee14ab2c09d44c4552f9a13fad3afc10" } diff --git a/workspace_hack/Cargo.toml b/workspace_hack/Cargo.toml index 4b250822e5..e2a65ad150 100644 --- a/workspace_hack/Cargo.toml +++ b/workspace_hack/Cargo.toml @@ -21,7 +21,7 @@ aws-smithy-http = { version = "0.56", default-features = false, features = ["eve axum = { version = "0.6", features = ["ws"] } base64 = { version = "0.21", features = ["alloc"] } bytes = { version = "1", features = ["serde"] } -chrono = { version = "0.4", default-features = false, features = ["clock", "serde"] } +chrono = { version = "0.4", default-features = false, features = ["clock", "serde", "wasmbind"] } clap = { version = "4", features = ["derive", "string"] } clap_builder = { version = "4", default-features = false, features = ["color", "help", "std", "string", "suggestions", "usage"] } crossbeam-utils = { version = "0.8" } @@ -55,9 +55,8 @@ scopeguard = { version = "1" } serde = { version = "1", features = ["alloc", "derive"] } serde_json = { version = "1", features = ["raw_value"] } smallvec = { version = "1", default-features = false, features = ["write"] } -socket2 = { version = "0.4", default-features = false, features = ["all"] } standback = { version = "0.2", default-features = false, features = ["std"] } -time = { version = "0.3", features = ["macros", "serde-well-known"] } +time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tokio = { version = "1", features = ["fs", "io-std", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "test-util"] } tokio-rustls = { version = "0.24" } tokio-util = { version = "0.7", features = ["codec", "io"] }