From e74e7aac936543e5fcb9d122bedc2d146e41ddb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Mon, 9 Dec 2024 16:50:06 +0100 Subject: [PATCH 01/44] Use updated patched azure SDK crates (#10036) For a while already, we've been unable to update the Azure SDK crates due to Azure adopting use of a non-tokio async runtime, see #7545. The effort to upstream the fix got stalled, and I think it's better to switch to a patched version of the SDK that is up to date. Now we have a fork of the SDK under the neondatabase github org, to which I have applied Conrad's rebased patches to: https://github.com/neondatabase/azure-sdk-for-rust/tree/neon . The existence of a fork will also help with shipping bulk delete support before it's upstreamed (#7931). Also, in related news, the Azure SDK has gotten a rift in development, where the main branch pertains to a future, to-be-officially-blessed release of the SDK, and the older versions, which we are currently using, are on the `legacy` branch. Upstream doesn't really want patches for the `legacy` branch any more, they want to focus on the `main` efforts. However, even then, the `legacy` branch is still newer than what we are having right now, so let's switch to `legacy` for now. Depending on how long it takes, we can switch to the official version of the SDK once it's released or switch to the upstream `main` branch if there is changes we want before that. As a nice side effect of this PR, we now use reqwest 0.12 everywhere, dropping the dependency on version 0.11. Fixes #7545 --- Cargo.lock | 151 +++++++------------------- Cargo.toml | 10 +- libs/remote_storage/src/azure_blob.rs | 8 +- 3 files changed, 47 insertions(+), 122 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e9004748ae..e2d5e03613 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -770,77 +770,74 @@ dependencies = [ [[package]] name = "azure_core" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70fd680c0d0424a518229b1150922f92653ba2ac933aa000abc8bf1ca08105f7" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "async-trait", - "base64 0.21.1", + "base64 0.22.1", "bytes", "dyn-clone", "futures", "getrandom 0.2.11", "hmac", "http-types", - "log", "once_cell", "paste", "pin-project", "quick-xml 0.31.0", "rand 0.8.5", - "reqwest 0.11.19", + "reqwest", "rustc_version", "serde", "serde_json", "sha2", "time", + "tracing", "url", "uuid", ] [[package]] name = "azure_identity" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d2060f5b2e1c664026ca4edd561306c473be887c1f7a81f10bf06f9b71c63f" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "async-lock", "async-trait", "azure_core", "futures", - "log", "oauth2", "pin-project", "serde", "time", + "tokio", + "tracing", "url", "uuid", ] [[package]] name = "azure_storage" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15d3da73bfa09350e1bd6ae2a260806fcf90048c7e78cd2d8f88be60b19a7266" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "RustyXML", "async-lock", "async-trait", "azure_core", "bytes", - "log", "serde", "serde_derive", "time", + "tracing", "url", "uuid", ] [[package]] name = "azure_storage_blobs" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "149c21834a4105d761e3dd33d91c2a3064acc05a3c978848ea8089102ae45c94" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "RustyXML", "azure_core", @@ -848,20 +845,19 @@ dependencies = [ "azure_svc_blobstorage", "bytes", "futures", - "log", "serde", "serde_derive", "serde_json", "time", + "tracing", "url", "uuid", ] [[package]] name = "azure_svc_blobstorage" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88c888b7bf522d5405218b8613bf0fae7ddaae6ef3bf4ad42ae005993c96ab8b" +version = "0.21.0" +source = "git+https://github.com/neondatabase/azure-sdk-for-rust.git?branch=neon#66e77bdd87bf87e773acf3b0c84b532c1124367d" dependencies = [ "azure_core", "bytes", @@ -1287,7 +1283,7 @@ dependencies = [ "prometheus", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "rlimit", "rust-ini", "serde", @@ -1395,7 +1391,7 @@ dependencies = [ "postgres_backend", "postgres_connection", "regex", - "reqwest 0.12.4", + "reqwest", "safekeeper_api", "scopeguard", "serde", @@ -1904,15 +1900,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "encoding_rs" -version = "0.8.32" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" -dependencies = [ - "cfg-if", -] - [[package]] name = "enum-map" version = "2.5.0" @@ -3709,7 +3696,7 @@ dependencies = [ "bytes", "http 1.1.0", "opentelemetry", - "reqwest 0.12.4", + "reqwest", ] [[package]] @@ -3726,7 +3713,7 @@ dependencies = [ "opentelemetry-proto", "opentelemetry_sdk", "prost", - "reqwest 0.12.4", + "reqwest", "thiserror", ] @@ -3935,7 +3922,7 @@ dependencies = [ "range-set-blaze", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "rpds", "scopeguard", "send-future", @@ -3988,7 +3975,7 @@ dependencies = [ "postgres_ffi", "rand 0.8.5", "remote_storage", - "reqwest 0.12.4", + "reqwest", "serde", "serde_json", "serde_with", @@ -4008,7 +3995,7 @@ dependencies = [ "futures", "pageserver_api", "postgres", - "reqwest 0.12.4", + "reqwest", "serde", "thiserror", "tokio", @@ -4725,7 +4712,7 @@ dependencies = [ "redis", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "reqwest-middleware", "reqwest-retry", "reqwest-tracing", @@ -5088,47 +5075,6 @@ dependencies = [ "utils", ] -[[package]] -name = "reqwest" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20b9b67e2ca7dd9e9f9285b759de30ff538aab981abaaf7bc9bd90b84a0126c3" -dependencies = [ - "base64 0.21.1", - "bytes", - "encoding_rs", - "futures-core", - "futures-util", - "h2 0.3.26", - "http 0.2.9", - "http-body 0.4.5", - "hyper 0.14.30", - "hyper-rustls 0.24.0", - "ipnet", - "js-sys", - "log", - "mime", - "once_cell", - "percent-encoding", - "pin-project-lite", - "rustls 0.21.12", - "rustls-pemfile 1.0.2", - "serde", - "serde_json", - "serde_urlencoded", - "tokio", - "tokio-rustls 0.24.0", - "tokio-util", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams 0.3.0", - "web-sys", - "webpki-roots 0.25.2", - "winreg 0.50.0", -] - [[package]] name = "reqwest" version = "0.12.4" @@ -5168,10 +5114,10 @@ dependencies = [ "url", "wasm-bindgen", "wasm-bindgen-futures", - "wasm-streams 0.4.0", + "wasm-streams", "web-sys", "webpki-roots 0.26.1", - "winreg 0.52.0", + "winreg", ] [[package]] @@ -5183,7 +5129,7 @@ dependencies = [ "anyhow", "async-trait", "http 1.1.0", - "reqwest 0.12.4", + "reqwest", "serde", "thiserror", "tower-service", @@ -5202,7 +5148,7 @@ dependencies = [ "http 1.1.0", "hyper 1.4.1", "parking_lot 0.11.2", - "reqwest 0.12.4", + "reqwest", "reqwest-middleware", "retry-policies", "thiserror", @@ -5223,7 +5169,7 @@ dependencies = [ "http 1.1.0", "matchit 0.8.2", "opentelemetry", - "reqwest 0.12.4", + "reqwest", "reqwest-middleware", "tracing", "tracing-opentelemetry", @@ -5587,7 +5533,7 @@ dependencies = [ "rand 0.8.5", "regex", "remote_storage", - "reqwest 0.12.4", + "reqwest", "safekeeper_api", "scopeguard", "sd-notify", @@ -5743,7 +5689,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00421ed8fa0c995f07cde48ba6c89e80f2b312f74ff637326f392fbfd23abe02" dependencies = [ "httpdate", - "reqwest 0.12.4", + "reqwest", "rustls 0.21.12", "sentry-backtrace", "sentry-contexts", @@ -6198,7 +6144,7 @@ dependencies = [ "postgres_connection", "r2d2", "rand 0.8.5", - "reqwest 0.12.4", + "reqwest", "routerify", "scopeguard", "serde", @@ -6218,7 +6164,7 @@ name = "storage_controller_client" version = "0.1.0" dependencies = [ "pageserver_client", - "reqwest 0.12.4", + "reqwest", "serde", "workspace_hack", ] @@ -6245,7 +6191,7 @@ dependencies = [ "pageserver_api", "postgres_ffi", "remote_storage", - "reqwest 0.12.4", + "reqwest", "rustls 0.23.18", "rustls-native-certs 0.8.0", "serde", @@ -6274,7 +6220,7 @@ dependencies = [ "humantime", "pageserver_api", "pageserver_client", - "reqwest 0.12.4", + "reqwest", "serde_json", "storage_controller_client", "tokio", @@ -7514,19 +7460,6 @@ version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" -[[package]] -name = "wasm-streams" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - [[package]] name = "wasm-streams" version = "0.4.0" @@ -7792,16 +7725,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "winreg" -version = "0.50.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1" -dependencies = [ - "cfg-if", - "windows-sys 0.48.0", -] - [[package]] name = "winreg" version = "0.52.0" @@ -7876,7 +7799,7 @@ dependencies = [ "regex", "regex-automata 0.4.3", "regex-syntax 0.8.2", - "reqwest 0.12.4", + "reqwest", "rustls 0.23.18", "scopeguard", "serde", diff --git a/Cargo.toml b/Cargo.toml index a35823e0c2..0654c25a3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,10 +51,6 @@ anyhow = { version = "1.0", features = ["backtrace"] } arc-swap = "1.6" async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] } atomic-take = "1.1.0" -azure_core = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } -azure_identity = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] } -azure_storage = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] } -azure_storage_blobs = { version = "0.19", default-features = false, features = ["enable_reqwest_rustls"] } flate2 = "1.0.26" async-stream = "0.3" async-trait = "0.1" @@ -216,6 +212,12 @@ postgres-protocol = { git = "https://github.com/neondatabase/rust-postgres.git", postgres-types = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" } tokio-postgres = { git = "https://github.com/neondatabase/rust-postgres.git", branch = "neon" } +## Azure SDK crates +azure_core = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls", "hmac_rust"] } +azure_identity = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } +azure_storage = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } +azure_storage_blobs = { git = "https://github.com/neondatabase/azure-sdk-for-rust.git", branch = "neon", default-features = false, features = ["enable_reqwest_rustls"] } + ## Local libraries compute_api = { version = "0.1", path = "./libs/compute_api/" } consumption_metrics = { version = "0.1", path = "./libs/consumption_metrics/" } diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index 8d1962fa29..a1d7569140 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -8,15 +8,14 @@ use std::io; use std::num::NonZeroU32; use std::pin::Pin; use std::str::FromStr; -use std::sync::Arc; use std::time::Duration; use std::time::SystemTime; use super::REMOTE_STORAGE_PREFIX_SEPARATOR; +use anyhow::Context; use anyhow::Result; use azure_core::request_options::{IfMatchCondition, MaxResults, Metadata, Range}; use azure_core::{Continuable, RetryOptions}; -use azure_identity::DefaultAzureCredential; use azure_storage::StorageCredentials; use azure_storage_blobs::blob::CopyStatus; use azure_storage_blobs::prelude::ClientBuilder; @@ -76,8 +75,9 @@ impl AzureBlobStorage { let credentials = if let Ok(access_key) = env::var("AZURE_STORAGE_ACCESS_KEY") { StorageCredentials::access_key(account.clone(), access_key) } else { - let token_credential = DefaultAzureCredential::default(); - StorageCredentials::token_credential(Arc::new(token_credential)) + let token_credential = azure_identity::create_default_credential() + .context("trying to obtain Azure default credentials")?; + StorageCredentials::token_credential(token_credential) }; // we have an outer retry From 92273b6d5e4133e637d7b5341659007acd0933fb Mon Sep 17 00:00:00 2001 From: a-masterov <72613290+a-masterov@users.noreply.github.com> Date: Mon, 9 Dec 2024 20:30:39 +0100 Subject: [PATCH 02/44] Enable the pg_regress tests on staging for PG17 (#9978) ## Problem Currently, we run the `pg_regress` tests only for PG16 However, PG17 is a part of Neon and should be tested as well ## Summary of changes Modified the workflow and added a patch for PG17 enabling the `pg_regress` tests. The problem with leftovers was solved by using branches. --- .github/actionlint.yml | 2 + .github/workflows/cloud-regress.yml | 32 +- compute/patches/cloud_regress_pg17.patch | 4047 +++++++++++++++++ test_runner/cloud_regress/README.md | 21 + .../cloud_regress/test_cloud_regress.py | 53 - 5 files changed, 4096 insertions(+), 59 deletions(-) create mode 100644 compute/patches/cloud_regress_pg17.patch create mode 100644 test_runner/cloud_regress/README.md diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 29c4d18f4a..27c8fb3c23 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -21,3 +21,5 @@ config-variables: - SLACK_UPCOMING_RELEASE_CHANNEL_ID - DEV_AWS_OIDC_ROLE_ARN - BENCHMARK_INGEST_TARGET_PROJECTID + - PGREGRESS_PG16_PROJECT_ID + - PGREGRESS_PG17_PROJECT_ID diff --git a/.github/workflows/cloud-regress.yml b/.github/workflows/cloud-regress.yml index 19ebf457b8..57194090cf 100644 --- a/.github/workflows/cloud-regress.yml +++ b/.github/workflows/cloud-regress.yml @@ -23,11 +23,14 @@ jobs: regress: env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install - DEFAULT_PG_VERSION: 16 TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} + strategy: + fail-fast: false + matrix: + pg-version: [16, 17] runs-on: us-east-2 container: @@ -40,9 +43,11 @@ jobs: submodules: true - name: Patch the test + env: + PG_VERSION: ${{matrix.pg-version}} run: | - cd "vendor/postgres-v${DEFAULT_PG_VERSION}" - patch -p1 < "../../compute/patches/cloud_regress_pg${DEFAULT_PG_VERSION}.patch" + cd "vendor/postgres-v${PG_VERSION}" + patch -p1 < "../../compute/patches/cloud_regress_pg${PG_VERSION}.patch" - name: Generate a random password id: pwgen @@ -55,8 +60,9 @@ jobs: - name: Change tests according to the generated password env: DBPASS: ${{ steps.pwgen.outputs.DBPASS }} + PG_VERSION: ${{matrix.pg-version}} run: | - cd vendor/postgres-v"${DEFAULT_PG_VERSION}"/src/test/regress + cd vendor/postgres-v"${PG_VERSION}"/src/test/regress for fname in sql/*.sql expected/*.out; do sed -i.bak s/NEON_PASSWORD_PLACEHOLDER/"'${DBPASS}'"/ "${fname}" done @@ -73,15 +79,29 @@ jobs: path: /tmp/neon/ prefix: latest + - name: Create a new branch + id: create-branch + uses: ./.github/actions/neon-branch-create + with: + api_key: ${{ secrets.NEON_STAGING_API_KEY }} + project_id: ${{ vars[format('PGREGRESS_PG{0}_PROJECT_ID', matrix.pg-version)] }} + - name: Run the regression tests uses: ./.github/actions/run-python-test-set with: build_type: ${{ env.BUILD_TYPE }} test_selection: cloud_regress - pg_version: ${{ env.DEFAULT_PG_VERSION }} + pg_version: ${{matrix.pg-version}} extra_params: -m remote_cluster env: - BENCHMARK_CONNSTR: ${{ secrets.PG_REGRESS_CONNSTR }} + BENCHMARK_CONNSTR: ${{steps.create-branch.outputs.dsn}} + + - name: Delete branch + uses: ./.github/actions/neon-branch-delete + with: + api_key: ${{ secrets.NEON_STAGING_API_KEY }} + project_id: ${{ vars[format('PGREGRESS_PG{0}_PROJECT_ID', matrix.pg-version)] }} + branch_id: ${{steps.create-branch.outputs.branch_id}} - name: Create Allure report id: create-allure-report diff --git a/compute/patches/cloud_regress_pg17.patch b/compute/patches/cloud_regress_pg17.patch new file mode 100644 index 0000000000..cbe84ef54b --- /dev/null +++ b/compute/patches/cloud_regress_pg17.patch @@ -0,0 +1,4047 @@ +diff --git a/src/test/regress/expected/aggregates.out b/src/test/regress/expected/aggregates.out +index 1c1ca7573a..6dfe537647 100644 +--- a/src/test/regress/expected/aggregates.out ++++ b/src/test/regress/expected/aggregates.out +@@ -11,7 +11,8 @@ CREATE TABLE aggtest ( + b float4 + ); + \set filename :abs_srcdir '/data/agg.data' +-COPY aggtest FROM :'filename'; ++\set command '\\copy aggtest FROM ' :'filename'; ++:command + ANALYZE aggtest; + SELECT avg(four) AS avg_1 FROM onek; + avg_1 +diff --git a/src/test/regress/expected/alter_generic.out b/src/test/regress/expected/alter_generic.out +index ae54cb254f..888e2ee8bc 100644 +--- a/src/test/regress/expected/alter_generic.out ++++ b/src/test/regress/expected/alter_generic.out +@@ -15,9 +15,9 @@ DROP ROLE IF EXISTS regress_alter_generic_user1; + DROP ROLE IF EXISTS regress_alter_generic_user2; + DROP ROLE IF EXISTS regress_alter_generic_user3; + RESET client_min_messages; +-CREATE USER regress_alter_generic_user3; +-CREATE USER regress_alter_generic_user2; +-CREATE USER regress_alter_generic_user1 IN ROLE regress_alter_generic_user3; ++CREATE USER regress_alter_generic_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_alter_generic_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_alter_generic_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE regress_alter_generic_user3; + CREATE SCHEMA alt_nsp1; + CREATE SCHEMA alt_nsp2; + GRANT ALL ON SCHEMA alt_nsp1, alt_nsp2 TO public; +@@ -370,7 +370,7 @@ ERROR: STORAGE cannot be specified in ALTER OPERATOR FAMILY + DROP OPERATOR FAMILY alt_opf4 USING btree; + -- Should fail. Need to be SUPERUSER to do ALTER OPERATOR FAMILY .. ADD / DROP + BEGIN TRANSACTION; +-CREATE ROLE regress_alter_generic_user5 NOSUPERUSER; ++CREATE ROLE regress_alter_generic_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER NOSUPERUSER; + CREATE OPERATOR FAMILY alt_opf5 USING btree; + SET ROLE regress_alter_generic_user5; + ALTER OPERATOR FAMILY alt_opf5 USING btree ADD OPERATOR 1 < (int4, int2), FUNCTION 1 btint42cmp(int4, int2); +@@ -382,7 +382,7 @@ ERROR: current transaction is aborted, commands ignored until end of transactio + ROLLBACK; + -- Should fail. Need rights to namespace for ALTER OPERATOR FAMILY .. ADD / DROP + BEGIN TRANSACTION; +-CREATE ROLE regress_alter_generic_user6; ++CREATE ROLE regress_alter_generic_user6 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE SCHEMA alt_nsp6; + REVOKE ALL ON SCHEMA alt_nsp6 FROM regress_alter_generic_user6; + CREATE OPERATOR FAMILY alt_nsp6.alt_opf6 USING btree; +diff --git a/src/test/regress/expected/alter_operator.out b/src/test/regress/expected/alter_operator.out +index 4217ba15de..d28e3ff86e 100644 +--- a/src/test/regress/expected/alter_operator.out ++++ b/src/test/regress/expected/alter_operator.out +@@ -119,7 +119,7 @@ ERROR: operator attribute "Restrict" not recognized + -- + -- Test permission check. Must be owner to ALTER OPERATOR. + -- +-CREATE USER regress_alter_op_user; ++CREATE USER regress_alter_op_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_alter_op_user; + ALTER OPERATOR === (boolean, boolean) SET (RESTRICT = NONE); + ERROR: must be owner of operator === +diff --git a/src/test/regress/expected/alter_table.out b/src/test/regress/expected/alter_table.out +index 6de74a26a9..cd59809194 100644 +--- a/src/test/regress/expected/alter_table.out ++++ b/src/test/regress/expected/alter_table.out +@@ -5,7 +5,7 @@ + SET client_min_messages TO 'warning'; + DROP ROLE IF EXISTS regress_alter_table_user1; + RESET client_min_messages; +-CREATE USER regress_alter_table_user1; ++CREATE USER regress_alter_table_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- + -- add attribute + -- +@@ -3928,8 +3928,8 @@ DROP TABLE fail_part; + ALTER TABLE list_parted ATTACH PARTITION nonexistent FOR VALUES IN (1); + ERROR: relation "nonexistent" does not exist + -- check ownership of the source table +-CREATE ROLE regress_test_me; +-CREATE ROLE regress_test_not_me; ++CREATE ROLE regress_test_me PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_not_me PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE not_owned_by_me (LIKE list_parted); + ALTER TABLE not_owned_by_me OWNER TO regress_test_not_me; + SET SESSION AUTHORIZATION regress_test_me; +diff --git a/src/test/regress/expected/arrays.out b/src/test/regress/expected/arrays.out +index a6d81fd5f9..afefd761cb 100644 +--- a/src/test/regress/expected/arrays.out ++++ b/src/test/regress/expected/arrays.out +@@ -18,7 +18,8 @@ CREATE TABLE array_op_test ( + t text[] + ); + \set filename :abs_srcdir '/data/array.data' +-COPY array_op_test FROM :'filename'; ++\set command '\\copy array_op_test FROM ' :'filename'; ++:command + ANALYZE array_op_test; + -- + -- only the 'e' array is 0-based, the others are 1-based. +diff --git a/src/test/regress/expected/btree_index.out b/src/test/regress/expected/btree_index.out +index 510646cbce..0b3ca1f720 100644 +--- a/src/test/regress/expected/btree_index.out ++++ b/src/test/regress/expected/btree_index.out +@@ -20,13 +20,17 @@ CREATE TABLE bt_f8_heap ( + random int4 + ); + \set filename :abs_srcdir '/data/desc.data' +-COPY bt_i4_heap FROM :'filename'; ++\set command '\\copy bt_i4_heap FROM ' :'filename'; ++:command + \set filename :abs_srcdir '/data/hash.data' +-COPY bt_name_heap FROM :'filename'; ++\set command '\\copy bt_name_heap FROM ' :'filename'; ++:command + \set filename :abs_srcdir '/data/desc.data' +-COPY bt_txt_heap FROM :'filename'; ++\set command '\\copy bt_txt_heap FROM ' :'filename'; ++:command + \set filename :abs_srcdir '/data/hash.data' +-COPY bt_f8_heap FROM :'filename'; ++\set command '\\copy bt_f8_heap FROM ' :'filename'; ++:command + ANALYZE bt_i4_heap; + ANALYZE bt_name_heap; + ANALYZE bt_txt_heap; +diff --git a/src/test/regress/expected/cluster.out b/src/test/regress/expected/cluster.out +index a13aafff0b..f0289b5c06 100644 +--- a/src/test/regress/expected/cluster.out ++++ b/src/test/regress/expected/cluster.out +@@ -308,7 +308,7 @@ WHERE pg_class.oid=indexrelid + -- Verify that toast tables are clusterable + CLUSTER pg_toast.pg_toast_826 USING pg_toast_826_index; + -- Verify that clustering all tables does in fact cluster the right ones +-CREATE USER regress_clstr_user; ++CREATE USER regress_clstr_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE clstr_1 (a INT PRIMARY KEY); + CREATE TABLE clstr_2 (a INT PRIMARY KEY); + CREATE TABLE clstr_3 (a INT PRIMARY KEY); +@@ -499,7 +499,7 @@ DROP TABLE clstrpart; + CREATE TABLE ptnowner(i int unique) PARTITION BY LIST (i); + CREATE INDEX ptnowner_i_idx ON ptnowner(i); + CREATE TABLE ptnowner1 PARTITION OF ptnowner FOR VALUES IN (1); +-CREATE ROLE regress_ptnowner; ++CREATE ROLE regress_ptnowner PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE ptnowner2 PARTITION OF ptnowner FOR VALUES IN (2); + ALTER TABLE ptnowner1 OWNER TO regress_ptnowner; + SET SESSION AUTHORIZATION regress_ptnowner; +diff --git a/src/test/regress/expected/collate.icu.utf8.out b/src/test/regress/expected/collate.icu.utf8.out +index 7a425afe1f..2756fb2d55 100644 +--- a/src/test/regress/expected/collate.icu.utf8.out ++++ b/src/test/regress/expected/collate.icu.utf8.out +@@ -1016,7 +1016,7 @@ select * from collate_test1 where b ilike 'ABC'; + + reset enable_seqscan; + -- schema manipulation commands +-CREATE ROLE regress_test_role; ++CREATE ROLE regress_test_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE SCHEMA test_schema; + -- We need to do this this way to cope with varying names for encodings: + SET client_min_messages TO WARNING; +diff --git a/src/test/regress/expected/constraints.out b/src/test/regress/expected/constraints.out +index cf0b80d616..e8e2a14a4a 100644 +--- a/src/test/regress/expected/constraints.out ++++ b/src/test/regress/expected/constraints.out +@@ -349,7 +349,8 @@ CREATE TABLE COPY_TBL (x INT, y TEXT, z INT, + CONSTRAINT COPY_CON + CHECK (x > 3 AND y <> 'check failed' AND x < 7 )); + \set filename :abs_srcdir '/data/constro.data' +-COPY COPY_TBL FROM :'filename'; ++\set command '\\copy COPY_TBL FROM ' :'filename'; ++:command + SELECT * FROM COPY_TBL; + x | y | z + ---+---------------+--- +@@ -358,7 +359,8 @@ SELECT * FROM COPY_TBL; + (2 rows) + + \set filename :abs_srcdir '/data/constrf.data' +-COPY COPY_TBL FROM :'filename'; ++\set command '\\copy COPY_TBL FROM ' :'filename'; ++:command + ERROR: new row for relation "copy_tbl" violates check constraint "copy_con" + DETAIL: Failing row contains (7, check failed, 6). + CONTEXT: COPY copy_tbl, line 2: "7 check failed 6" +@@ -799,7 +801,7 @@ DETAIL: Key (f1)=(3) conflicts with key (f1)=(3). + DROP TABLE deferred_excl; + -- Comments + -- Setup a low-level role to enforce non-superuser checks. +-CREATE ROLE regress_constraint_comments; ++CREATE ROLE regress_constraint_comments PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_constraint_comments; + CREATE TABLE constraint_comments_tbl (a int CONSTRAINT the_constraint CHECK (a > 0)); + CREATE DOMAIN constraint_comments_dom AS int CONSTRAINT the_constraint CHECK (value > 0); +@@ -819,7 +821,7 @@ COMMENT ON CONSTRAINT the_constraint ON constraint_comments_tbl IS NULL; + COMMENT ON CONSTRAINT the_constraint ON DOMAIN constraint_comments_dom IS NULL; + -- unauthorized user + RESET SESSION AUTHORIZATION; +-CREATE ROLE regress_constraint_comments_noaccess; ++CREATE ROLE regress_constraint_comments_noaccess PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_constraint_comments_noaccess; + COMMENT ON CONSTRAINT the_constraint ON constraint_comments_tbl IS 'no, the comment'; + ERROR: must be owner of relation constraint_comments_tbl +diff --git a/src/test/regress/expected/conversion.out b/src/test/regress/expected/conversion.out +index 442e7aff2b..525f732b03 100644 +--- a/src/test/regress/expected/conversion.out ++++ b/src/test/regress/expected/conversion.out +@@ -8,7 +8,7 @@ + CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, result OUT bytea) + AS :'regresslib', 'test_enc_conversion' + LANGUAGE C STRICT; +-CREATE USER regress_conversion_user WITH NOCREATEDB NOCREATEROLE; ++CREATE USER regress_conversion_user WITH NOCREATEDB NOCREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_conversion_user; + CREATE CONVERSION myconv FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8; + -- +diff --git a/src/test/regress/expected/copy.out b/src/test/regress/expected/copy.out +index 44114089a6..fc1894a0f2 100644 +--- a/src/test/regress/expected/copy.out ++++ b/src/test/regress/expected/copy.out +@@ -15,9 +15,11 @@ insert into copytest values('Unix',E'abc\ndef',2); + insert into copytest values('Mac',E'abc\rdef',3); + insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4); + \set filename :abs_builddir '/results/copytest.csv' +-copy copytest to :'filename' csv; ++\set command '\\copy copytest to ' :'filename' csv; ++:command + create temp table copytest2 (like copytest); +-copy copytest2 from :'filename' csv; ++\set command '\\copy copytest2 from ' :'filename' csv; ++:command + select * from copytest except select * from copytest2; + style | test | filler + -------+------+-------- +@@ -25,8 +27,10 @@ select * from copytest except select * from copytest2; + + truncate copytest2; + --- same test but with an escape char different from quote char +-copy copytest to :'filename' csv quote '''' escape E'\\'; +-copy copytest2 from :'filename' csv quote '''' escape E'\\'; ++\set command '\\copy copytest to ' :'filename' ' csv quote ' '\'\'\'\'' ' escape ' 'E\'' '\\\\\''; ++:command ++\set command '\\copy copytest2 from ' :'filename' ' csv quote ' '\'\'\'\'' ' escape ' 'E\'' '\\\\\''; ++:command + select * from copytest except select * from copytest2; + style | test | filler + -------+------+-------- +@@ -66,13 +70,16 @@ insert into parted_copytest select x,1,'One' from generate_series(1,1000) x; + insert into parted_copytest select x,2,'Two' from generate_series(1001,1010) x; + insert into parted_copytest select x,1,'One' from generate_series(1011,1020) x; + \set filename :abs_builddir '/results/parted_copytest.csv' +-copy (select * from parted_copytest order by a) to :'filename'; ++\set command '\\copy (select * from parted_copytest order by a) to ' :'filename'; ++:command + truncate parted_copytest; +-copy parted_copytest from :'filename'; ++\set command '\\copy parted_copytest from ' :'filename'; ++:command + -- Ensure COPY FREEZE errors for partitioned tables. + begin; + truncate parted_copytest; +-copy parted_copytest from :'filename' (freeze); ++\set command '\\copy parted_copytest from ' :'filename' (freeze); ++:command + ERROR: cannot perform COPY FREEZE on a partitioned table + rollback; + select tableoid::regclass,count(*),sum(a) from parted_copytest +@@ -94,7 +101,8 @@ create trigger part_ins_trig + before insert on parted_copytest_a2 + for each row + execute procedure part_ins_func(); +-copy parted_copytest from :'filename'; ++\set command '\\copy parted_copytest from ' :'filename'; ++:command + select tableoid::regclass,count(*),sum(a) from parted_copytest + group by tableoid order by tableoid::regclass::name; + tableoid | count | sum +@@ -106,7 +114,8 @@ group by tableoid order by tableoid::regclass::name; + truncate table parted_copytest; + create index on parted_copytest (b); + drop trigger part_ins_trig on parted_copytest_a2; +-copy parted_copytest from stdin; ++\set command '\\copy parted_copytest from ' stdin; ++:command + -- Ensure index entries were properly added during the copy. + select * from parted_copytest where b = 1; + a | b | c +@@ -170,9 +179,9 @@ INFO: progress: {"type": "PIPE", "command": "COPY FROM", "relname": "tab_progre + -- Generate COPY FROM report with FILE, with some excluded tuples. + truncate tab_progress_reporting; + \set filename :abs_srcdir '/data/emp.data' +-copy tab_progress_reporting from :'filename' +- where (salary < 2000); +-INFO: progress: {"type": "FILE", "command": "COPY FROM", "relname": "tab_progress_reporting", "has_bytes_total": true, "tuples_excluded": 1, "tuples_processed": 2, "has_bytes_processed": true} ++\set command '\\copy tab_progress_reporting from ' :'filename' 'where (salary < 2000)'; ++:command ++INFO: progress: {"type": "PIPE", "command": "COPY FROM", "relname": "tab_progress_reporting", "has_bytes_total": false, "tuples_excluded": 1, "tuples_processed": 2, "has_bytes_processed": true} + drop trigger check_after_tab_progress_reporting on tab_progress_reporting; + drop function notice_after_tab_progress_reporting(); + drop table tab_progress_reporting; +@@ -281,7 +290,8 @@ CREATE TABLE parted_si_p_odd PARTITION OF parted_si FOR VALUES IN (1); + -- https://postgr.es/m/18130-7a86a7356a75209d%40postgresql.org + -- https://postgr.es/m/257696.1695670946%40sss.pgh.pa.us + \set filename :abs_srcdir '/data/desc.data' +-COPY parted_si(id, data) FROM :'filename'; ++\set command '\\COPY parted_si(id, data) FROM ' :'filename'; ++:command + -- An earlier bug (see commit b1ecb9b3fcf) could end up using a buffer from + -- the wrong partition. This test is *not* guaranteed to trigger that bug, but + -- does so when shared_buffers is small enough. To test if we encountered the +diff --git a/src/test/regress/expected/copy2.out b/src/test/regress/expected/copy2.out +index 695b1b2d63..9c9addead6 100644 +--- a/src/test/regress/expected/copy2.out ++++ b/src/test/regress/expected/copy2.out +@@ -631,8 +631,8 @@ select * from check_con_tbl; + (2 rows) + + -- test with RLS enabled. +-CREATE ROLE regress_rls_copy_user; +-CREATE ROLE regress_rls_copy_user_colperms; ++CREATE ROLE regress_rls_copy_user PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_copy_user_colperms PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE rls_t1 (a int, b int, c int); + COPY rls_t1 (a, b, c) from stdin; + CREATE POLICY p1 ON rls_t1 FOR SELECT USING (a % 2 = 0); +diff --git a/src/test/regress/expected/create_function_sql.out b/src/test/regress/expected/create_function_sql.out +index 50aca5940f..42527142f6 100644 +--- a/src/test/regress/expected/create_function_sql.out ++++ b/src/test/regress/expected/create_function_sql.out +@@ -4,7 +4,7 @@ + -- Assorted tests using SQL-language functions + -- + -- All objects made in this test are in temp_func_test schema +-CREATE USER regress_unpriv_user; ++CREATE USER regress_unpriv_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE SCHEMA temp_func_test; + GRANT ALL ON SCHEMA temp_func_test TO public; + SET search_path TO temp_func_test, public; +diff --git a/src/test/regress/expected/create_index.out b/src/test/regress/expected/create_index.out +index cf6eac5734..3e56ea09d7 100644 +--- a/src/test/regress/expected/create_index.out ++++ b/src/test/regress/expected/create_index.out +@@ -51,7 +51,8 @@ CREATE TABLE fast_emp4000 ( + home_base box + ); + \set filename :abs_srcdir '/data/rect.data' +-COPY slow_emp4000 FROM :'filename'; ++\set command '\\copy slow_emp4000 FROM ' :'filename'; ++:command + INSERT INTO fast_emp4000 SELECT * FROM slow_emp4000; + ANALYZE slow_emp4000; + ANALYZE fast_emp4000; +@@ -655,7 +656,8 @@ CREATE TABLE array_index_op_test ( + t text[] + ); + \set filename :abs_srcdir '/data/array.data' +-COPY array_index_op_test FROM :'filename'; ++\set command '\\copy array_index_op_test FROM ' :'filename'; ++:command + ANALYZE array_index_op_test; + SELECT * FROM array_index_op_test WHERE i = '{NULL}' ORDER BY seqno; + seqno | i | t +@@ -2966,7 +2968,7 @@ END; + -- concurrently + REINDEX SCHEMA CONCURRENTLY schema_to_reindex; + -- Failure for unauthorized user +-CREATE ROLE regress_reindexuser NOLOGIN; ++CREATE ROLE regress_reindexuser NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION ROLE regress_reindexuser; + REINDEX SCHEMA schema_to_reindex; + ERROR: must be owner of schema schema_to_reindex +diff --git a/src/test/regress/expected/create_procedure.out b/src/test/regress/expected/create_procedure.out +index 2177ba3509..ae3ca94d00 100644 +--- a/src/test/regress/expected/create_procedure.out ++++ b/src/test/regress/expected/create_procedure.out +@@ -421,7 +421,7 @@ ERROR: cp_testfunc1(integer) is not a procedure + DROP PROCEDURE nonexistent(); + ERROR: procedure nonexistent() does not exist + -- privileges +-CREATE USER regress_cp_user1; ++CREATE USER regress_cp_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT INSERT ON cp_test TO regress_cp_user1; + REVOKE EXECUTE ON PROCEDURE ptest1(text) FROM PUBLIC; + SET ROLE regress_cp_user1; +diff --git a/src/test/regress/expected/create_role.out b/src/test/regress/expected/create_role.out +index 46d4f9efe9..fc2a28a2f6 100644 +--- a/src/test/regress/expected/create_role.out ++++ b/src/test/regress/expected/create_role.out +@@ -1,28 +1,28 @@ + -- ok, superuser can create users with any set of privileges +-CREATE ROLE regress_role_super SUPERUSER; +-CREATE ROLE regress_role_admin CREATEDB CREATEROLE REPLICATION BYPASSRLS; ++CREATE ROLE regress_role_super SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_role_admin CREATEDB CREATEROLE REPLICATION BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT CREATE ON DATABASE regression TO regress_role_admin WITH GRANT OPTION; +-CREATE ROLE regress_role_limited_admin CREATEROLE; +-CREATE ROLE regress_role_normal; ++CREATE ROLE regress_role_limited_admin CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_role_normal PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- fail, CREATEROLE user can't give away role attributes without having them + SET SESSION AUTHORIZATION regress_role_limited_admin; +-CREATE ROLE regress_nosuch_superuser SUPERUSER; ++CREATE ROLE regress_nosuch_superuser SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: permission denied to create role + DETAIL: Only roles with the SUPERUSER attribute may create roles with the SUPERUSER attribute. +-CREATE ROLE regress_nosuch_replication_bypassrls REPLICATION BYPASSRLS; ++CREATE ROLE regress_nosuch_replication_bypassrls REPLICATION BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: permission denied to create role + DETAIL: Only roles with the REPLICATION attribute may create roles with the REPLICATION attribute. +-CREATE ROLE regress_nosuch_replication REPLICATION; ++CREATE ROLE regress_nosuch_replication REPLICATION PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: permission denied to create role + DETAIL: Only roles with the REPLICATION attribute may create roles with the REPLICATION attribute. +-CREATE ROLE regress_nosuch_bypassrls BYPASSRLS; ++CREATE ROLE regress_nosuch_bypassrls BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: permission denied to create role + DETAIL: Only roles with the BYPASSRLS attribute may create roles with the BYPASSRLS attribute. +-CREATE ROLE regress_nosuch_createdb CREATEDB; ++CREATE ROLE regress_nosuch_createdb CREATEDB PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: permission denied to create role + DETAIL: Only roles with the CREATEDB attribute may create roles with the CREATEDB attribute. + -- ok, can create a role without any special attributes +-CREATE ROLE regress_role_limited; ++CREATE ROLE regress_role_limited PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- fail, can't give it in any of the restricted attributes + ALTER ROLE regress_role_limited SUPERUSER; + ERROR: permission denied to alter role +@@ -39,10 +39,10 @@ DETAIL: Only roles with the BYPASSRLS attribute may change the BYPASSRLS attrib + DROP ROLE regress_role_limited; + -- ok, can give away these role attributes if you have them + SET SESSION AUTHORIZATION regress_role_admin; +-CREATE ROLE regress_replication_bypassrls REPLICATION BYPASSRLS; +-CREATE ROLE regress_replication REPLICATION; +-CREATE ROLE regress_bypassrls BYPASSRLS; +-CREATE ROLE regress_createdb CREATEDB; ++CREATE ROLE regress_replication_bypassrls REPLICATION BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_replication REPLICATION PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_bypassrls BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_createdb CREATEDB PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- ok, can toggle these role attributes off and on if you have them + ALTER ROLE regress_replication NOREPLICATION; + ALTER ROLE regress_replication REPLICATION; +@@ -58,48 +58,48 @@ ALTER ROLE regress_createdb NOSUPERUSER; + ERROR: permission denied to alter role + DETAIL: Only roles with the SUPERUSER attribute may change the SUPERUSER attribute. + -- ok, having CREATEROLE is enough to create users with these privileges +-CREATE ROLE regress_createrole CREATEROLE NOINHERIT; ++CREATE ROLE regress_createrole CREATEROLE NOINHERIT PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT CREATE ON DATABASE regression TO regress_createrole WITH GRANT OPTION; +-CREATE ROLE regress_login LOGIN; +-CREATE ROLE regress_inherit INHERIT; +-CREATE ROLE regress_connection_limit CONNECTION LIMIT 5; +-CREATE ROLE regress_encrypted_password ENCRYPTED PASSWORD 'foo'; +-CREATE ROLE regress_password_null PASSWORD NULL; ++CREATE ROLE regress_login LOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_inherit INHERIT PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_connection_limit CONNECTION LIMIT 5 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_encrypted_password ENCRYPTED PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_password_null PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- ok, backwards compatible noise words should be ignored +-CREATE ROLE regress_noiseword SYSID 12345; ++CREATE ROLE regress_noiseword SYSID 12345 PASSWORD NEON_PASSWORD_PLACEHOLDER; + NOTICE: SYSID can no longer be specified + -- fail, cannot grant membership in superuser role +-CREATE ROLE regress_nosuch_super IN ROLE regress_role_super; ++CREATE ROLE regress_nosuch_super IN ROLE regress_role_super PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: permission denied to grant role "regress_role_super" + DETAIL: Only roles with the SUPERUSER attribute may grant roles with the SUPERUSER attribute. + -- fail, database owner cannot have members +-CREATE ROLE regress_nosuch_dbowner IN ROLE pg_database_owner; ++CREATE ROLE regress_nosuch_dbowner IN ROLE pg_database_owner PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: role "pg_database_owner" cannot have explicit members + -- ok, can grant other users into a role + CREATE ROLE regress_inroles ROLE + regress_role_super, regress_createdb, regress_createrole, regress_login, +- regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null; ++ regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- fail, cannot grant a role into itself +-CREATE ROLE regress_nosuch_recursive ROLE regress_nosuch_recursive; ++CREATE ROLE regress_nosuch_recursive ROLE regress_nosuch_recursive PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: role "regress_nosuch_recursive" is a member of role "regress_nosuch_recursive" + -- ok, can grant other users into a role with admin option + CREATE ROLE regress_adminroles ADMIN + regress_role_super, regress_createdb, regress_createrole, regress_login, +- regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null; ++ regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- fail, cannot grant a role into itself with admin option +-CREATE ROLE regress_nosuch_admin_recursive ADMIN regress_nosuch_admin_recursive; ++CREATE ROLE regress_nosuch_admin_recursive ADMIN regress_nosuch_admin_recursive PASSWORD NEON_PASSWORD_PLACEHOLDER; + ERROR: role "regress_nosuch_admin_recursive" is a member of role "regress_nosuch_admin_recursive" + -- fail, regress_createrole does not have CREATEDB privilege + SET SESSION AUTHORIZATION regress_createrole; + CREATE DATABASE regress_nosuch_db; + ERROR: permission denied to create database + -- ok, regress_createrole can create new roles +-CREATE ROLE regress_plainrole; ++CREATE ROLE regress_plainrole PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- ok, roles with CREATEROLE can create new roles with it +-CREATE ROLE regress_rolecreator CREATEROLE; ++CREATE ROLE regress_rolecreator CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- ok, roles with CREATEROLE can create new roles with different role + -- attributes, including CREATEROLE +-CREATE ROLE regress_hasprivs CREATEROLE LOGIN INHERIT CONNECTION LIMIT 5; ++CREATE ROLE regress_hasprivs CREATEROLE LOGIN INHERIT CONNECTION LIMIT 5 PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- ok, we should be able to modify a role we created + COMMENT ON ROLE regress_hasprivs IS 'some comment'; + ALTER ROLE regress_hasprivs RENAME TO regress_tenant; +@@ -141,7 +141,7 @@ ERROR: permission denied to reassign objects + DETAIL: Only roles with privileges of role "regress_tenant" may reassign objects owned by it. + -- ok, create a role with a value for createrole_self_grant + SET createrole_self_grant = 'set, inherit'; +-CREATE ROLE regress_tenant2; ++CREATE ROLE regress_tenant2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT CREATE ON DATABASE regression TO regress_tenant2; + -- ok, regress_tenant2 can create objects within the database + SET SESSION AUTHORIZATION regress_tenant2; +@@ -165,34 +165,34 @@ ALTER TABLE tenant2_table OWNER TO regress_tenant2; + ERROR: must be able to SET ROLE "regress_tenant2" + DROP TABLE tenant2_table; + -- fail, CREATEROLE is not enough to create roles in privileged roles +-CREATE ROLE regress_read_all_data IN ROLE pg_read_all_data; ++CREATE ROLE regress_read_all_data PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_all_data; + ERROR: permission denied to grant role "pg_read_all_data" + DETAIL: Only roles with the ADMIN option on role "pg_read_all_data" may grant this role. +-CREATE ROLE regress_write_all_data IN ROLE pg_write_all_data; ++CREATE ROLE regress_write_all_data PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_write_all_data; + ERROR: permission denied to grant role "pg_write_all_data" + DETAIL: Only roles with the ADMIN option on role "pg_write_all_data" may grant this role. +-CREATE ROLE regress_monitor IN ROLE pg_monitor; ++CREATE ROLE regress_monitor PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_monitor; + ERROR: permission denied to grant role "pg_monitor" + DETAIL: Only roles with the ADMIN option on role "pg_monitor" may grant this role. +-CREATE ROLE regress_read_all_settings IN ROLE pg_read_all_settings; ++CREATE ROLE regress_read_all_settings PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_all_settings; + ERROR: permission denied to grant role "pg_read_all_settings" + DETAIL: Only roles with the ADMIN option on role "pg_read_all_settings" may grant this role. +-CREATE ROLE regress_read_all_stats IN ROLE pg_read_all_stats; ++CREATE ROLE regress_read_all_stats PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_all_stats; + ERROR: permission denied to grant role "pg_read_all_stats" + DETAIL: Only roles with the ADMIN option on role "pg_read_all_stats" may grant this role. +-CREATE ROLE regress_stat_scan_tables IN ROLE pg_stat_scan_tables; ++CREATE ROLE regress_stat_scan_tables PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_stat_scan_tables; + ERROR: permission denied to grant role "pg_stat_scan_tables" + DETAIL: Only roles with the ADMIN option on role "pg_stat_scan_tables" may grant this role. +-CREATE ROLE regress_read_server_files IN ROLE pg_read_server_files; ++CREATE ROLE regress_read_server_files PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_server_files; + ERROR: permission denied to grant role "pg_read_server_files" + DETAIL: Only roles with the ADMIN option on role "pg_read_server_files" may grant this role. +-CREATE ROLE regress_write_server_files IN ROLE pg_write_server_files; ++CREATE ROLE regress_write_server_files PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_write_server_files; + ERROR: permission denied to grant role "pg_write_server_files" + DETAIL: Only roles with the ADMIN option on role "pg_write_server_files" may grant this role. +-CREATE ROLE regress_execute_server_program IN ROLE pg_execute_server_program; ++CREATE ROLE regress_execute_server_program PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_execute_server_program; + ERROR: permission denied to grant role "pg_execute_server_program" + DETAIL: Only roles with the ADMIN option on role "pg_execute_server_program" may grant this role. +-CREATE ROLE regress_signal_backend IN ROLE pg_signal_backend; ++CREATE ROLE regress_signal_backend PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_signal_backend; + ERROR: permission denied to grant role "pg_signal_backend" + DETAIL: Only roles with the ADMIN option on role "pg_signal_backend" may grant this role. + -- fail, role still owns database objects +diff --git a/src/test/regress/expected/create_schema.out b/src/test/regress/expected/create_schema.out +index 93302a07ef..1a73f083ac 100644 +--- a/src/test/regress/expected/create_schema.out ++++ b/src/test/regress/expected/create_schema.out +@@ -2,7 +2,7 @@ + -- CREATE_SCHEMA + -- + -- Schema creation with elements. +-CREATE ROLE regress_create_schema_role SUPERUSER; ++CREATE ROLE regress_create_schema_role SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- Cases where schema creation fails as objects are qualified with a schema + -- that does not match with what's expected. + -- This checks all the object types that include schema qualifications. +diff --git a/src/test/regress/expected/create_view.out b/src/test/regress/expected/create_view.out +index f551624afb..57f1e432d4 100644 +--- a/src/test/regress/expected/create_view.out ++++ b/src/test/regress/expected/create_view.out +@@ -18,7 +18,8 @@ CREATE TABLE real_city ( + outline path + ); + \set filename :abs_srcdir '/data/real_city.data' +-COPY real_city FROM :'filename'; ++\set command '\\copy real_city FROM ' :'filename'; ++:command + ANALYZE real_city; + SELECT * + INTO TABLE ramp +diff --git a/src/test/regress/expected/database.out b/src/test/regress/expected/database.out +index 454db91ec0..01378d7081 100644 +--- a/src/test/regress/expected/database.out ++++ b/src/test/regress/expected/database.out +@@ -1,8 +1,7 @@ + CREATE DATABASE regression_tbd + ENCODING utf8 LC_COLLATE "C" LC_CTYPE "C" TEMPLATE template0; + ALTER DATABASE regression_tbd RENAME TO regression_utf8; +-ALTER DATABASE regression_utf8 SET TABLESPACE regress_tblspace; +-ALTER DATABASE regression_utf8 RESET TABLESPACE; ++WARNING: you need to manually restart any running background workers after this command + ALTER DATABASE regression_utf8 CONNECTION_LIMIT 123; + -- Test PgDatabaseToastTable. Doing this with GRANT would be slow. + BEGIN; +diff --git a/src/test/regress/expected/dependency.out b/src/test/regress/expected/dependency.out +index 74d9ff2998..fad0151614 100644 +--- a/src/test/regress/expected/dependency.out ++++ b/src/test/regress/expected/dependency.out +@@ -1,10 +1,10 @@ + -- + -- DEPENDENCIES + -- +-CREATE USER regress_dep_user; +-CREATE USER regress_dep_user2; +-CREATE USER regress_dep_user3; +-CREATE GROUP regress_dep_group; ++CREATE USER regress_dep_user PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE GROUP regress_dep_group PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE deptest (f1 serial primary key, f2 text); + GRANT SELECT ON TABLE deptest TO GROUP regress_dep_group; + GRANT ALL ON TABLE deptest TO regress_dep_user, regress_dep_user2; +@@ -41,9 +41,9 @@ ERROR: role "regress_dep_user3" cannot be dropped because some objects depend o + DROP TABLE deptest; + DROP USER regress_dep_user3; + -- Test DROP OWNED +-CREATE USER regress_dep_user0; +-CREATE USER regress_dep_user1; +-CREATE USER regress_dep_user2; ++CREATE USER regress_dep_user0 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_dep_user0; + -- permission denied + DROP OWNED BY regress_dep_user1; +diff --git a/src/test/regress/expected/drop_if_exists.out b/src/test/regress/expected/drop_if_exists.out +index 5e44c2c3ce..eb3bb329fb 100644 +--- a/src/test/regress/expected/drop_if_exists.out ++++ b/src/test/regress/expected/drop_if_exists.out +@@ -64,9 +64,9 @@ ERROR: type "test_domain_exists" does not exist + --- + --- role/user/group + --- +-CREATE USER regress_test_u1; +-CREATE ROLE regress_test_r1; +-CREATE GROUP regress_test_g1; ++CREATE USER regress_test_u1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_r1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE GROUP regress_test_g1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + DROP USER regress_test_u2; + ERROR: role "regress_test_u2" does not exist + DROP USER IF EXISTS regress_test_u1, regress_test_u2; +diff --git a/src/test/regress/expected/equivclass.out b/src/test/regress/expected/equivclass.out +index 126f7047fe..0e2cc73426 100644 +--- a/src/test/regress/expected/equivclass.out ++++ b/src/test/regress/expected/equivclass.out +@@ -384,7 +384,7 @@ set enable_nestloop = on; + set enable_mergejoin = off; + alter table ec1 enable row level security; + create policy p1 on ec1 using (f1 < '5'::int8alias1); +-create user regress_user_ectest; ++create user regress_user_ectest PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant select on ec0 to regress_user_ectest; + grant select on ec1 to regress_user_ectest; + -- without any RLS, we'll treat {a.ff, b.ff, 43} as an EquivalenceClass +diff --git a/src/test/regress/expected/event_trigger.out b/src/test/regress/expected/event_trigger.out +index 7b2198eac6..39919697ad 100644 +--- a/src/test/regress/expected/event_trigger.out ++++ b/src/test/regress/expected/event_trigger.out +@@ -85,7 +85,7 @@ create event trigger regress_event_trigger2 on ddl_command_start + -- OK + comment on event trigger regress_event_trigger is 'test comment'; + -- drop as non-superuser should fail +-create role regress_evt_user; ++create role regress_evt_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + set role regress_evt_user; + create event trigger regress_event_trigger_noperms on ddl_command_start + execute procedure test_event_trigger(); +diff --git a/src/test/regress/expected/foreign_data.out b/src/test/regress/expected/foreign_data.out +index 6ed50fdcfa..caa00a345d 100644 +--- a/src/test/regress/expected/foreign_data.out ++++ b/src/test/regress/expected/foreign_data.out +@@ -14,13 +14,13 @@ CREATE FUNCTION test_fdw_handler() + SET client_min_messages TO 'warning'; + DROP ROLE IF EXISTS regress_foreign_data_user, regress_test_role, regress_test_role2, regress_test_role_super, regress_test_indirect, regress_unprivileged_role; + RESET client_min_messages; +-CREATE ROLE regress_foreign_data_user LOGIN SUPERUSER; ++CREATE ROLE regress_foreign_data_user LOGIN SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION 'regress_foreign_data_user'; +-CREATE ROLE regress_test_role; +-CREATE ROLE regress_test_role2; +-CREATE ROLE regress_test_role_super SUPERUSER; +-CREATE ROLE regress_test_indirect; +-CREATE ROLE regress_unprivileged_role; ++CREATE ROLE regress_test_role PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_role2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_role_super SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_indirect PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_unprivileged_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE FOREIGN DATA WRAPPER dummy; + COMMENT ON FOREIGN DATA WRAPPER dummy IS 'useless'; + CREATE FOREIGN DATA WRAPPER postgresql VALIDATOR postgresql_fdw_validator; +diff --git a/src/test/regress/expected/foreign_key.out b/src/test/regress/expected/foreign_key.out +index 69994c98e3..129abcfbe8 100644 +--- a/src/test/regress/expected/foreign_key.out ++++ b/src/test/regress/expected/foreign_key.out +@@ -1985,7 +1985,7 @@ ALTER TABLE fk_partitioned_fk_6 ATTACH PARTITION fk_partitioned_pk_6 FOR VALUES + ERROR: cannot ALTER TABLE "fk_partitioned_pk_61" because it is being used by active queries in this session + DROP TABLE fk_partitioned_pk_6, fk_partitioned_fk_6; + -- test the case when the referenced table is owned by a different user +-create role regress_other_partitioned_fk_owner; ++create role regress_other_partitioned_fk_owner PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant references on fk_notpartitioned_pk to regress_other_partitioned_fk_owner; + set role regress_other_partitioned_fk_owner; + create table other_partitioned_fk(a int, b int) partition by list (a); +diff --git a/src/test/regress/expected/generated.out b/src/test/regress/expected/generated.out +index 499072e14c..bd7a8b3f18 100644 +--- a/src/test/regress/expected/generated.out ++++ b/src/test/regress/expected/generated.out +@@ -534,7 +534,7 @@ CREATE TABLE gtest10a (a int PRIMARY KEY, b int GENERATED ALWAYS AS (a * 2) STOR + ALTER TABLE gtest10a DROP COLUMN b; + INSERT INTO gtest10a (a) VALUES (1); + -- privileges +-CREATE USER regress_user11; ++CREATE USER regress_user11 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE gtest11s (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (b * 2) STORED); + INSERT INTO gtest11s VALUES (1, 10), (2, 20); + GRANT SELECT (a, c) ON gtest11s TO regress_user11; +diff --git a/src/test/regress/expected/guc.out b/src/test/regress/expected/guc.out +index 455b6d6c0c..12fa350c6d 100644 +--- a/src/test/regress/expected/guc.out ++++ b/src/test/regress/expected/guc.out +@@ -584,7 +584,7 @@ PREPARE foo AS SELECT 1; + LISTEN foo_event; + SET vacuum_cost_delay = 13; + CREATE TEMP TABLE tmp_foo (data text) ON COMMIT DELETE ROWS; +-CREATE ROLE regress_guc_user; ++CREATE ROLE regress_guc_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_guc_user; + -- look changes + SELECT pg_listening_channels(); +diff --git a/src/test/regress/expected/hash_index.out b/src/test/regress/expected/hash_index.out +index 0d4bdb2ade..9a5a9b5407 100644 +--- a/src/test/regress/expected/hash_index.out ++++ b/src/test/regress/expected/hash_index.out +@@ -20,10 +20,14 @@ CREATE TABLE hash_f8_heap ( + random float8 + ); + \set filename :abs_srcdir '/data/hash.data' +-COPY hash_i4_heap FROM :'filename'; +-COPY hash_name_heap FROM :'filename'; +-COPY hash_txt_heap FROM :'filename'; +-COPY hash_f8_heap FROM :'filename'; ++\set command '\\copy hash_i4_heap FROM ' :'filename'; ++:command ++\set command '\\copy hash_name_heap FROM ' :'filename'; ++:command ++\set command '\\copy hash_txt_heap FROM ' :'filename'; ++:command ++\set command '\\copy hash_f8_heap FROM ' :'filename'; ++:command + -- the data in this file has a lot of duplicates in the index key + -- fields, leading to long bucket chains and lots of table expansion. + -- this is therefore a stress test of the bucket overflow code (unlike +diff --git a/src/test/regress/expected/identity.out b/src/test/regress/expected/identity.out +index f14bfccfb1..bbb2092df9 100644 +--- a/src/test/regress/expected/identity.out ++++ b/src/test/regress/expected/identity.out +@@ -520,7 +520,7 @@ ALTER TABLE itest7 ALTER COLUMN a SET GENERATED BY DEFAULT; + ALTER TABLE itest7 ALTER COLUMN a RESTART; + ALTER TABLE itest7 ALTER COLUMN a DROP IDENTITY; + -- privileges +-CREATE USER regress_identity_user1; ++CREATE USER regress_identity_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE itest8 (a int GENERATED ALWAYS AS IDENTITY, b text); + GRANT SELECT, INSERT ON itest8 TO regress_identity_user1; + SET ROLE regress_identity_user1; +diff --git a/src/test/regress/expected/inherit.out b/src/test/regress/expected/inherit.out +index 85240a9b0b..5294f7557d 100644 +--- a/src/test/regress/expected/inherit.out ++++ b/src/test/regress/expected/inherit.out +@@ -2055,8 +2055,8 @@ NOTICE: drop cascades to table cnullchild + -- + -- Mixed ownership inheritance tree + -- +-create role regress_alice; +-create role regress_bob; ++create role regress_alice password NEON_PASSWORD_PLACEHOLDER; ++create role regress_bob password NEON_PASSWORD_PLACEHOLDER; + grant all on schema public to regress_alice, regress_bob; + grant regress_alice to regress_bob; + set session authorization regress_alice; +@@ -2789,7 +2789,7 @@ create index on permtest_parent (left(c, 3)); + insert into permtest_parent + select 1, 'a', left(fipshash(i::text), 5) from generate_series(0, 100) i; + analyze permtest_parent; +-create role regress_no_child_access; ++create role regress_no_child_access PASSWORD NEON_PASSWORD_PLACEHOLDER; + revoke all on permtest_grandchild from regress_no_child_access; + grant select on permtest_parent to regress_no_child_access; + set session authorization regress_no_child_access; +diff --git a/src/test/regress/expected/insert.out b/src/test/regress/expected/insert.out +index cf4b5221a8..fa6ccb639c 100644 +--- a/src/test/regress/expected/insert.out ++++ b/src/test/regress/expected/insert.out +@@ -802,7 +802,7 @@ drop table mlparted5; + -- appropriate key description (or none) in various situations + create table key_desc (a int, b int) partition by list ((a+0)); + create table key_desc_1 partition of key_desc for values in (1) partition by range (b); +-create user regress_insert_other_user; ++create user regress_insert_other_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant select (a) on key_desc_1 to regress_insert_other_user; + grant insert on key_desc to regress_insert_other_user; + set role regress_insert_other_user; +@@ -914,7 +914,7 @@ DETAIL: Failing row contains (2, hi there). + -- check that the message shows the appropriate column description in a + -- situation where the partitioned table is not the primary ModifyTable node + create table inserttest3 (f1 text default 'foo', f2 text default 'bar', f3 int); +-create role regress_coldesc_role; ++create role regress_coldesc_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant insert on inserttest3 to regress_coldesc_role; + grant insert on brtrigpartcon to regress_coldesc_role; + revoke select on brtrigpartcon from regress_coldesc_role; +diff --git a/src/test/regress/expected/jsonb.out b/src/test/regress/expected/jsonb.out +index e66d760189..86348fd416 100644 +--- a/src/test/regress/expected/jsonb.out ++++ b/src/test/regress/expected/jsonb.out +@@ -4,7 +4,8 @@ CREATE TABLE testjsonb ( + j jsonb + ); + \set filename :abs_srcdir '/data/jsonb.data' +-COPY testjsonb FROM :'filename'; ++\set command '\\copy testjsonb FROM ' :'filename'; ++:command + -- Strings. + SELECT '""'::jsonb; -- OK. + jsonb +diff --git a/src/test/regress/expected/largeobject.out b/src/test/regress/expected/largeobject.out +index 4921dd79ae..d18a3cdd66 100644 +--- a/src/test/regress/expected/largeobject.out ++++ b/src/test/regress/expected/largeobject.out +@@ -7,7 +7,7 @@ + -- ensure consistent test output regardless of the default bytea format + SET bytea_output TO escape; + -- Test ALTER LARGE OBJECT OWNER +-CREATE ROLE regress_lo_user; ++CREATE ROLE regress_lo_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + SELECT lo_create(42); + lo_create + ----------- +@@ -346,7 +346,8 @@ SELECT lo_unlink(loid) from lotest_stash_values; + + TRUNCATE lotest_stash_values; + \set filename :abs_srcdir '/data/tenk.data' +-INSERT INTO lotest_stash_values (loid) SELECT lo_import(:'filename'); ++\lo_import :filename ++INSERT INTO lotest_stash_values (loid) VALUES (:LASTOID); + BEGIN; + UPDATE lotest_stash_values SET fd=lo_open(loid, CAST(x'20000' | x'40000' AS integer)); + -- verify length of large object +@@ -410,12 +411,8 @@ SELECT lo_close(fd) FROM lotest_stash_values; + + END; + \set filename :abs_builddir '/results/lotest.txt' +-SELECT lo_export(loid, :'filename') FROM lotest_stash_values; +- lo_export +------------ +- 1 +-(1 row) +- ++SELECT loid FROM lotest_stash_values \gset ++\lo_export :loid, :filename + \lo_import :filename + \set newloid :LASTOID + -- just make sure \lo_export does not barf +diff --git a/src/test/regress/expected/lock.out b/src/test/regress/expected/lock.out +index ad137d3645..8dac447436 100644 +--- a/src/test/regress/expected/lock.out ++++ b/src/test/regress/expected/lock.out +@@ -16,7 +16,7 @@ CREATE VIEW lock_view3 AS SELECT * from lock_view2; + CREATE VIEW lock_view4 AS SELECT (select a from lock_tbl1a limit 1) from lock_tbl1; + CREATE VIEW lock_view5 AS SELECT * from lock_tbl1 where a in (select * from lock_tbl1a); + CREATE VIEW lock_view6 AS SELECT * from (select * from lock_tbl1) sub; +-CREATE ROLE regress_rol_lock1; ++CREATE ROLE regress_rol_lock1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER ROLE regress_rol_lock1 SET search_path = lock_schema1; + GRANT USAGE ON SCHEMA lock_schema1 TO regress_rol_lock1; + -- Try all valid lock options; also try omitting the optional TABLE keyword. +diff --git a/src/test/regress/expected/matview.out b/src/test/regress/expected/matview.out +index 038ab73517..bd471f9fac 100644 +--- a/src/test/regress/expected/matview.out ++++ b/src/test/regress/expected/matview.out +@@ -549,7 +549,7 @@ SELECT * FROM mvtest_mv_v; + DROP TABLE mvtest_v CASCADE; + NOTICE: drop cascades to materialized view mvtest_mv_v + -- make sure running as superuser works when MV owned by another role (bug #11208) +-CREATE ROLE regress_user_mvtest; ++CREATE ROLE regress_user_mvtest PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET ROLE regress_user_mvtest; + -- this test case also checks for ambiguity in the queries issued by + -- refresh_by_match_merge(), by choosing column names that intentionally +@@ -617,7 +617,7 @@ HINT: Use the REFRESH MATERIALIZED VIEW command. + ROLLBACK; + -- INSERT privileges if relation owner is not allowed to insert. + CREATE SCHEMA matview_schema; +-CREATE USER regress_matview_user; ++CREATE USER regress_matview_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER DEFAULT PRIVILEGES FOR ROLE regress_matview_user + REVOKE INSERT ON TABLES FROM regress_matview_user; + GRANT ALL ON SCHEMA matview_schema TO public; +diff --git a/src/test/regress/expected/merge.out b/src/test/regress/expected/merge.out +index 521d70a891..7fd218f3d8 100644 +--- a/src/test/regress/expected/merge.out ++++ b/src/test/regress/expected/merge.out +@@ -1,9 +1,9 @@ + -- + -- MERGE + -- +-CREATE USER regress_merge_privs; +-CREATE USER regress_merge_no_privs; +-CREATE USER regress_merge_none; ++CREATE USER regress_merge_privs PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_merge_no_privs PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_merge_none PASSWORD NEON_PASSWORD_PLACEHOLDER; + DROP TABLE IF EXISTS target; + NOTICE: table "target" does not exist, skipping + DROP TABLE IF EXISTS source; +diff --git a/src/test/regress/expected/misc.out b/src/test/regress/expected/misc.out +index 6e816c57f1..6ef45b468e 100644 +--- a/src/test/regress/expected/misc.out ++++ b/src/test/regress/expected/misc.out +@@ -59,9 +59,11 @@ DROP TABLE tmp; + -- copy + -- + \set filename :abs_builddir '/results/onek.data' +-COPY onek TO :'filename'; ++\set command '\\copy onek TO ' :'filename'; ++:command + CREATE TEMP TABLE onek_copy (LIKE onek); +-COPY onek_copy FROM :'filename'; ++\set command '\\copy onek_copy FROM ' :'filename'; ++:command + SELECT * FROM onek EXCEPT ALL SELECT * FROM onek_copy; + unique1 | unique2 | two | four | ten | twenty | hundred | thousand | twothousand | fivethous | tenthous | odd | even | stringu1 | stringu2 | string4 + ---------+---------+-----+------+-----+--------+---------+----------+-------------+-----------+----------+-----+------+----------+----------+--------- +@@ -73,9 +75,11 @@ SELECT * FROM onek_copy EXCEPT ALL SELECT * FROM onek; + (0 rows) + + \set filename :abs_builddir '/results/stud_emp.data' +-COPY BINARY stud_emp TO :'filename'; ++\set command '\\COPY BINARY stud_emp TO ' :'filename'; ++:command + CREATE TEMP TABLE stud_emp_copy (LIKE stud_emp); +-COPY BINARY stud_emp_copy FROM :'filename'; ++\set command '\\COPY BINARY stud_emp_copy FROM ' :'filename'; ++:command + SELECT * FROM stud_emp_copy; + name | age | location | salary | manager | gpa | percent + -------+-----+------------+--------+---------+-----+--------- +diff --git a/src/test/regress/expected/misc_functions.out b/src/test/regress/expected/misc_functions.out +index d94056862a..f8270d8343 100644 +--- a/src/test/regress/expected/misc_functions.out ++++ b/src/test/regress/expected/misc_functions.out +@@ -297,7 +297,7 @@ SELECT pg_log_backend_memory_contexts(pid) FROM pg_stat_activity + t + (1 row) + +-CREATE ROLE regress_log_memory; ++CREATE ROLE regress_log_memory PASSWORD NEON_PASSWORD_PLACEHOLDER; + SELECT has_function_privilege('regress_log_memory', + 'pg_log_backend_memory_contexts(integer)', 'EXECUTE'); -- no + has_function_privilege +@@ -483,7 +483,7 @@ select count(*) > 0 from + -- + -- Test replication slot directory functions + -- +-CREATE ROLE regress_slot_dir_funcs; ++CREATE ROLE regress_slot_dir_funcs PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- Not available by default. + SELECT has_function_privilege('regress_slot_dir_funcs', + 'pg_ls_logicalsnapdir()', 'EXECUTE'); +@@ -671,7 +671,7 @@ FROM pg_walfile_name_offset('0/0'::pg_lsn + :segment_size - 1), + (1 row) + + -- pg_current_logfile +-CREATE ROLE regress_current_logfile; ++CREATE ROLE regress_current_logfile PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- not available by default + SELECT has_function_privilege('regress_current_logfile', + 'pg_current_logfile()', 'EXECUTE'); +diff --git a/src/test/regress/expected/multirangetypes.out b/src/test/regress/expected/multirangetypes.out +index c6363ebeb2..8f43732404 100644 +--- a/src/test/regress/expected/multirangetypes.out ++++ b/src/test/regress/expected/multirangetypes.out +@@ -3118,7 +3118,7 @@ drop type textrange2; + -- Multiranges don't have their own ownership or permissions. + -- + create type textrange1 as range(subtype=text, multirange_type_name=multitextrange1, collation="C"); +-create role regress_multirange_owner; ++create role regress_multirange_owner password NEON_PASSWORD_PLACEHOLDER; + alter type multitextrange1 owner to regress_multirange_owner; -- fail + ERROR: cannot alter multirange type multitextrange1 + HINT: You can alter type textrange1, which will alter the multirange type as well. +diff --git a/src/test/regress/expected/object_address.out b/src/test/regress/expected/object_address.out +index fc42d418bf..e38f517574 100644 +--- a/src/test/regress/expected/object_address.out ++++ b/src/test/regress/expected/object_address.out +@@ -5,7 +5,7 @@ + SET client_min_messages TO 'warning'; + DROP ROLE IF EXISTS regress_addr_user; + RESET client_min_messages; +-CREATE USER regress_addr_user; ++CREATE USER regress_addr_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- Test generic object addressing/identification functions + CREATE SCHEMA addr_nsp; + SET search_path TO 'addr_nsp'; +diff --git a/src/test/regress/expected/password.out b/src/test/regress/expected/password.out +index 924d6e001d..5966531db6 100644 +--- a/src/test/regress/expected/password.out ++++ b/src/test/regress/expected/password.out +@@ -12,13 +12,13 @@ SET password_encryption = 'md5'; -- ok + SET password_encryption = 'scram-sha-256'; -- ok + -- consistency of password entries + SET password_encryption = 'md5'; +-CREATE ROLE regress_passwd1; +-ALTER ROLE regress_passwd1 PASSWORD 'role_pwd1'; +-CREATE ROLE regress_passwd2; +-ALTER ROLE regress_passwd2 PASSWORD 'role_pwd2'; ++CREATE ROLE regress_passwd1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++ALTER ROLE regress_passwd1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_passwd2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++ALTER ROLE regress_passwd2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET password_encryption = 'scram-sha-256'; +-CREATE ROLE regress_passwd3 PASSWORD 'role_pwd3'; +-CREATE ROLE regress_passwd4 PASSWORD NULL; ++CREATE ROLE regress_passwd3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_passwd4 PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- check list of created entries + -- + -- The scram secret will look something like: +@@ -32,10 +32,10 @@ SELECT rolname, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+ + ORDER BY rolname, rolpassword; + rolname | rolpassword_masked + -----------------+--------------------------------------------------- +- regress_passwd1 | md5783277baca28003b33453252be4dbb34 +- regress_passwd2 | md54044304ba511dd062133eb5b4b84a2a3 ++ regress_passwd1 | NEON_MD5_PLACEHOLDER_regress_passwd1 ++ regress_passwd2 | NEON_MD5_PLACEHOLDER_regress_passwd2 + regress_passwd3 | SCRAM-SHA-256$4096:$: +- regress_passwd4 | ++ regress_passwd4 | SCRAM-SHA-256$4096:$: + (4 rows) + + -- Rename a role +@@ -56,24 +56,30 @@ ALTER ROLE regress_passwd2_new RENAME TO regress_passwd2; + -- passwords. + SET password_encryption = 'md5'; + -- encrypt with MD5 +-ALTER ROLE regress_passwd2 PASSWORD 'foo'; ++ALTER ROLE regress_passwd2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- already encrypted, use as they are + ALTER ROLE regress_passwd1 PASSWORD 'md5cd3578025fe2c3d7ed1b9a9b26238b70'; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + ALTER ROLE regress_passwd3 PASSWORD 'SCRAM-SHA-256$4096:VLK4RMaQLCvNtQ==$6YtlR4t69SguDiwFvbVgVZtuz6gpJQQqUMZ7IQJK5yI=:ps75jrHeYU4lXCcXI4O8oIdJ3eO8o2jirjruw9phBTo='; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + SET password_encryption = 'scram-sha-256'; + -- create SCRAM secret +-ALTER ROLE regress_passwd4 PASSWORD 'foo'; ++ALTER ROLE regress_passwd4 PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- already encrypted with MD5, use as it is + CREATE ROLE regress_passwd5 PASSWORD 'md5e73a4b11df52a6068f8b39f90be36023'; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + -- This looks like a valid SCRAM-SHA-256 secret, but it is not + -- so it should be hashed with SCRAM-SHA-256. + CREATE ROLE regress_passwd6 PASSWORD 'SCRAM-SHA-256$1234'; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + -- These may look like valid MD5 secrets, but they are not, so they + -- should be hashed with SCRAM-SHA-256. + -- trailing garbage at the end + CREATE ROLE regress_passwd7 PASSWORD 'md5012345678901234567890123456789zz'; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + -- invalid length + CREATE ROLE regress_passwd8 PASSWORD 'md501234567890123456789012345678901zz'; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + -- Changing the SCRAM iteration count + SET scram_iterations = 1024; + CREATE ROLE regress_passwd9 PASSWORD 'alterediterationcount'; +@@ -83,63 +89,67 @@ SELECT rolname, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+ + ORDER BY rolname, rolpassword; + rolname | rolpassword_masked + -----------------+--------------------------------------------------- +- regress_passwd1 | md5cd3578025fe2c3d7ed1b9a9b26238b70 +- regress_passwd2 | md5dfa155cadd5f4ad57860162f3fab9cdb ++ regress_passwd1 | NEON_MD5_PLACEHOLDER_regress_passwd1 ++ regress_passwd2 | NEON_MD5_PLACEHOLDER_regress_passwd2 + regress_passwd3 | SCRAM-SHA-256$4096:$: + regress_passwd4 | SCRAM-SHA-256$4096:$: +- regress_passwd5 | md5e73a4b11df52a6068f8b39f90be36023 +- regress_passwd6 | SCRAM-SHA-256$4096:$: +- regress_passwd7 | SCRAM-SHA-256$4096:$: +- regress_passwd8 | SCRAM-SHA-256$4096:$: + regress_passwd9 | SCRAM-SHA-256$1024:$: +-(9 rows) ++(5 rows) + + -- An empty password is not allowed, in any form + CREATE ROLE regress_passwd_empty PASSWORD ''; + NOTICE: empty string is not a valid password, clearing password ++ERROR: Failed to get encrypted password: User "regress_passwd_empty" has no password assigned. + ALTER ROLE regress_passwd_empty PASSWORD 'md585939a5ce845f1a1b620742e3c659e0a'; +-NOTICE: empty string is not a valid password, clearing password ++ERROR: role "regress_passwd_empty" does not exist + ALTER ROLE regress_passwd_empty PASSWORD 'SCRAM-SHA-256$4096:hpFyHTUsSWcR7O9P$LgZFIt6Oqdo27ZFKbZ2nV+vtnYM995pDh9ca6WSi120=:qVV5NeluNfUPkwm7Vqat25RjSPLkGeoZBQs6wVv+um4='; +-NOTICE: empty string is not a valid password, clearing password ++ERROR: role "regress_passwd_empty" does not exist + SELECT rolpassword FROM pg_authid WHERE rolname='regress_passwd_empty'; + rolpassword + ------------- +- +-(1 row) ++(0 rows) + + -- Test with invalid stored and server keys. + -- + -- The first is valid, to act as a control. The others have too long + -- stored/server keys. They will be re-hashed. + CREATE ROLE regress_passwd_sha_len0 PASSWORD 'SCRAM-SHA-256$4096:A6xHKoH/494E941doaPOYg==$Ky+A30sewHIH3VHQLRN9vYsuzlgNyGNKCh37dy96Rqw=:COPdlNiIkrsacU5QoxydEuOH6e/KfiipeETb/bPw8ZI='; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + CREATE ROLE regress_passwd_sha_len1 PASSWORD 'SCRAM-SHA-256$4096:A6xHKoH/494E941doaPOYg==$Ky+A30sewHIH3VHQLRN9vYsuzlgNyGNKCh37dy96RqwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=:COPdlNiIkrsacU5QoxydEuOH6e/KfiipeETb/bPw8ZI='; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + CREATE ROLE regress_passwd_sha_len2 PASSWORD 'SCRAM-SHA-256$4096:A6xHKoH/494E941doaPOYg==$Ky+A30sewHIH3VHQLRN9vYsuzlgNyGNKCh37dy96Rqw=:COPdlNiIkrsacU5QoxydEuOH6e/KfiipeETb/bPw8ZIAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA='; ++ERROR: Received HTTP code 400 from control plane: {"error":"Neon only supports being given plaintext passwords"} + -- Check that the invalid secrets were re-hashed. A re-hashed secret + -- should not contain the original salt. + SELECT rolname, rolpassword not like '%A6xHKoH/494E941doaPOYg==%' as is_rolpassword_rehashed + FROM pg_authid + WHERE rolname LIKE 'regress_passwd_sha_len%' + ORDER BY rolname; +- rolname | is_rolpassword_rehashed +--------------------------+------------------------- +- regress_passwd_sha_len0 | f +- regress_passwd_sha_len1 | t +- regress_passwd_sha_len2 | t +-(3 rows) ++ rolname | is_rolpassword_rehashed ++---------+------------------------- ++(0 rows) + + DROP ROLE regress_passwd1; + DROP ROLE regress_passwd2; + DROP ROLE regress_passwd3; + DROP ROLE regress_passwd4; + DROP ROLE regress_passwd5; ++ERROR: role "regress_passwd5" does not exist + DROP ROLE regress_passwd6; ++ERROR: role "regress_passwd6" does not exist + DROP ROLE regress_passwd7; ++ERROR: role "regress_passwd7" does not exist + DROP ROLE regress_passwd8; ++ERROR: role "regress_passwd8" does not exist + DROP ROLE regress_passwd9; + DROP ROLE regress_passwd_empty; ++ERROR: role "regress_passwd_empty" does not exist + DROP ROLE regress_passwd_sha_len0; ++ERROR: role "regress_passwd_sha_len0" does not exist + DROP ROLE regress_passwd_sha_len1; ++ERROR: role "regress_passwd_sha_len1" does not exist + DROP ROLE regress_passwd_sha_len2; ++ERROR: role "regress_passwd_sha_len2" does not exist + -- all entries should have been removed + SELECT rolname, rolpassword + FROM pg_authid +diff --git a/src/test/regress/expected/privileges.out b/src/test/regress/expected/privileges.out +index 1296da0d57..f43fffa44c 100644 +--- a/src/test/regress/expected/privileges.out ++++ b/src/test/regress/expected/privileges.out +@@ -20,19 +20,19 @@ SELECT lo_unlink(oid) FROM pg_largeobject_metadata WHERE oid >= 1000 AND oid < 3 + + RESET client_min_messages; + -- test proper begins here +-CREATE USER regress_priv_user1; +-CREATE USER regress_priv_user2; +-CREATE USER regress_priv_user3; +-CREATE USER regress_priv_user4; +-CREATE USER regress_priv_user5; +-CREATE USER regress_priv_user5; -- duplicate ++CREATE USER regress_priv_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user4 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER; -- duplicate + ERROR: role "regress_priv_user5" already exists +-CREATE USER regress_priv_user6; +-CREATE USER regress_priv_user7; +-CREATE USER regress_priv_user8; +-CREATE USER regress_priv_user9; +-CREATE USER regress_priv_user10; +-CREATE ROLE regress_priv_role; ++CREATE USER regress_priv_user6 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user7 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user8 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user9 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user10 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_priv_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- circular ADMIN OPTION grants should be disallowed + GRANT regress_priv_user1 TO regress_priv_user2 WITH ADMIN OPTION; + GRANT regress_priv_user1 TO regress_priv_user3 WITH ADMIN OPTION GRANTED BY regress_priv_user2; +@@ -108,11 +108,11 @@ ERROR: role "regress_priv_user5" cannot be dropped because some objects depend + DETAIL: privileges for membership of role regress_priv_user6 in role regress_priv_user1 + DROP ROLE regress_priv_user1, regress_priv_user5; -- ok, despite order + -- recreate the roles we just dropped +-CREATE USER regress_priv_user1; +-CREATE USER regress_priv_user2; +-CREATE USER regress_priv_user3; +-CREATE USER regress_priv_user4; +-CREATE USER regress_priv_user5; ++CREATE USER regress_priv_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user4 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT pg_read_all_data TO regress_priv_user6; + GRANT pg_write_all_data TO regress_priv_user7; + GRANT pg_read_all_settings TO regress_priv_user8 WITH ADMIN OPTION; +@@ -212,8 +212,8 @@ REVOKE pg_read_all_settings FROM regress_priv_user8; + DROP USER regress_priv_user10; + DROP USER regress_priv_user9; + DROP USER regress_priv_user8; +-CREATE GROUP regress_priv_group1; +-CREATE GROUP regress_priv_group2 WITH ADMIN regress_priv_user1 USER regress_priv_user2; ++CREATE GROUP regress_priv_group1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE GROUP regress_priv_group2 WITH ADMIN regress_priv_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER USER regress_priv_user2; + ALTER GROUP regress_priv_group1 ADD USER regress_priv_user4; + GRANT regress_priv_group2 TO regress_priv_user2 GRANTED BY regress_priv_user1; + SET SESSION AUTHORIZATION regress_priv_user1; +@@ -239,12 +239,16 @@ GRANT regress_priv_role TO regress_priv_user1 WITH ADMIN OPTION GRANTED BY regre + ERROR: permission denied to grant privileges as role "regress_priv_role" + DETAIL: The grantor must have the ADMIN option on role "regress_priv_role". + GRANT regress_priv_role TO regress_priv_user1 WITH ADMIN OPTION GRANTED BY CURRENT_ROLE; ++ERROR: permission denied to grant privileges as role "neondb_owner" ++DETAIL: The grantor must have the ADMIN option on role "regress_priv_role". + REVOKE ADMIN OPTION FOR regress_priv_role FROM regress_priv_user1 GRANTED BY foo; -- error + ERROR: role "foo" does not exist + REVOKE ADMIN OPTION FOR regress_priv_role FROM regress_priv_user1 GRANTED BY regress_priv_user2; -- warning, noop + WARNING: role "regress_priv_user1" has not been granted membership in role "regress_priv_role" by role "regress_priv_user2" + REVOKE ADMIN OPTION FOR regress_priv_role FROM regress_priv_user1 GRANTED BY CURRENT_USER; ++WARNING: role "regress_priv_user1" has not been granted membership in role "regress_priv_role" by role "neondb_owner" + REVOKE regress_priv_role FROM regress_priv_user1 GRANTED BY CURRENT_ROLE; ++WARNING: role "regress_priv_user1" has not been granted membership in role "regress_priv_role" by role "neondb_owner" + DROP ROLE regress_priv_role; + SET SESSION AUTHORIZATION regress_priv_user1; + SELECT session_user, current_user; +@@ -1776,7 +1780,7 @@ SELECT has_table_privilege('regress_priv_user1', 'atest4', 'SELECT WITH GRANT OP + + -- security-restricted operations + \c - +-CREATE ROLE regress_sro_user; ++CREATE ROLE regress_sro_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- Check that index expressions and predicates are run as the table's owner + -- A dummy index function checking current_user + CREATE FUNCTION sro_ifun(int) RETURNS int AS $$ +@@ -2668,8 +2672,8 @@ drop cascades to function testns.priv_testagg(integer) + drop cascades to function testns.priv_testproc(integer) + -- Change owner of the schema & and rename of new schema owner + \c - +-CREATE ROLE regress_schemauser1 superuser login; +-CREATE ROLE regress_schemauser2 superuser login; ++CREATE ROLE regress_schemauser1 superuser login PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_schemauser2 superuser login PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION ROLE regress_schemauser1; + CREATE SCHEMA testns; + SELECT nspname, rolname FROM pg_namespace, pg_roles WHERE pg_namespace.nspname = 'testns' AND pg_namespace.nspowner = pg_roles.oid; +@@ -2792,7 +2796,7 @@ DROP USER regress_priv_user7; + DROP USER regress_priv_user8; -- does not exist + ERROR: role "regress_priv_user8" does not exist + -- permissions with LOCK TABLE +-CREATE USER regress_locktable_user; ++CREATE USER regress_locktable_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE lock_table (a int); + -- LOCK TABLE and SELECT permission + GRANT SELECT ON lock_table TO regress_locktable_user; +@@ -2888,7 +2892,7 @@ DROP USER regress_locktable_user; + -- pg_backend_memory_contexts. + -- switch to superuser + \c - +-CREATE ROLE regress_readallstats; ++CREATE ROLE regress_readallstats PASSWORD NEON_PASSWORD_PLACEHOLDER; + SELECT has_table_privilege('regress_readallstats','pg_backend_memory_contexts','SELECT'); -- no + has_table_privilege + --------------------- +@@ -2932,10 +2936,10 @@ RESET ROLE; + -- clean up + DROP ROLE regress_readallstats; + -- test role grantor machinery +-CREATE ROLE regress_group; +-CREATE ROLE regress_group_direct_manager; +-CREATE ROLE regress_group_indirect_manager; +-CREATE ROLE regress_group_member; ++CREATE ROLE regress_group PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_group_direct_manager PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_group_indirect_manager PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_group_member PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT regress_group TO regress_group_direct_manager WITH INHERIT FALSE, ADMIN TRUE; + GRANT regress_group_direct_manager TO regress_group_indirect_manager; + SET SESSION AUTHORIZATION regress_group_direct_manager; +@@ -2964,9 +2968,9 @@ DROP ROLE regress_group_direct_manager; + DROP ROLE regress_group_indirect_manager; + DROP ROLE regress_group_member; + -- test SET and INHERIT options with object ownership changes +-CREATE ROLE regress_roleoption_protagonist; +-CREATE ROLE regress_roleoption_donor; +-CREATE ROLE regress_roleoption_recipient; ++CREATE ROLE regress_roleoption_protagonist PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_roleoption_donor PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_roleoption_recipient PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE SCHEMA regress_roleoption; + GRANT CREATE, USAGE ON SCHEMA regress_roleoption TO PUBLIC; + GRANT regress_roleoption_donor TO regress_roleoption_protagonist WITH INHERIT TRUE, SET FALSE; +@@ -2995,9 +2999,9 @@ DROP ROLE regress_roleoption_protagonist; + DROP ROLE regress_roleoption_donor; + DROP ROLE regress_roleoption_recipient; + -- MAINTAIN +-CREATE ROLE regress_no_maintain; +-CREATE ROLE regress_maintain; +-CREATE ROLE regress_maintain_all IN ROLE pg_maintain; ++CREATE ROLE regress_no_maintain PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_maintain PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_maintain_all IN ROLE pg_maintain PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE maintain_test (a INT); + CREATE INDEX ON maintain_test (a); + GRANT MAINTAIN ON maintain_test TO regress_maintain; +diff --git a/src/test/regress/expected/psql.out b/src/test/regress/expected/psql.out +index 3bbe4c5f97..e742a46a63 100644 +--- a/src/test/regress/expected/psql.out ++++ b/src/test/regress/expected/psql.out +@@ -2862,7 +2862,7 @@ Type | func + -- check conditional am display + \pset expanded off + CREATE SCHEMA tableam_display; +-CREATE ROLE regress_display_role; ++CREATE ROLE regress_display_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER SCHEMA tableam_display OWNER TO regress_display_role; + SET search_path TO tableam_display; + CREATE ACCESS METHOD heap_psql TYPE TABLE HANDLER heap_tableam_handler; +@@ -4817,7 +4817,7 @@ last error code: 22012 + reset debug_parallel_query; + \unset FETCH_COUNT + create schema testpart; +-create role regress_partitioning_role; ++create role regress_partitioning_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + alter schema testpart owner to regress_partitioning_role; + set role to regress_partitioning_role; + -- run test inside own schema and hide other partitions +@@ -5269,7 +5269,7 @@ reset work_mem; + + -- check \df+ + -- we have to use functions with a predictable owner name, so make a role +-create role regress_psql_user superuser; ++create role regress_psql_user superuser PASSWORD NEON_PASSWORD_PLACEHOLDER; + begin; + set session authorization regress_psql_user; + create function psql_df_internal (float8) +@@ -5557,11 +5557,14 @@ CREATE TEMPORARY TABLE reload_output( + line text + ); + SELECT 1 AS a \g :g_out_file +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + SELECT 2 AS b\; SELECT 3 AS c\; SELECT 4 AS d \g :g_out_file +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + COPY (SELECT 'foo') TO STDOUT \; COPY (SELECT 'bar') TO STDOUT \g :g_out_file +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + line + --------- +@@ -5600,13 +5603,15 @@ SELECT 1 AS a\; SELECT 2 AS b\; SELECT 3 AS c; + -- COPY TO file + -- The data goes to :g_out_file and the status to :o_out_file + \set QUIET false +-COPY (SELECT unique1 FROM onek ORDER BY unique1 LIMIT 10) TO :'g_out_file'; ++\set command '\\COPY (SELECT unique1 FROM onek ORDER BY unique1 LIMIT 10) TO ' :'g_out_file'; ++:command + -- DML command status + UPDATE onek SET unique1 = unique1 WHERE false; + \set QUIET true + \o + -- Check the contents of the files generated. +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + line + ------ +@@ -5623,7 +5628,8 @@ SELECT line FROM reload_output ORDER BY lineno; + (10 rows) + + TRUNCATE TABLE reload_output; +-COPY reload_output(line) FROM :'o_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'o_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + line + ---------- +@@ -5660,7 +5666,8 @@ COPY (SELECT 'foo1') TO STDOUT \; COPY (SELECT 'bar1') TO STDOUT; + COPY (SELECT 'foo2') TO STDOUT \; COPY (SELECT 'bar2') TO STDOUT \g :g_out_file + \o + -- Check the contents of the files generated. +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + line + ------ +@@ -5669,7 +5676,8 @@ SELECT line FROM reload_output ORDER BY lineno; + (2 rows) + + TRUNCATE TABLE reload_output; +-COPY reload_output(line) FROM :'o_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'o_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + line + ------ +@@ -6633,10 +6641,10 @@ cross-database references are not implemented: "no.such.database"."no.such.schem + \dX "no.such.database"."no.such.schema"."no.such.extended.statistics" + cross-database references are not implemented: "no.such.database"."no.such.schema"."no.such.extended.statistics" + -- check \drg and \du +-CREATE ROLE regress_du_role0; +-CREATE ROLE regress_du_role1; +-CREATE ROLE regress_du_role2; +-CREATE ROLE regress_du_admin; ++CREATE ROLE regress_du_role0 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_du_role1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_du_role2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_du_admin PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT regress_du_role0 TO regress_du_admin WITH ADMIN TRUE; + GRANT regress_du_role1 TO regress_du_admin WITH ADMIN TRUE; + GRANT regress_du_role2 TO regress_du_admin WITH ADMIN TRUE; +diff --git a/src/test/regress/expected/publication.out b/src/test/regress/expected/publication.out +index 30b6371134..cc01076c22 100644 +--- a/src/test/regress/expected/publication.out ++++ b/src/test/regress/expected/publication.out +@@ -1,9 +1,9 @@ + -- + -- PUBLICATION + -- +-CREATE ROLE regress_publication_user LOGIN SUPERUSER; +-CREATE ROLE regress_publication_user2; +-CREATE ROLE regress_publication_user_dummy LOGIN NOSUPERUSER; ++CREATE ROLE regress_publication_user LOGIN SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_publication_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_publication_user_dummy LOGIN NOSUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION 'regress_publication_user'; + -- suppress warning that depends on wal_level + SET client_min_messages = 'ERROR'; +@@ -1221,7 +1221,7 @@ ALTER PUBLICATION testpub2 ADD TABLE testpub_tbl1; -- ok + DROP PUBLICATION testpub2; + DROP PUBLICATION testpub3; + SET ROLE regress_publication_user; +-CREATE ROLE regress_publication_user3; ++CREATE ROLE regress_publication_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT regress_publication_user2 TO regress_publication_user3; + SET client_min_messages = 'ERROR'; + CREATE PUBLICATION testpub4 FOR TABLES IN SCHEMA pub_test; +diff --git a/src/test/regress/expected/regproc.out b/src/test/regress/expected/regproc.out +index 97b917502c..e9428535cb 100644 +--- a/src/test/regress/expected/regproc.out ++++ b/src/test/regress/expected/regproc.out +@@ -2,7 +2,7 @@ + -- regproc + -- + /* If objects exist, return oids */ +-CREATE ROLE regress_regrole_test; ++CREATE ROLE regress_regrole_test PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- without schemaname + SELECT regoper('||/'); + regoper +diff --git a/src/test/regress/expected/roleattributes.out b/src/test/regress/expected/roleattributes.out +index 5e6969b173..2c4d52237f 100644 +--- a/src/test/regress/expected/roleattributes.out ++++ b/src/test/regress/expected/roleattributes.out +@@ -1,233 +1,233 @@ + -- default for superuser is false +-CREATE ROLE regress_test_def_superuser; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_superuser'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_superuser | f | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_def_superuser PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_superuser'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_superuser | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE ROLE regress_test_superuser WITH SUPERUSER; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_superuser | t | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_superuser WITH SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_superuser | t | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_superuser WITH NOSUPERUSER; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_superuser | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_superuser | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_superuser WITH SUPERUSER; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_superuser | t | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_superuser | t | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- default for inherit is true +-CREATE ROLE regress_test_def_inherit; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_inherit'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +---------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_inherit | f | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_def_inherit PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_inherit'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++--------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_inherit | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE ROLE regress_test_inherit WITH NOINHERIT; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_inherit | f | f | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_inherit WITH NOINHERIT PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_inherit | f | f | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_inherit WITH INHERIT; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_inherit | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_inherit | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_inherit WITH NOINHERIT; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_inherit | f | f | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_inherit | f | f | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- default for create role is false +-CREATE ROLE regress_test_def_createrole; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createrole'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +------------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_createrole | f | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_def_createrole PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createrole'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_createrole | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE ROLE regress_test_createrole WITH CREATEROLE; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +--------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_createrole | f | t | t | f | f | f | f | -1 | | ++CREATE ROLE regress_test_createrole WITH CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_createrole | f | t | t | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_createrole WITH NOCREATEROLE; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +--------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_createrole | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_createrole | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_createrole WITH CREATEROLE; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +--------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_createrole | f | t | t | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_createrole | f | t | t | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- default for create database is false +-CREATE ROLE regress_test_def_createdb; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createdb'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_createdb | f | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_def_createdb PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createdb'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++---------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_createdb | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE ROLE regress_test_createdb WITH CREATEDB; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_createdb | f | t | f | t | f | f | f | -1 | | ++CREATE ROLE regress_test_createdb WITH CREATEDB PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++-----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_createdb | f | t | f | t | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_createdb WITH NOCREATEDB; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_createdb | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++-----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_createdb | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_createdb WITH CREATEDB; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_createdb | f | t | f | t | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++-----------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_createdb | f | t | f | t | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- default for can login is false for role +-CREATE ROLE regress_test_def_role_canlogin; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_role_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +---------------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_role_canlogin | f | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_def_role_canlogin PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_role_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++--------------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_role_canlogin | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE ROLE regress_test_role_canlogin WITH LOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_role_canlogin | f | t | f | f | t | f | f | -1 | | ++CREATE ROLE regress_test_role_canlogin WITH LOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_role_canlogin | f | t | f | f | t | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_role_canlogin WITH NOLOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_role_canlogin | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_role_canlogin | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_role_canlogin WITH LOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_role_canlogin | f | t | f | f | t | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_role_canlogin | f | t | f | f | t | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- default for can login is true for user +-CREATE USER regress_test_def_user_canlogin; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_user_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +---------------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_user_canlogin | f | t | f | f | t | f | f | -1 | | ++CREATE USER regress_test_def_user_canlogin PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_user_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++--------------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_user_canlogin | f | t | f | f | t | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE USER regress_test_user_canlogin WITH NOLOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_user_canlogin | f | t | f | f | f | f | f | -1 | | ++CREATE USER regress_test_user_canlogin WITH NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_user_canlogin | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER USER regress_test_user_canlogin WITH LOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_user_canlogin | f | t | f | f | t | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_user_canlogin | f | t | f | f | t | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER USER regress_test_user_canlogin WITH NOLOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_user_canlogin | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_user_canlogin | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- default for replication is false +-CREATE ROLE regress_test_def_replication; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_replication'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-------------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_replication | f | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_def_replication PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_replication'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++------------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_replication | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE ROLE regress_test_replication WITH REPLICATION; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +---------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_replication | f | t | f | f | f | t | f | -1 | | ++CREATE ROLE regress_test_replication WITH REPLICATION PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++--------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_replication | f | t | f | f | f | t | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_replication WITH NOREPLICATION; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +---------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_replication | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++--------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_replication | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_replication WITH REPLICATION; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +---------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_replication | f | t | f | f | f | t | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++--------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_replication | f | t | f | f | f | t | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- default for bypassrls is false +-CREATE ROLE regress_test_def_bypassrls; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_bypassrls'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_def_bypassrls | f | t | f | f | f | f | f | -1 | | ++CREATE ROLE regress_test_def_bypassrls PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_bypassrls'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++----------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_def_bypassrls | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + +-CREATE ROLE regress_test_bypassrls WITH BYPASSRLS; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_bypassrls | f | t | f | f | f | f | t | -1 | | ++CREATE ROLE regress_test_bypassrls WITH BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_bypassrls | f | t | f | f | f | f | t | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_bypassrls WITH NOBYPASSRLS; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_bypassrls | f | t | f | f | f | f | f | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_bypassrls | f | t | f | f | f | f | f | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + ALTER ROLE regress_test_bypassrls WITH BYPASSRLS; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; +- rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | rolpassword | rolvaliduntil +-------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+-------------+--------------- +- regress_test_bypassrls | f | t | f | f | f | f | t | -1 | | ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; ++ rolname | rolsuper | rolinherit | rolcreaterole | rolcreatedb | rolcanlogin | rolreplication | rolbypassrls | rolconnlimit | regexp_replace | rolvaliduntil ++------------------------+----------+------------+---------------+-------------+-------------+----------------+--------------+--------------+---------------------------------------------------+--------------- ++ regress_test_bypassrls | f | t | f | f | f | f | t | -1 | SCRAM-SHA-256$4096:$: | + (1 row) + + -- clean up roles +diff --git a/src/test/regress/expected/rowsecurity.out b/src/test/regress/expected/rowsecurity.out +index 51bba175ec..45355a9c66 100644 +--- a/src/test/regress/expected/rowsecurity.out ++++ b/src/test/regress/expected/rowsecurity.out +@@ -14,13 +14,13 @@ DROP ROLE IF EXISTS regress_rls_group2; + DROP SCHEMA IF EXISTS regress_rls_schema CASCADE; + RESET client_min_messages; + -- initial setup +-CREATE USER regress_rls_alice NOLOGIN; +-CREATE USER regress_rls_bob NOLOGIN; +-CREATE USER regress_rls_carol NOLOGIN; +-CREATE USER regress_rls_dave NOLOGIN; +-CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN; +-CREATE ROLE regress_rls_group1 NOLOGIN; +-CREATE ROLE regress_rls_group2 NOLOGIN; ++CREATE USER regress_rls_alice NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_bob NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_carol NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_dave NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_group1 NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_group2 NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT regress_rls_group1 TO regress_rls_bob; + GRANT regress_rls_group2 TO regress_rls_carol; + CREATE SCHEMA regress_rls_schema; +@@ -4423,8 +4423,8 @@ SELECT count(*) = 0 FROM pg_depend + + -- DROP OWNED BY testing + RESET SESSION AUTHORIZATION; +-CREATE ROLE regress_rls_dob_role1; +-CREATE ROLE regress_rls_dob_role2; ++CREATE ROLE regress_rls_dob_role1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_dob_role2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE dob_t1 (c1 int); + CREATE TABLE dob_t2 (c1 int) PARTITION BY RANGE (c1); + CREATE POLICY p1 ON dob_t1 TO regress_rls_dob_role1 USING (true); +diff --git a/src/test/regress/expected/rules.out b/src/test/regress/expected/rules.out +index 13178e2b3d..9a3ebfea3c 100644 +--- a/src/test/regress/expected/rules.out ++++ b/src/test/regress/expected/rules.out +@@ -3799,7 +3799,7 @@ DROP TABLE ruletest2; + -- Test non-SELECT rule on security invoker view. + -- Should use view owner's permissions. + -- +-CREATE USER regress_rule_user1; ++CREATE USER regress_rule_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE ruletest_t1 (x int); + CREATE TABLE ruletest_t2 (x int); + CREATE VIEW ruletest_v1 WITH (security_invoker=true) AS +diff --git a/src/test/regress/expected/security_label.out b/src/test/regress/expected/security_label.out +index a8e01a6220..83543b250a 100644 +--- a/src/test/regress/expected/security_label.out ++++ b/src/test/regress/expected/security_label.out +@@ -6,8 +6,8 @@ SET client_min_messages TO 'warning'; + DROP ROLE IF EXISTS regress_seclabel_user1; + DROP ROLE IF EXISTS regress_seclabel_user2; + RESET client_min_messages; +-CREATE USER regress_seclabel_user1 WITH CREATEROLE; +-CREATE USER regress_seclabel_user2; ++CREATE USER regress_seclabel_user1 WITH CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_seclabel_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE seclabel_tbl1 (a int, b text); + CREATE TABLE seclabel_tbl2 (x int, y text); + CREATE VIEW seclabel_view1 AS SELECT * FROM seclabel_tbl2; +diff --git a/src/test/regress/expected/select_into.out b/src/test/regress/expected/select_into.out +index b79fe9a1c0..e29fab88ab 100644 +--- a/src/test/regress/expected/select_into.out ++++ b/src/test/regress/expected/select_into.out +@@ -15,7 +15,7 @@ DROP TABLE sitmp1; + -- SELECT INTO and INSERT permission, if owner is not allowed to insert. + -- + CREATE SCHEMA selinto_schema; +-CREATE USER regress_selinto_user; ++CREATE USER regress_selinto_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER DEFAULT PRIVILEGES FOR ROLE regress_selinto_user + REVOKE INSERT ON TABLES FROM regress_selinto_user; + GRANT ALL ON SCHEMA selinto_schema TO public; +diff --git a/src/test/regress/expected/select_parallel.out b/src/test/regress/expected/select_parallel.out +index 496ddb1289..a4fea8e367 100644 +--- a/src/test/regress/expected/select_parallel.out ++++ b/src/test/regress/expected/select_parallel.out +@@ -1295,7 +1295,7 @@ SELECT 1 FROM tenk1_vw_sec + + rollback; + -- test that function option SET ROLE works in parallel workers. +-create role regress_parallel_worker; ++create role regress_parallel_worker PASSWORD NEON_PASSWORD_PLACEHOLDER; + create function set_and_report_role() returns text as + $$ select current_setting('role') $$ language sql parallel safe + set role = regress_parallel_worker; +diff --git a/src/test/regress/expected/select_views.out b/src/test/regress/expected/select_views.out +index 1aeed8452b..7d9427d070 100644 +--- a/src/test/regress/expected/select_views.out ++++ b/src/test/regress/expected/select_views.out +@@ -1250,7 +1250,7 @@ SELECT * FROM toyemp WHERE name = 'sharon'; + -- + -- Test for Leaky view scenario + -- +-CREATE ROLE regress_alice; ++CREATE ROLE regress_alice PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE FUNCTION f_leak (text) + RETURNS bool LANGUAGE 'plpgsql' COST 0.0000001 + AS 'BEGIN RAISE NOTICE ''f_leak => %'', $1; RETURN true; END'; +diff --git a/src/test/regress/expected/sequence.out b/src/test/regress/expected/sequence.out +index fa8059dbcd..190d41afc7 100644 +--- a/src/test/regress/expected/sequence.out ++++ b/src/test/regress/expected/sequence.out +@@ -22,7 +22,7 @@ CREATE SEQUENCE sequence_testx OWNED BY pg_class_oid_index.oid; -- not a table + ERROR: sequence cannot be owned by relation "pg_class_oid_index" + DETAIL: This operation is not supported for indexes. + CREATE SEQUENCE sequence_testx OWNED BY pg_class.relname; -- not same schema +-ERROR: sequence must be in same schema as table it is linked to ++ERROR: sequence must have same owner as table it is linked to + CREATE TABLE sequence_test_table (a int); + CREATE SEQUENCE sequence_testx OWNED BY sequence_test_table.b; -- wrong column + ERROR: column "b" of relation "sequence_test_table" does not exist +@@ -640,7 +640,7 @@ SELECT setval('sequence_test2', 1); -- error + ERROR: cannot execute setval() in a read-only transaction + ROLLBACK; + -- privileges tests +-CREATE USER regress_seq_user; ++CREATE USER regress_seq_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- nextval + BEGIN; + SET LOCAL SESSION AUTHORIZATION regress_seq_user; +diff --git a/src/test/regress/expected/stats.out b/src/test/regress/expected/stats.out +index 6e08898b18..7eb5385b7a 100644 +--- a/src/test/regress/expected/stats.out ++++ b/src/test/regress/expected/stats.out +@@ -1301,37 +1301,6 @@ SELECT current_setting('fsync') = 'off' + t + (1 row) + +--- Change the tablespace so that the table is rewritten directly, then SELECT +--- from it to cause it to be read back into shared buffers. +-SELECT sum(reads) AS io_sum_shared_before_reads +- FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +--- Do this in a transaction to prevent spurious failures due to concurrent accesses to our newly +--- rewritten table, e.g. by autovacuum. +-BEGIN; +-ALTER TABLE test_io_shared SET TABLESPACE regress_tblspace; +--- SELECT from the table so that the data is read into shared buffers and +--- context 'normal', object 'relation' reads are counted. +-SELECT COUNT(*) FROM test_io_shared; +- count +-------- +- 100 +-(1 row) +- +-COMMIT; +-SELECT pg_stat_force_next_flush(); +- pg_stat_force_next_flush +--------------------------- +- +-(1 row) +- +-SELECT sum(reads) AS io_sum_shared_after_reads +- FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +-SELECT :io_sum_shared_after_reads > :io_sum_shared_before_reads; +- ?column? +----------- +- t +-(1 row) +- + SELECT sum(hits) AS io_sum_shared_before_hits + FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset + -- Select from the table again to count hits. +@@ -1433,6 +1402,7 @@ SELECT :io_sum_local_after_evictions > :io_sum_local_before_evictions, + -- local buffers, exercising a different codepath than standard local buffer + -- writes. + ALTER TABLE test_io_local SET TABLESPACE regress_tblspace; ++ERROR: tablespace "regress_tblspace" does not exist + SELECT pg_stat_force_next_flush(); + pg_stat_force_next_flush + -------------------------- +@@ -1444,7 +1414,7 @@ SELECT sum(writes) AS io_sum_local_new_tblspc_writes + SELECT :io_sum_local_new_tblspc_writes > :io_sum_local_after_writes; + ?column? + ---------- +- t ++ f + (1 row) + + RESET temp_buffers; +diff --git a/src/test/regress/expected/stats_ext.out b/src/test/regress/expected/stats_ext.out +index 8c4da95508..346961f92a 100644 +--- a/src/test/regress/expected/stats_ext.out ++++ b/src/test/regress/expected/stats_ext.out +@@ -70,7 +70,7 @@ DROP TABLE ext_stats_test; + CREATE TABLE ab1 (a INTEGER, b INTEGER, c INTEGER); + CREATE STATISTICS IF NOT EXISTS ab1_a_b_stats ON a, b FROM ab1; + COMMENT ON STATISTICS ab1_a_b_stats IS 'new comment'; +-CREATE ROLE regress_stats_ext; ++CREATE ROLE regress_stats_ext PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_stats_ext; + COMMENT ON STATISTICS ab1_a_b_stats IS 'changed comment'; + ERROR: must be owner of statistics object ab1_a_b_stats +@@ -3214,7 +3214,7 @@ set search_path to public, stts_s1; + stts_s1 | stts_foo | col1, col2 FROM stts_t3 | defined | defined | defined + (10 rows) + +-create role regress_stats_ext nosuperuser; ++create role regress_stats_ext nosuperuser PASSWORD NEON_PASSWORD_PLACEHOLDER; + set role regress_stats_ext; + \dX + List of extended statistics +@@ -3237,7 +3237,7 @@ drop schema stts_s1, stts_s2 cascade; + drop user regress_stats_ext; + reset search_path; + -- User with no access +-CREATE USER regress_stats_user1; ++CREATE USER regress_stats_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT USAGE ON SCHEMA tststats TO regress_stats_user1; + SET SESSION AUTHORIZATION regress_stats_user1; + SELECT * FROM tststats.priv_test_tbl; -- Permission denied +diff --git a/src/test/regress/expected/subscription.out b/src/test/regress/expected/subscription.out +index 0f2a25cdc1..de168e39d9 100644 +--- a/src/test/regress/expected/subscription.out ++++ b/src/test/regress/expected/subscription.out +@@ -1,10 +1,10 @@ + -- + -- SUBSCRIPTION + -- +-CREATE ROLE regress_subscription_user LOGIN SUPERUSER; +-CREATE ROLE regress_subscription_user2; +-CREATE ROLE regress_subscription_user3 IN ROLE pg_create_subscription; +-CREATE ROLE regress_subscription_user_dummy LOGIN NOSUPERUSER; ++CREATE ROLE regress_subscription_user LOGIN SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_subscription_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_subscription_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_create_subscription; ++CREATE ROLE regress_subscription_user_dummy LOGIN NOSUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION 'regress_subscription_user'; + -- fail - no publications + CREATE SUBSCRIPTION regress_testsub CONNECTION 'foo'; +diff --git a/src/test/regress/expected/test_setup.out b/src/test/regress/expected/test_setup.out +index 3d0eeec996..2c3932139d 100644 +--- a/src/test/regress/expected/test_setup.out ++++ b/src/test/regress/expected/test_setup.out +@@ -21,6 +21,7 @@ GRANT ALL ON SCHEMA public TO public; + -- Create a tablespace we can use in tests. + SET allow_in_place_tablespaces = true; + CREATE TABLESPACE regress_tblspace LOCATION ''; ++ERROR: CREATE TABLESPACE is not supported on Neon + -- + -- These tables have traditionally been referenced by many tests, + -- so create and populate them. Insert only non-error values here. +@@ -111,7 +112,8 @@ CREATE TABLE onek ( + string4 name + ); + \set filename :abs_srcdir '/data/onek.data' +-COPY onek FROM :'filename'; ++\set command '\\copy onek FROM ' :'filename'; ++:command + VACUUM ANALYZE onek; + CREATE TABLE onek2 AS SELECT * FROM onek; + VACUUM ANALYZE onek2; +@@ -134,7 +136,8 @@ CREATE TABLE tenk1 ( + string4 name + ); + \set filename :abs_srcdir '/data/tenk.data' +-COPY tenk1 FROM :'filename'; ++\set command '\\copy tenk1 FROM ' :'filename'; ++:command + VACUUM ANALYZE tenk1; + CREATE TABLE tenk2 AS SELECT * FROM tenk1; + VACUUM ANALYZE tenk2; +@@ -144,20 +147,23 @@ CREATE TABLE person ( + location point + ); + \set filename :abs_srcdir '/data/person.data' +-COPY person FROM :'filename'; ++\set command '\\copy person FROM ' :'filename'; ++:command + VACUUM ANALYZE person; + CREATE TABLE emp ( + salary int4, + manager name + ) INHERITS (person); + \set filename :abs_srcdir '/data/emp.data' +-COPY emp FROM :'filename'; ++\set command '\\copy emp FROM ' :'filename'; ++:command + VACUUM ANALYZE emp; + CREATE TABLE student ( + gpa float8 + ) INHERITS (person); + \set filename :abs_srcdir '/data/student.data' +-COPY student FROM :'filename'; ++\set command '\\copy student FROM ' :'filename'; ++:command + VACUUM ANALYZE student; + CREATE TABLE stud_emp ( + percent int4 +@@ -166,14 +172,16 @@ NOTICE: merging multiple inherited definitions of column "name" + NOTICE: merging multiple inherited definitions of column "age" + NOTICE: merging multiple inherited definitions of column "location" + \set filename :abs_srcdir '/data/stud_emp.data' +-COPY stud_emp FROM :'filename'; ++\set command '\\copy stud_emp FROM ' :'filename'; ++:command + VACUUM ANALYZE stud_emp; + CREATE TABLE road ( + name text, + thepath path + ); + \set filename :abs_srcdir '/data/streets.data' +-COPY road FROM :'filename'; ++\set command '\\copy road FROM ' :'filename'; ++:command + VACUUM ANALYZE road; + CREATE TABLE ihighway () INHERITS (road); + INSERT INTO ihighway +diff --git a/src/test/regress/expected/tsearch.out b/src/test/regress/expected/tsearch.out +index 9fad6c8b04..a1b8e82389 100644 +--- a/src/test/regress/expected/tsearch.out ++++ b/src/test/regress/expected/tsearch.out +@@ -63,7 +63,8 @@ CREATE TABLE test_tsvector( + a tsvector + ); + \set filename :abs_srcdir '/data/tsearch.data' +-COPY test_tsvector FROM :'filename'; ++\set command '\\copy test_tsvector FROM ' :'filename'; ++:command + ANALYZE test_tsvector; + -- test basic text search behavior without indexes, then with + SELECT count(*) FROM test_tsvector WHERE a @@ 'wr|qh'; +diff --git a/src/test/regress/expected/updatable_views.out b/src/test/regress/expected/updatable_views.out +index 442b55120c..7224709d6f 100644 +--- a/src/test/regress/expected/updatable_views.out ++++ b/src/test/regress/expected/updatable_views.out +@@ -1338,9 +1338,9 @@ NOTICE: drop cascades to 2 other objects + DETAIL: drop cascades to view rw_view1 + drop cascades to function rw_view1_aa(rw_view1) + -- permissions checks +-CREATE USER regress_view_user1; +-CREATE USER regress_view_user2; +-CREATE USER regress_view_user3; ++CREATE USER regress_view_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_view_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_view_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_view_user1; + CREATE TABLE base_tbl(a int, b text, c float); + INSERT INTO base_tbl VALUES (1, 'Row 1', 1.0); +@@ -3734,8 +3734,8 @@ DETAIL: View columns that are not columns of their base relation are not updata + drop view uv_iocu_view; + drop table uv_iocu_tab; + -- ON CONFLICT DO UPDATE permissions checks +-create user regress_view_user1; +-create user regress_view_user2; ++create user regress_view_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++create user regress_view_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + set session authorization regress_view_user1; + create table base_tbl(a int unique, b text, c float); + insert into base_tbl values (1,'xxx',1.0); +diff --git a/src/test/regress/expected/update.out b/src/test/regress/expected/update.out +index 1b27d132d7..25b109d609 100644 +--- a/src/test/regress/expected/update.out ++++ b/src/test/regress/expected/update.out +@@ -608,7 +608,7 @@ DROP FUNCTION func_parted_mod_b(); + -- RLS policies with update-row-movement + ----------------------------------------- + ALTER TABLE range_parted ENABLE ROW LEVEL SECURITY; +-CREATE USER regress_range_parted_user; ++CREATE USER regress_range_parted_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT ALL ON range_parted, mintab TO regress_range_parted_user; + CREATE POLICY seeall ON range_parted AS PERMISSIVE FOR SELECT USING (true); + CREATE POLICY policy_range_parted ON range_parted for UPDATE USING (true) WITH CHECK (c % 2 = 0); +diff --git a/src/test/regress/expected/vacuum.out b/src/test/regress/expected/vacuum.out +index 2eba712887..d46877aca9 100644 +--- a/src/test/regress/expected/vacuum.out ++++ b/src/test/regress/expected/vacuum.out +@@ -433,7 +433,7 @@ CREATE TABLE vacowned (a int); + CREATE TABLE vacowned_parted (a int) PARTITION BY LIST (a); + CREATE TABLE vacowned_part1 PARTITION OF vacowned_parted FOR VALUES IN (1); + CREATE TABLE vacowned_part2 PARTITION OF vacowned_parted FOR VALUES IN (2); +-CREATE ROLE regress_vacuum; ++CREATE ROLE regress_vacuum PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET ROLE regress_vacuum; + -- Simple table + VACUUM vacowned; +diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule +index f53a526f7c..c07b093476 100644 +--- a/src/test/regress/parallel_schedule ++++ b/src/test/regress/parallel_schedule +@@ -135,4 +135,4 @@ test: fast_default + + # run tablespace test at the end because it drops the tablespace created during + # setup that other tests may use. +-test: tablespace ++#test: tablespace +diff --git a/src/test/regress/sql/aggregates.sql b/src/test/regress/sql/aggregates.sql +index 1a18ca3d8f..b2009628d0 100644 +--- a/src/test/regress/sql/aggregates.sql ++++ b/src/test/regress/sql/aggregates.sql +@@ -15,7 +15,8 @@ CREATE TABLE aggtest ( + ); + + \set filename :abs_srcdir '/data/agg.data' +-COPY aggtest FROM :'filename'; ++\set command '\\copy aggtest FROM ' :'filename'; ++:command + + ANALYZE aggtest; + +diff --git a/src/test/regress/sql/alter_generic.sql b/src/test/regress/sql/alter_generic.sql +index de58d268d3..9d38df7f42 100644 +--- a/src/test/regress/sql/alter_generic.sql ++++ b/src/test/regress/sql/alter_generic.sql +@@ -22,9 +22,9 @@ DROP ROLE IF EXISTS regress_alter_generic_user3; + + RESET client_min_messages; + +-CREATE USER regress_alter_generic_user3; +-CREATE USER regress_alter_generic_user2; +-CREATE USER regress_alter_generic_user1 IN ROLE regress_alter_generic_user3; ++CREATE USER regress_alter_generic_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_alter_generic_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_alter_generic_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE regress_alter_generic_user3; + + CREATE SCHEMA alt_nsp1; + CREATE SCHEMA alt_nsp2; +@@ -316,7 +316,7 @@ DROP OPERATOR FAMILY alt_opf4 USING btree; + + -- Should fail. Need to be SUPERUSER to do ALTER OPERATOR FAMILY .. ADD / DROP + BEGIN TRANSACTION; +-CREATE ROLE regress_alter_generic_user5 NOSUPERUSER; ++CREATE ROLE regress_alter_generic_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER NOSUPERUSER; + CREATE OPERATOR FAMILY alt_opf5 USING btree; + SET ROLE regress_alter_generic_user5; + ALTER OPERATOR FAMILY alt_opf5 USING btree ADD OPERATOR 1 < (int4, int2), FUNCTION 1 btint42cmp(int4, int2); +@@ -326,7 +326,7 @@ ROLLBACK; + + -- Should fail. Need rights to namespace for ALTER OPERATOR FAMILY .. ADD / DROP + BEGIN TRANSACTION; +-CREATE ROLE regress_alter_generic_user6; ++CREATE ROLE regress_alter_generic_user6 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE SCHEMA alt_nsp6; + REVOKE ALL ON SCHEMA alt_nsp6 FROM regress_alter_generic_user6; + CREATE OPERATOR FAMILY alt_nsp6.alt_opf6 USING btree; +diff --git a/src/test/regress/sql/alter_operator.sql b/src/test/regress/sql/alter_operator.sql +index 8faecf7830..bb8b8e14ea 100644 +--- a/src/test/regress/sql/alter_operator.sql ++++ b/src/test/regress/sql/alter_operator.sql +@@ -83,7 +83,7 @@ ALTER OPERATOR & (bit, bit) SET ("Restrict" = _int_contsel, "Join" = _int_contjo + -- + -- Test permission check. Must be owner to ALTER OPERATOR. + -- +-CREATE USER regress_alter_op_user; ++CREATE USER regress_alter_op_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_alter_op_user; + + ALTER OPERATOR === (boolean, boolean) SET (RESTRICT = NONE); +diff --git a/src/test/regress/sql/alter_table.sql b/src/test/regress/sql/alter_table.sql +index da12724473..86f5ae5444 100644 +--- a/src/test/regress/sql/alter_table.sql ++++ b/src/test/regress/sql/alter_table.sql +@@ -7,7 +7,7 @@ SET client_min_messages TO 'warning'; + DROP ROLE IF EXISTS regress_alter_table_user1; + RESET client_min_messages; + +-CREATE USER regress_alter_table_user1; ++CREATE USER regress_alter_table_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- + -- add attribute +@@ -2404,8 +2404,8 @@ DROP TABLE fail_part; + ALTER TABLE list_parted ATTACH PARTITION nonexistent FOR VALUES IN (1); + + -- check ownership of the source table +-CREATE ROLE regress_test_me; +-CREATE ROLE regress_test_not_me; ++CREATE ROLE regress_test_me PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_not_me PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE not_owned_by_me (LIKE list_parted); + ALTER TABLE not_owned_by_me OWNER TO regress_test_not_me; + SET SESSION AUTHORIZATION regress_test_me; +diff --git a/src/test/regress/sql/arrays.sql b/src/test/regress/sql/arrays.sql +index 47058dfde5..f8962592e4 100644 +--- a/src/test/regress/sql/arrays.sql ++++ b/src/test/regress/sql/arrays.sql +@@ -22,7 +22,8 @@ CREATE TABLE array_op_test ( + ); + + \set filename :abs_srcdir '/data/array.data' +-COPY array_op_test FROM :'filename'; ++\set command '\\copy array_op_test FROM ' :'filename'; ++:command + ANALYZE array_op_test; + + -- +diff --git a/src/test/regress/sql/btree_index.sql b/src/test/regress/sql/btree_index.sql +index 0d2a33f370..df86e6b050 100644 +--- a/src/test/regress/sql/btree_index.sql ++++ b/src/test/regress/sql/btree_index.sql +@@ -26,16 +26,20 @@ CREATE TABLE bt_f8_heap ( + ); + + \set filename :abs_srcdir '/data/desc.data' +-COPY bt_i4_heap FROM :'filename'; ++\set command '\\copy bt_i4_heap FROM ' :'filename'; ++:command + + \set filename :abs_srcdir '/data/hash.data' +-COPY bt_name_heap FROM :'filename'; ++\set command '\\copy bt_name_heap FROM ' :'filename'; ++:command + + \set filename :abs_srcdir '/data/desc.data' +-COPY bt_txt_heap FROM :'filename'; ++\set command '\\copy bt_txt_heap FROM ' :'filename'; ++:command + + \set filename :abs_srcdir '/data/hash.data' +-COPY bt_f8_heap FROM :'filename'; ++\set command '\\copy bt_f8_heap FROM ' :'filename'; ++:command + + ANALYZE bt_i4_heap; + ANALYZE bt_name_heap; +diff --git a/src/test/regress/sql/cluster.sql b/src/test/regress/sql/cluster.sql +index b7115f8610..a753f2c794 100644 +--- a/src/test/regress/sql/cluster.sql ++++ b/src/test/regress/sql/cluster.sql +@@ -108,7 +108,7 @@ WHERE pg_class.oid=indexrelid + CLUSTER pg_toast.pg_toast_826 USING pg_toast_826_index; + + -- Verify that clustering all tables does in fact cluster the right ones +-CREATE USER regress_clstr_user; ++CREATE USER regress_clstr_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE clstr_1 (a INT PRIMARY KEY); + CREATE TABLE clstr_2 (a INT PRIMARY KEY); + CREATE TABLE clstr_3 (a INT PRIMARY KEY); +@@ -235,7 +235,7 @@ DROP TABLE clstrpart; + CREATE TABLE ptnowner(i int unique) PARTITION BY LIST (i); + CREATE INDEX ptnowner_i_idx ON ptnowner(i); + CREATE TABLE ptnowner1 PARTITION OF ptnowner FOR VALUES IN (1); +-CREATE ROLE regress_ptnowner; ++CREATE ROLE regress_ptnowner PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE ptnowner2 PARTITION OF ptnowner FOR VALUES IN (2); + ALTER TABLE ptnowner1 OWNER TO regress_ptnowner; + SET SESSION AUTHORIZATION regress_ptnowner; +diff --git a/src/test/regress/sql/collate.icu.utf8.sql b/src/test/regress/sql/collate.icu.utf8.sql +index 4eb1adf028..28636ec711 100644 +--- a/src/test/regress/sql/collate.icu.utf8.sql ++++ b/src/test/regress/sql/collate.icu.utf8.sql +@@ -353,7 +353,7 @@ reset enable_seqscan; + + -- schema manipulation commands + +-CREATE ROLE regress_test_role; ++CREATE ROLE regress_test_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE SCHEMA test_schema; + + -- We need to do this this way to cope with varying names for encodings: +diff --git a/src/test/regress/sql/constraints.sql b/src/test/regress/sql/constraints.sql +index e3e3bea709..fa86ddc326 100644 +--- a/src/test/regress/sql/constraints.sql ++++ b/src/test/regress/sql/constraints.sql +@@ -243,12 +243,14 @@ CREATE TABLE COPY_TBL (x INT, y TEXT, z INT, + CHECK (x > 3 AND y <> 'check failed' AND x < 7 )); + + \set filename :abs_srcdir '/data/constro.data' +-COPY COPY_TBL FROM :'filename'; ++\set command '\\copy COPY_TBL FROM ' :'filename'; ++:command + + SELECT * FROM COPY_TBL; + + \set filename :abs_srcdir '/data/constrf.data' +-COPY COPY_TBL FROM :'filename'; ++\set command '\\copy COPY_TBL FROM ' :'filename'; ++:command + + SELECT * FROM COPY_TBL; + +@@ -599,7 +601,7 @@ DROP TABLE deferred_excl; + + -- Comments + -- Setup a low-level role to enforce non-superuser checks. +-CREATE ROLE regress_constraint_comments; ++CREATE ROLE regress_constraint_comments PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_constraint_comments; + + CREATE TABLE constraint_comments_tbl (a int CONSTRAINT the_constraint CHECK (a > 0)); +@@ -621,7 +623,7 @@ COMMENT ON CONSTRAINT the_constraint ON DOMAIN constraint_comments_dom IS NULL; + + -- unauthorized user + RESET SESSION AUTHORIZATION; +-CREATE ROLE regress_constraint_comments_noaccess; ++CREATE ROLE regress_constraint_comments_noaccess PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_constraint_comments_noaccess; + COMMENT ON CONSTRAINT the_constraint ON constraint_comments_tbl IS 'no, the comment'; + COMMENT ON CONSTRAINT the_constraint ON DOMAIN constraint_comments_dom IS 'no, another comment'; +diff --git a/src/test/regress/sql/conversion.sql b/src/test/regress/sql/conversion.sql +index 9a65fca91f..58431a3056 100644 +--- a/src/test/regress/sql/conversion.sql ++++ b/src/test/regress/sql/conversion.sql +@@ -12,7 +12,7 @@ CREATE FUNCTION test_enc_conversion(bytea, name, name, bool, validlen OUT int, r + AS :'regresslib', 'test_enc_conversion' + LANGUAGE C STRICT; + +-CREATE USER regress_conversion_user WITH NOCREATEDB NOCREATEROLE; ++CREATE USER regress_conversion_user WITH NOCREATEDB NOCREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_conversion_user; + CREATE CONVERSION myconv FOR 'LATIN1' TO 'UTF8' FROM iso8859_1_to_utf8; + -- +diff --git a/src/test/regress/sql/copy.sql b/src/test/regress/sql/copy.sql +index e2dd24cb35..4a186750f8 100644 +--- a/src/test/regress/sql/copy.sql ++++ b/src/test/regress/sql/copy.sql +@@ -20,11 +20,13 @@ insert into copytest values('Mac',E'abc\rdef',3); + insert into copytest values(E'esc\\ape',E'a\\r\\\r\\\n\\nb',4); + + \set filename :abs_builddir '/results/copytest.csv' +-copy copytest to :'filename' csv; ++\set command '\\copy copytest to ' :'filename' csv; ++:command + + create temp table copytest2 (like copytest); + +-copy copytest2 from :'filename' csv; ++\set command '\\copy copytest2 from ' :'filename' csv; ++:command + + select * from copytest except select * from copytest2; + +@@ -32,9 +34,11 @@ truncate copytest2; + + --- same test but with an escape char different from quote char + +-copy copytest to :'filename' csv quote '''' escape E'\\'; ++\set command '\\copy copytest to ' :'filename' ' csv quote ' '\'\'\'\'' ' escape ' 'E\'' '\\\\\''; ++:command + +-copy copytest2 from :'filename' csv quote '''' escape E'\\'; ++\set command '\\copy copytest2 from ' :'filename' ' csv quote ' '\'\'\'\'' ' escape ' 'E\'' '\\\\\''; ++:command + + select * from copytest except select * from copytest2; + +@@ -86,16 +90,19 @@ insert into parted_copytest select x,2,'Two' from generate_series(1001,1010) x; + insert into parted_copytest select x,1,'One' from generate_series(1011,1020) x; + + \set filename :abs_builddir '/results/parted_copytest.csv' +-copy (select * from parted_copytest order by a) to :'filename'; ++\set command '\\copy (select * from parted_copytest order by a) to ' :'filename'; ++:command + + truncate parted_copytest; + +-copy parted_copytest from :'filename'; ++\set command '\\copy parted_copytest from ' :'filename'; ++:command + + -- Ensure COPY FREEZE errors for partitioned tables. + begin; + truncate parted_copytest; +-copy parted_copytest from :'filename' (freeze); ++\set command '\\copy parted_copytest from ' :'filename' (freeze); ++:command + rollback; + + select tableoid::regclass,count(*),sum(a) from parted_copytest +@@ -115,7 +122,8 @@ create trigger part_ins_trig + for each row + execute procedure part_ins_func(); + +-copy parted_copytest from :'filename'; ++\set command '\\copy parted_copytest from ' :'filename'; ++:command + + select tableoid::regclass,count(*),sum(a) from parted_copytest + group by tableoid order by tableoid::regclass::name; +@@ -124,7 +132,8 @@ truncate table parted_copytest; + create index on parted_copytest (b); + drop trigger part_ins_trig on parted_copytest_a2; + +-copy parted_copytest from stdin; ++\set command '\\copy parted_copytest from ' stdin; ++:command + 1 1 str1 + 2 2 str2 + \. +@@ -191,8 +200,8 @@ bill 20 (11,10) 1000 sharon + -- Generate COPY FROM report with FILE, with some excluded tuples. + truncate tab_progress_reporting; + \set filename :abs_srcdir '/data/emp.data' +-copy tab_progress_reporting from :'filename' +- where (salary < 2000); ++\set command '\\copy tab_progress_reporting from ' :'filename' 'where (salary < 2000)'; ++:command + + drop trigger check_after_tab_progress_reporting on tab_progress_reporting; + drop function notice_after_tab_progress_reporting(); +@@ -311,7 +320,8 @@ CREATE TABLE parted_si_p_odd PARTITION OF parted_si FOR VALUES IN (1); + -- https://postgr.es/m/18130-7a86a7356a75209d%40postgresql.org + -- https://postgr.es/m/257696.1695670946%40sss.pgh.pa.us + \set filename :abs_srcdir '/data/desc.data' +-COPY parted_si(id, data) FROM :'filename'; ++\set command '\\COPY parted_si(id, data) FROM ' :'filename'; ++:command + + -- An earlier bug (see commit b1ecb9b3fcf) could end up using a buffer from + -- the wrong partition. This test is *not* guaranteed to trigger that bug, but +diff --git a/src/test/regress/sql/copy2.sql b/src/test/regress/sql/copy2.sql +index 6b75b6c7ea..f3655b413c 100644 +--- a/src/test/regress/sql/copy2.sql ++++ b/src/test/regress/sql/copy2.sql +@@ -407,8 +407,8 @@ copy check_con_tbl from stdin; + select * from check_con_tbl; + + -- test with RLS enabled. +-CREATE ROLE regress_rls_copy_user; +-CREATE ROLE regress_rls_copy_user_colperms; ++CREATE ROLE regress_rls_copy_user PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_copy_user_colperms PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE rls_t1 (a int, b int, c int); + + COPY rls_t1 (a, b, c) from stdin; +diff --git a/src/test/regress/sql/create_function_sql.sql b/src/test/regress/sql/create_function_sql.sql +index 89e9af3a49..2b86fe2285 100644 +--- a/src/test/regress/sql/create_function_sql.sql ++++ b/src/test/regress/sql/create_function_sql.sql +@@ -6,7 +6,7 @@ + + -- All objects made in this test are in temp_func_test schema + +-CREATE USER regress_unpriv_user; ++CREATE USER regress_unpriv_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE SCHEMA temp_func_test; + GRANT ALL ON SCHEMA temp_func_test TO public; +diff --git a/src/test/regress/sql/create_index.sql b/src/test/regress/sql/create_index.sql +index e296891cab..70cea565e4 100644 +--- a/src/test/regress/sql/create_index.sql ++++ b/src/test/regress/sql/create_index.sql +@@ -71,7 +71,8 @@ CREATE TABLE fast_emp4000 ( + ); + + \set filename :abs_srcdir '/data/rect.data' +-COPY slow_emp4000 FROM :'filename'; ++\set command '\\copy slow_emp4000 FROM ' :'filename'; ++:command + + INSERT INTO fast_emp4000 SELECT * FROM slow_emp4000; + +@@ -269,7 +270,8 @@ CREATE TABLE array_index_op_test ( + ); + + \set filename :abs_srcdir '/data/array.data' +-COPY array_index_op_test FROM :'filename'; ++\set command '\\copy array_index_op_test FROM ' :'filename'; ++:command + ANALYZE array_index_op_test; + + SELECT * FROM array_index_op_test WHERE i = '{NULL}' ORDER BY seqno; +@@ -1298,7 +1300,7 @@ END; + REINDEX SCHEMA CONCURRENTLY schema_to_reindex; + + -- Failure for unauthorized user +-CREATE ROLE regress_reindexuser NOLOGIN; ++CREATE ROLE regress_reindexuser NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION ROLE regress_reindexuser; + REINDEX SCHEMA schema_to_reindex; + -- Permission failures with toast tables and indexes (pg_authid here) +diff --git a/src/test/regress/sql/create_procedure.sql b/src/test/regress/sql/create_procedure.sql +index 069a3727ce..faeeb3f744 100644 +--- a/src/test/regress/sql/create_procedure.sql ++++ b/src/test/regress/sql/create_procedure.sql +@@ -255,7 +255,7 @@ DROP PROCEDURE nonexistent(); + + -- privileges + +-CREATE USER regress_cp_user1; ++CREATE USER regress_cp_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT INSERT ON cp_test TO regress_cp_user1; + REVOKE EXECUTE ON PROCEDURE ptest1(text) FROM PUBLIC; + SET ROLE regress_cp_user1; +diff --git a/src/test/regress/sql/create_role.sql b/src/test/regress/sql/create_role.sql +index 4491a28a8a..3045434865 100644 +--- a/src/test/regress/sql/create_role.sql ++++ b/src/test/regress/sql/create_role.sql +@@ -1,20 +1,20 @@ + -- ok, superuser can create users with any set of privileges +-CREATE ROLE regress_role_super SUPERUSER; +-CREATE ROLE regress_role_admin CREATEDB CREATEROLE REPLICATION BYPASSRLS; ++CREATE ROLE regress_role_super SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_role_admin CREATEDB CREATEROLE REPLICATION BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT CREATE ON DATABASE regression TO regress_role_admin WITH GRANT OPTION; +-CREATE ROLE regress_role_limited_admin CREATEROLE; +-CREATE ROLE regress_role_normal; ++CREATE ROLE regress_role_limited_admin CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_role_normal PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- fail, CREATEROLE user can't give away role attributes without having them + SET SESSION AUTHORIZATION regress_role_limited_admin; +-CREATE ROLE regress_nosuch_superuser SUPERUSER; +-CREATE ROLE regress_nosuch_replication_bypassrls REPLICATION BYPASSRLS; +-CREATE ROLE regress_nosuch_replication REPLICATION; +-CREATE ROLE regress_nosuch_bypassrls BYPASSRLS; +-CREATE ROLE regress_nosuch_createdb CREATEDB; ++CREATE ROLE regress_nosuch_superuser SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_nosuch_replication_bypassrls REPLICATION BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_nosuch_replication REPLICATION PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_nosuch_bypassrls BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_nosuch_createdb CREATEDB PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, can create a role without any special attributes +-CREATE ROLE regress_role_limited; ++CREATE ROLE regress_role_limited PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- fail, can't give it in any of the restricted attributes + ALTER ROLE regress_role_limited SUPERUSER; +@@ -25,10 +25,10 @@ DROP ROLE regress_role_limited; + + -- ok, can give away these role attributes if you have them + SET SESSION AUTHORIZATION regress_role_admin; +-CREATE ROLE regress_replication_bypassrls REPLICATION BYPASSRLS; +-CREATE ROLE regress_replication REPLICATION; +-CREATE ROLE regress_bypassrls BYPASSRLS; +-CREATE ROLE regress_createdb CREATEDB; ++CREATE ROLE regress_replication_bypassrls REPLICATION BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_replication REPLICATION PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_bypassrls BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_createdb CREATEDB PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, can toggle these role attributes off and on if you have them + ALTER ROLE regress_replication NOREPLICATION; +@@ -43,52 +43,52 @@ ALTER ROLE regress_createdb SUPERUSER; + ALTER ROLE regress_createdb NOSUPERUSER; + + -- ok, having CREATEROLE is enough to create users with these privileges +-CREATE ROLE regress_createrole CREATEROLE NOINHERIT; ++CREATE ROLE regress_createrole CREATEROLE NOINHERIT PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT CREATE ON DATABASE regression TO regress_createrole WITH GRANT OPTION; +-CREATE ROLE regress_login LOGIN; +-CREATE ROLE regress_inherit INHERIT; +-CREATE ROLE regress_connection_limit CONNECTION LIMIT 5; +-CREATE ROLE regress_encrypted_password ENCRYPTED PASSWORD 'foo'; +-CREATE ROLE regress_password_null PASSWORD NULL; ++CREATE ROLE regress_login LOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_inherit INHERIT PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_connection_limit CONNECTION LIMIT 5 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_encrypted_password ENCRYPTED PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_password_null PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, backwards compatible noise words should be ignored +-CREATE ROLE regress_noiseword SYSID 12345; ++CREATE ROLE regress_noiseword SYSID 12345 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- fail, cannot grant membership in superuser role +-CREATE ROLE regress_nosuch_super IN ROLE regress_role_super; ++CREATE ROLE regress_nosuch_super IN ROLE regress_role_super PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- fail, database owner cannot have members +-CREATE ROLE regress_nosuch_dbowner IN ROLE pg_database_owner; ++CREATE ROLE regress_nosuch_dbowner IN ROLE pg_database_owner PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, can grant other users into a role + CREATE ROLE regress_inroles ROLE + regress_role_super, regress_createdb, regress_createrole, regress_login, +- regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null; ++ regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- fail, cannot grant a role into itself +-CREATE ROLE regress_nosuch_recursive ROLE regress_nosuch_recursive; ++CREATE ROLE regress_nosuch_recursive ROLE regress_nosuch_recursive PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, can grant other users into a role with admin option + CREATE ROLE regress_adminroles ADMIN + regress_role_super, regress_createdb, regress_createrole, regress_login, +- regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null; ++ regress_inherit, regress_connection_limit, regress_encrypted_password, regress_password_null PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- fail, cannot grant a role into itself with admin option +-CREATE ROLE regress_nosuch_admin_recursive ADMIN regress_nosuch_admin_recursive; ++CREATE ROLE regress_nosuch_admin_recursive ADMIN regress_nosuch_admin_recursive PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- fail, regress_createrole does not have CREATEDB privilege + SET SESSION AUTHORIZATION regress_createrole; + CREATE DATABASE regress_nosuch_db; + + -- ok, regress_createrole can create new roles +-CREATE ROLE regress_plainrole; ++CREATE ROLE regress_plainrole PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, roles with CREATEROLE can create new roles with it +-CREATE ROLE regress_rolecreator CREATEROLE; ++CREATE ROLE regress_rolecreator CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, roles with CREATEROLE can create new roles with different role + -- attributes, including CREATEROLE +-CREATE ROLE regress_hasprivs CREATEROLE LOGIN INHERIT CONNECTION LIMIT 5; ++CREATE ROLE regress_hasprivs CREATEROLE LOGIN INHERIT CONNECTION LIMIT 5 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- ok, we should be able to modify a role we created + COMMENT ON ROLE regress_hasprivs IS 'some comment'; +@@ -123,7 +123,7 @@ REASSIGN OWNED BY regress_tenant TO regress_createrole; + + -- ok, create a role with a value for createrole_self_grant + SET createrole_self_grant = 'set, inherit'; +-CREATE ROLE regress_tenant2; ++CREATE ROLE regress_tenant2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT CREATE ON DATABASE regression TO regress_tenant2; + + -- ok, regress_tenant2 can create objects within the database +@@ -150,16 +150,16 @@ ALTER TABLE tenant2_table OWNER TO regress_tenant2; + DROP TABLE tenant2_table; + + -- fail, CREATEROLE is not enough to create roles in privileged roles +-CREATE ROLE regress_read_all_data IN ROLE pg_read_all_data; +-CREATE ROLE regress_write_all_data IN ROLE pg_write_all_data; +-CREATE ROLE regress_monitor IN ROLE pg_monitor; +-CREATE ROLE regress_read_all_settings IN ROLE pg_read_all_settings; +-CREATE ROLE regress_read_all_stats IN ROLE pg_read_all_stats; +-CREATE ROLE regress_stat_scan_tables IN ROLE pg_stat_scan_tables; +-CREATE ROLE regress_read_server_files IN ROLE pg_read_server_files; +-CREATE ROLE regress_write_server_files IN ROLE pg_write_server_files; +-CREATE ROLE regress_execute_server_program IN ROLE pg_execute_server_program; +-CREATE ROLE regress_signal_backend IN ROLE pg_signal_backend; ++CREATE ROLE regress_read_all_data PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_all_data; ++CREATE ROLE regress_write_all_data PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_write_all_data; ++CREATE ROLE regress_monitor PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_monitor; ++CREATE ROLE regress_read_all_settings PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_all_settings; ++CREATE ROLE regress_read_all_stats PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_all_stats; ++CREATE ROLE regress_stat_scan_tables PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_stat_scan_tables; ++CREATE ROLE regress_read_server_files PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_read_server_files; ++CREATE ROLE regress_write_server_files PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_write_server_files; ++CREATE ROLE regress_execute_server_program PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_execute_server_program; ++CREATE ROLE regress_signal_backend PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_signal_backend; + + -- fail, role still owns database objects + DROP ROLE regress_tenant; +diff --git a/src/test/regress/sql/create_schema.sql b/src/test/regress/sql/create_schema.sql +index 1b7064247a..be5b662ce1 100644 +--- a/src/test/regress/sql/create_schema.sql ++++ b/src/test/regress/sql/create_schema.sql +@@ -4,7 +4,7 @@ + + -- Schema creation with elements. + +-CREATE ROLE regress_create_schema_role SUPERUSER; ++CREATE ROLE regress_create_schema_role SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- Cases where schema creation fails as objects are qualified with a schema + -- that does not match with what's expected. +diff --git a/src/test/regress/sql/create_view.sql b/src/test/regress/sql/create_view.sql +index ae6841308b..47bc792e30 100644 +--- a/src/test/regress/sql/create_view.sql ++++ b/src/test/regress/sql/create_view.sql +@@ -23,7 +23,8 @@ CREATE TABLE real_city ( + ); + + \set filename :abs_srcdir '/data/real_city.data' +-COPY real_city FROM :'filename'; ++\set command '\\copy real_city FROM ' :'filename'; ++:command + ANALYZE real_city; + + SELECT * +diff --git a/src/test/regress/sql/database.sql b/src/test/regress/sql/database.sql +index 0367c0e37a..a23b98c4bd 100644 +--- a/src/test/regress/sql/database.sql ++++ b/src/test/regress/sql/database.sql +@@ -1,8 +1,6 @@ + CREATE DATABASE regression_tbd + ENCODING utf8 LC_COLLATE "C" LC_CTYPE "C" TEMPLATE template0; + ALTER DATABASE regression_tbd RENAME TO regression_utf8; +-ALTER DATABASE regression_utf8 SET TABLESPACE regress_tblspace; +-ALTER DATABASE regression_utf8 RESET TABLESPACE; + ALTER DATABASE regression_utf8 CONNECTION_LIMIT 123; + + -- Test PgDatabaseToastTable. Doing this with GRANT would be slow. +diff --git a/src/test/regress/sql/dependency.sql b/src/test/regress/sql/dependency.sql +index 8d74ed7122..293194615e 100644 +--- a/src/test/regress/sql/dependency.sql ++++ b/src/test/regress/sql/dependency.sql +@@ -2,10 +2,10 @@ + -- DEPENDENCIES + -- + +-CREATE USER regress_dep_user; +-CREATE USER regress_dep_user2; +-CREATE USER regress_dep_user3; +-CREATE GROUP regress_dep_group; ++CREATE USER regress_dep_user PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE GROUP regress_dep_group PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE TABLE deptest (f1 serial primary key, f2 text); + +@@ -45,9 +45,9 @@ DROP TABLE deptest; + DROP USER regress_dep_user3; + + -- Test DROP OWNED +-CREATE USER regress_dep_user0; +-CREATE USER regress_dep_user1; +-CREATE USER regress_dep_user2; ++CREATE USER regress_dep_user0 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_dep_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_dep_user0; + -- permission denied + DROP OWNED BY regress_dep_user1; +diff --git a/src/test/regress/sql/drop_if_exists.sql b/src/test/regress/sql/drop_if_exists.sql +index ac6168b91f..4270062ec7 100644 +--- a/src/test/regress/sql/drop_if_exists.sql ++++ b/src/test/regress/sql/drop_if_exists.sql +@@ -86,9 +86,9 @@ DROP DOMAIN test_domain_exists; + --- role/user/group + --- + +-CREATE USER regress_test_u1; +-CREATE ROLE regress_test_r1; +-CREATE GROUP regress_test_g1; ++CREATE USER regress_test_u1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_r1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE GROUP regress_test_g1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + DROP USER regress_test_u2; + +diff --git a/src/test/regress/sql/equivclass.sql b/src/test/regress/sql/equivclass.sql +index 247b0a3105..bf018fd3a1 100644 +--- a/src/test/regress/sql/equivclass.sql ++++ b/src/test/regress/sql/equivclass.sql +@@ -230,7 +230,7 @@ set enable_mergejoin = off; + alter table ec1 enable row level security; + create policy p1 on ec1 using (f1 < '5'::int8alias1); + +-create user regress_user_ectest; ++create user regress_user_ectest PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant select on ec0 to regress_user_ectest; + grant select on ec1 to regress_user_ectest; + +diff --git a/src/test/regress/sql/event_trigger.sql b/src/test/regress/sql/event_trigger.sql +index 013546b830..616a46da1d 100644 +--- a/src/test/regress/sql/event_trigger.sql ++++ b/src/test/regress/sql/event_trigger.sql +@@ -86,7 +86,7 @@ create event trigger regress_event_trigger2 on ddl_command_start + comment on event trigger regress_event_trigger is 'test comment'; + + -- drop as non-superuser should fail +-create role regress_evt_user; ++create role regress_evt_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + set role regress_evt_user; + create event trigger regress_event_trigger_noperms on ddl_command_start + execute procedure test_event_trigger(); +diff --git a/src/test/regress/sql/foreign_data.sql b/src/test/regress/sql/foreign_data.sql +index aa147b14a9..370e0dd570 100644 +--- a/src/test/regress/sql/foreign_data.sql ++++ b/src/test/regress/sql/foreign_data.sql +@@ -22,14 +22,14 @@ DROP ROLE IF EXISTS regress_foreign_data_user, regress_test_role, regress_test_r + + RESET client_min_messages; + +-CREATE ROLE regress_foreign_data_user LOGIN SUPERUSER; ++CREATE ROLE regress_foreign_data_user LOGIN SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION 'regress_foreign_data_user'; + +-CREATE ROLE regress_test_role; +-CREATE ROLE regress_test_role2; +-CREATE ROLE regress_test_role_super SUPERUSER; +-CREATE ROLE regress_test_indirect; +-CREATE ROLE regress_unprivileged_role; ++CREATE ROLE regress_test_role PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_role2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_role_super SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_test_indirect PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_unprivileged_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE FOREIGN DATA WRAPPER dummy; + COMMENT ON FOREIGN DATA WRAPPER dummy IS 'useless'; +diff --git a/src/test/regress/sql/foreign_key.sql b/src/test/regress/sql/foreign_key.sql +index 2e710e419c..89cd481a54 100644 +--- a/src/test/regress/sql/foreign_key.sql ++++ b/src/test/regress/sql/foreign_key.sql +@@ -1435,7 +1435,7 @@ ALTER TABLE fk_partitioned_fk_6 ATTACH PARTITION fk_partitioned_pk_6 FOR VALUES + DROP TABLE fk_partitioned_pk_6, fk_partitioned_fk_6; + + -- test the case when the referenced table is owned by a different user +-create role regress_other_partitioned_fk_owner; ++create role regress_other_partitioned_fk_owner PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant references on fk_notpartitioned_pk to regress_other_partitioned_fk_owner; + set role regress_other_partitioned_fk_owner; + create table other_partitioned_fk(a int, b int) partition by list (a); +diff --git a/src/test/regress/sql/generated.sql b/src/test/regress/sql/generated.sql +index cb55d77821..9c15ae954c 100644 +--- a/src/test/regress/sql/generated.sql ++++ b/src/test/regress/sql/generated.sql +@@ -263,7 +263,7 @@ ALTER TABLE gtest10a DROP COLUMN b; + INSERT INTO gtest10a (a) VALUES (1); + + -- privileges +-CREATE USER regress_user11; ++CREATE USER regress_user11 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE TABLE gtest11s (a int PRIMARY KEY, b int, c int GENERATED ALWAYS AS (b * 2) STORED); + INSERT INTO gtest11s VALUES (1, 10), (2, 20); +diff --git a/src/test/regress/sql/guc.sql b/src/test/regress/sql/guc.sql +index dc79761955..a9ead75349 100644 +--- a/src/test/regress/sql/guc.sql ++++ b/src/test/regress/sql/guc.sql +@@ -188,7 +188,7 @@ PREPARE foo AS SELECT 1; + LISTEN foo_event; + SET vacuum_cost_delay = 13; + CREATE TEMP TABLE tmp_foo (data text) ON COMMIT DELETE ROWS; +-CREATE ROLE regress_guc_user; ++CREATE ROLE regress_guc_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_guc_user; + -- look changes + SELECT pg_listening_channels(); +diff --git a/src/test/regress/sql/hash_index.sql b/src/test/regress/sql/hash_index.sql +index 219da82981..bf99d2ec4c 100644 +--- a/src/test/regress/sql/hash_index.sql ++++ b/src/test/regress/sql/hash_index.sql +@@ -26,10 +26,14 @@ CREATE TABLE hash_f8_heap ( + ); + + \set filename :abs_srcdir '/data/hash.data' +-COPY hash_i4_heap FROM :'filename'; +-COPY hash_name_heap FROM :'filename'; +-COPY hash_txt_heap FROM :'filename'; +-COPY hash_f8_heap FROM :'filename'; ++\set command '\\copy hash_i4_heap FROM ' :'filename'; ++:command ++\set command '\\copy hash_name_heap FROM ' :'filename'; ++:command ++\set command '\\copy hash_txt_heap FROM ' :'filename'; ++:command ++\set command '\\copy hash_f8_heap FROM ' :'filename'; ++:command + + -- the data in this file has a lot of duplicates in the index key + -- fields, leading to long bucket chains and lots of table expansion. +diff --git a/src/test/regress/sql/identity.sql b/src/test/regress/sql/identity.sql +index cb0e05a2f1..b11492bd31 100644 +--- a/src/test/regress/sql/identity.sql ++++ b/src/test/regress/sql/identity.sql +@@ -287,7 +287,7 @@ ALTER TABLE itest7 ALTER COLUMN a RESTART; + ALTER TABLE itest7 ALTER COLUMN a DROP IDENTITY; + + -- privileges +-CREATE USER regress_identity_user1; ++CREATE USER regress_identity_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE itest8 (a int GENERATED ALWAYS AS IDENTITY, b text); + GRANT SELECT, INSERT ON itest8 TO regress_identity_user1; + SET ROLE regress_identity_user1; +diff --git a/src/test/regress/sql/inherit.sql b/src/test/regress/sql/inherit.sql +index 51251b0e51..3492f1cfef 100644 +--- a/src/test/regress/sql/inherit.sql ++++ b/src/test/regress/sql/inherit.sql +@@ -770,8 +770,8 @@ drop table cnullparent cascade; + -- + -- Mixed ownership inheritance tree + -- +-create role regress_alice; +-create role regress_bob; ++create role regress_alice password NEON_PASSWORD_PLACEHOLDER; ++create role regress_bob password NEON_PASSWORD_PLACEHOLDER; + grant all on schema public to regress_alice, regress_bob; + grant regress_alice to regress_bob; + set session authorization regress_alice; +@@ -1031,7 +1031,7 @@ create index on permtest_parent (left(c, 3)); + insert into permtest_parent + select 1, 'a', left(fipshash(i::text), 5) from generate_series(0, 100) i; + analyze permtest_parent; +-create role regress_no_child_access; ++create role regress_no_child_access PASSWORD NEON_PASSWORD_PLACEHOLDER; + revoke all on permtest_grandchild from regress_no_child_access; + grant select on permtest_parent to regress_no_child_access; + set session authorization regress_no_child_access; +diff --git a/src/test/regress/sql/insert.sql b/src/test/regress/sql/insert.sql +index 2b086eeb6d..913d8a0aed 100644 +--- a/src/test/regress/sql/insert.sql ++++ b/src/test/regress/sql/insert.sql +@@ -513,7 +513,7 @@ drop table mlparted5; + create table key_desc (a int, b int) partition by list ((a+0)); + create table key_desc_1 partition of key_desc for values in (1) partition by range (b); + +-create user regress_insert_other_user; ++create user regress_insert_other_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant select (a) on key_desc_1 to regress_insert_other_user; + grant insert on key_desc to regress_insert_other_user; + +@@ -597,7 +597,7 @@ insert into brtrigpartcon1 values (1, 'hi there'); + -- check that the message shows the appropriate column description in a + -- situation where the partitioned table is not the primary ModifyTable node + create table inserttest3 (f1 text default 'foo', f2 text default 'bar', f3 int); +-create role regress_coldesc_role; ++create role regress_coldesc_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + grant insert on inserttest3 to regress_coldesc_role; + grant insert on brtrigpartcon to regress_coldesc_role; + revoke select on brtrigpartcon from regress_coldesc_role; +diff --git a/src/test/regress/sql/jsonb.sql b/src/test/regress/sql/jsonb.sql +index 97bc2242a1..88c8b1dcdb 100644 +--- a/src/test/regress/sql/jsonb.sql ++++ b/src/test/regress/sql/jsonb.sql +@@ -6,7 +6,8 @@ CREATE TABLE testjsonb ( + ); + + \set filename :abs_srcdir '/data/jsonb.data' +-COPY testjsonb FROM :'filename'; ++\set command '\\copy testjsonb FROM ' :'filename'; ++:command + + -- Strings. + SELECT '""'::jsonb; -- OK. +diff --git a/src/test/regress/sql/largeobject.sql b/src/test/regress/sql/largeobject.sql +index a4aee02e3a..8839c9496a 100644 +--- a/src/test/regress/sql/largeobject.sql ++++ b/src/test/regress/sql/largeobject.sql +@@ -10,7 +10,7 @@ + SET bytea_output TO escape; + + -- Test ALTER LARGE OBJECT OWNER +-CREATE ROLE regress_lo_user; ++CREATE ROLE regress_lo_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + SELECT lo_create(42); + ALTER LARGE OBJECT 42 OWNER TO regress_lo_user; + +@@ -189,7 +189,8 @@ SELECT lo_unlink(loid) from lotest_stash_values; + TRUNCATE lotest_stash_values; + + \set filename :abs_srcdir '/data/tenk.data' +-INSERT INTO lotest_stash_values (loid) SELECT lo_import(:'filename'); ++\lo_import :filename ++INSERT INTO lotest_stash_values (loid) VALUES (:LASTOID); + + BEGIN; + UPDATE lotest_stash_values SET fd=lo_open(loid, CAST(x'20000' | x'40000' AS integer)); +@@ -219,8 +220,8 @@ SELECT lo_close(fd) FROM lotest_stash_values; + END; + + \set filename :abs_builddir '/results/lotest.txt' +-SELECT lo_export(loid, :'filename') FROM lotest_stash_values; +- ++SELECT loid FROM lotest_stash_values \gset ++\lo_export :loid, :filename + \lo_import :filename + + \set newloid :LASTOID +diff --git a/src/test/regress/sql/lock.sql b/src/test/regress/sql/lock.sql +index b88488c6d0..78b31e6dd3 100644 +--- a/src/test/regress/sql/lock.sql ++++ b/src/test/regress/sql/lock.sql +@@ -19,7 +19,7 @@ CREATE VIEW lock_view3 AS SELECT * from lock_view2; + CREATE VIEW lock_view4 AS SELECT (select a from lock_tbl1a limit 1) from lock_tbl1; + CREATE VIEW lock_view5 AS SELECT * from lock_tbl1 where a in (select * from lock_tbl1a); + CREATE VIEW lock_view6 AS SELECT * from (select * from lock_tbl1) sub; +-CREATE ROLE regress_rol_lock1; ++CREATE ROLE regress_rol_lock1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER ROLE regress_rol_lock1 SET search_path = lock_schema1; + GRANT USAGE ON SCHEMA lock_schema1 TO regress_rol_lock1; + +diff --git a/src/test/regress/sql/matview.sql b/src/test/regress/sql/matview.sql +index b74ee305e0..33b8b690fc 100644 +--- a/src/test/regress/sql/matview.sql ++++ b/src/test/regress/sql/matview.sql +@@ -209,7 +209,7 @@ SELECT * FROM mvtest_mv_v; + DROP TABLE mvtest_v CASCADE; + + -- make sure running as superuser works when MV owned by another role (bug #11208) +-CREATE ROLE regress_user_mvtest; ++CREATE ROLE regress_user_mvtest PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET ROLE regress_user_mvtest; + -- this test case also checks for ambiguity in the queries issued by + -- refresh_by_match_merge(), by choosing column names that intentionally +@@ -266,7 +266,7 @@ ROLLBACK; + + -- INSERT privileges if relation owner is not allowed to insert. + CREATE SCHEMA matview_schema; +-CREATE USER regress_matview_user; ++CREATE USER regress_matview_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER DEFAULT PRIVILEGES FOR ROLE regress_matview_user + REVOKE INSERT ON TABLES FROM regress_matview_user; + GRANT ALL ON SCHEMA matview_schema TO public; +diff --git a/src/test/regress/sql/merge.sql b/src/test/regress/sql/merge.sql +index 5ddcca84f8..99f4cef9ef 100644 +--- a/src/test/regress/sql/merge.sql ++++ b/src/test/regress/sql/merge.sql +@@ -2,9 +2,9 @@ + -- MERGE + -- + +-CREATE USER regress_merge_privs; +-CREATE USER regress_merge_no_privs; +-CREATE USER regress_merge_none; ++CREATE USER regress_merge_privs PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_merge_no_privs PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_merge_none PASSWORD NEON_PASSWORD_PLACEHOLDER; + + DROP TABLE IF EXISTS target; + DROP TABLE IF EXISTS source; +diff --git a/src/test/regress/sql/misc.sql b/src/test/regress/sql/misc.sql +index 165a2e175f..08d7096e2c 100644 +--- a/src/test/regress/sql/misc.sql ++++ b/src/test/regress/sql/misc.sql +@@ -74,22 +74,26 @@ DROP TABLE tmp; + -- copy + -- + \set filename :abs_builddir '/results/onek.data' +-COPY onek TO :'filename'; ++\set command '\\copy onek TO ' :'filename'; ++:command + + CREATE TEMP TABLE onek_copy (LIKE onek); + +-COPY onek_copy FROM :'filename'; ++\set command '\\copy onek_copy FROM ' :'filename'; ++:command + + SELECT * FROM onek EXCEPT ALL SELECT * FROM onek_copy; + + SELECT * FROM onek_copy EXCEPT ALL SELECT * FROM onek; + + \set filename :abs_builddir '/results/stud_emp.data' +-COPY BINARY stud_emp TO :'filename'; ++\set command '\\COPY BINARY stud_emp TO ' :'filename'; ++:command + + CREATE TEMP TABLE stud_emp_copy (LIKE stud_emp); + +-COPY BINARY stud_emp_copy FROM :'filename'; ++\set command '\\COPY BINARY stud_emp_copy FROM ' :'filename'; ++:command + + SELECT * FROM stud_emp_copy; + +diff --git a/src/test/regress/sql/misc_functions.sql b/src/test/regress/sql/misc_functions.sql +index 76470fcb3f..09746de223 100644 +--- a/src/test/regress/sql/misc_functions.sql ++++ b/src/test/regress/sql/misc_functions.sql +@@ -82,7 +82,7 @@ SELECT pg_log_backend_memory_contexts(pg_backend_pid()); + SELECT pg_log_backend_memory_contexts(pid) FROM pg_stat_activity + WHERE backend_type = 'checkpointer'; + +-CREATE ROLE regress_log_memory; ++CREATE ROLE regress_log_memory PASSWORD NEON_PASSWORD_PLACEHOLDER; + + SELECT has_function_privilege('regress_log_memory', + 'pg_log_backend_memory_contexts(integer)', 'EXECUTE'); -- no +@@ -169,7 +169,7 @@ select count(*) > 0 from + -- + -- Test replication slot directory functions + -- +-CREATE ROLE regress_slot_dir_funcs; ++CREATE ROLE regress_slot_dir_funcs PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- Not available by default. + SELECT has_function_privilege('regress_slot_dir_funcs', + 'pg_ls_logicalsnapdir()', 'EXECUTE'); +@@ -252,7 +252,7 @@ FROM pg_walfile_name_offset('0/0'::pg_lsn + :segment_size - 1), + pg_split_walfile_name(file_name); + + -- pg_current_logfile +-CREATE ROLE regress_current_logfile; ++CREATE ROLE regress_current_logfile PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- not available by default + SELECT has_function_privilege('regress_current_logfile', + 'pg_current_logfile()', 'EXECUTE'); +diff --git a/src/test/regress/sql/multirangetypes.sql b/src/test/regress/sql/multirangetypes.sql +index 41d5524285..373be031a2 100644 +--- a/src/test/regress/sql/multirangetypes.sql ++++ b/src/test/regress/sql/multirangetypes.sql +@@ -704,7 +704,7 @@ drop type textrange2; + -- Multiranges don't have their own ownership or permissions. + -- + create type textrange1 as range(subtype=text, multirange_type_name=multitextrange1, collation="C"); +-create role regress_multirange_owner; ++create role regress_multirange_owner password NEON_PASSWORD_PLACEHOLDER; + + alter type multitextrange1 owner to regress_multirange_owner; -- fail + alter type textrange1 owner to regress_multirange_owner; +diff --git a/src/test/regress/sql/object_address.sql b/src/test/regress/sql/object_address.sql +index 1a6c61f49d..1c31ac6a53 100644 +--- a/src/test/regress/sql/object_address.sql ++++ b/src/test/regress/sql/object_address.sql +@@ -7,7 +7,7 @@ SET client_min_messages TO 'warning'; + DROP ROLE IF EXISTS regress_addr_user; + RESET client_min_messages; + +-CREATE USER regress_addr_user; ++CREATE USER regress_addr_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- Test generic object addressing/identification functions + CREATE SCHEMA addr_nsp; +diff --git a/src/test/regress/sql/password.sql b/src/test/regress/sql/password.sql +index bb82aa4aa2..7424c91b10 100644 +--- a/src/test/regress/sql/password.sql ++++ b/src/test/regress/sql/password.sql +@@ -10,13 +10,13 @@ SET password_encryption = 'scram-sha-256'; -- ok + + -- consistency of password entries + SET password_encryption = 'md5'; +-CREATE ROLE regress_passwd1; +-ALTER ROLE regress_passwd1 PASSWORD 'role_pwd1'; +-CREATE ROLE regress_passwd2; +-ALTER ROLE regress_passwd2 PASSWORD 'role_pwd2'; ++CREATE ROLE regress_passwd1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++ALTER ROLE regress_passwd1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_passwd2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++ALTER ROLE regress_passwd2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET password_encryption = 'scram-sha-256'; +-CREATE ROLE regress_passwd3 PASSWORD 'role_pwd3'; +-CREATE ROLE regress_passwd4 PASSWORD NULL; ++CREATE ROLE regress_passwd3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_passwd4 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- check list of created entries + -- +@@ -44,14 +44,14 @@ ALTER ROLE regress_passwd2_new RENAME TO regress_passwd2; + SET password_encryption = 'md5'; + + -- encrypt with MD5 +-ALTER ROLE regress_passwd2 PASSWORD 'foo'; ++ALTER ROLE regress_passwd2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- already encrypted, use as they are + ALTER ROLE regress_passwd1 PASSWORD 'md5cd3578025fe2c3d7ed1b9a9b26238b70'; + ALTER ROLE regress_passwd3 PASSWORD 'SCRAM-SHA-256$4096:VLK4RMaQLCvNtQ==$6YtlR4t69SguDiwFvbVgVZtuz6gpJQQqUMZ7IQJK5yI=:ps75jrHeYU4lXCcXI4O8oIdJ3eO8o2jirjruw9phBTo='; + + SET password_encryption = 'scram-sha-256'; + -- create SCRAM secret +-ALTER ROLE regress_passwd4 PASSWORD 'foo'; ++ALTER ROLE regress_passwd4 PASSWORD NEON_PASSWORD_PLACEHOLDER; + -- already encrypted with MD5, use as it is + CREATE ROLE regress_passwd5 PASSWORD 'md5e73a4b11df52a6068f8b39f90be36023'; + +diff --git a/src/test/regress/sql/privileges.sql b/src/test/regress/sql/privileges.sql +index 5880bc018d..27aa952b18 100644 +--- a/src/test/regress/sql/privileges.sql ++++ b/src/test/regress/sql/privileges.sql +@@ -24,18 +24,18 @@ RESET client_min_messages; + + -- test proper begins here + +-CREATE USER regress_priv_user1; +-CREATE USER regress_priv_user2; +-CREATE USER regress_priv_user3; +-CREATE USER regress_priv_user4; +-CREATE USER regress_priv_user5; +-CREATE USER regress_priv_user5; -- duplicate +-CREATE USER regress_priv_user6; +-CREATE USER regress_priv_user7; +-CREATE USER regress_priv_user8; +-CREATE USER regress_priv_user9; +-CREATE USER regress_priv_user10; +-CREATE ROLE regress_priv_role; ++CREATE USER regress_priv_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user4 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER; -- duplicate ++CREATE USER regress_priv_user6 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user7 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user8 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user9 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user10 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_priv_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- circular ADMIN OPTION grants should be disallowed + GRANT regress_priv_user1 TO regress_priv_user2 WITH ADMIN OPTION; +@@ -84,11 +84,11 @@ DROP ROLE regress_priv_user5; -- should fail, dependency + DROP ROLE regress_priv_user1, regress_priv_user5; -- ok, despite order + + -- recreate the roles we just dropped +-CREATE USER regress_priv_user1; +-CREATE USER regress_priv_user2; +-CREATE USER regress_priv_user3; +-CREATE USER regress_priv_user4; +-CREATE USER regress_priv_user5; ++CREATE USER regress_priv_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user4 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_priv_user5 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + GRANT pg_read_all_data TO regress_priv_user6; + GRANT pg_write_all_data TO regress_priv_user7; +@@ -163,8 +163,8 @@ DROP USER regress_priv_user10; + DROP USER regress_priv_user9; + DROP USER regress_priv_user8; + +-CREATE GROUP regress_priv_group1; +-CREATE GROUP regress_priv_group2 WITH ADMIN regress_priv_user1 USER regress_priv_user2; ++CREATE GROUP regress_priv_group1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE GROUP regress_priv_group2 WITH ADMIN regress_priv_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER USER regress_priv_user2; + + ALTER GROUP regress_priv_group1 ADD USER regress_priv_user4; + +@@ -1157,7 +1157,7 @@ SELECT has_table_privilege('regress_priv_user1', 'atest4', 'SELECT WITH GRANT OP + + -- security-restricted operations + \c - +-CREATE ROLE regress_sro_user; ++CREATE ROLE regress_sro_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- Check that index expressions and predicates are run as the table's owner + +@@ -1653,8 +1653,8 @@ DROP SCHEMA testns CASCADE; + -- Change owner of the schema & and rename of new schema owner + \c - + +-CREATE ROLE regress_schemauser1 superuser login; +-CREATE ROLE regress_schemauser2 superuser login; ++CREATE ROLE regress_schemauser1 superuser login PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_schemauser2 superuser login PASSWORD NEON_PASSWORD_PLACEHOLDER; + + SET SESSION ROLE regress_schemauser1; + CREATE SCHEMA testns; +@@ -1748,7 +1748,7 @@ DROP USER regress_priv_user8; -- does not exist + + + -- permissions with LOCK TABLE +-CREATE USER regress_locktable_user; ++CREATE USER regress_locktable_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE lock_table (a int); + + -- LOCK TABLE and SELECT permission +@@ -1851,7 +1851,7 @@ DROP USER regress_locktable_user; + -- switch to superuser + \c - + +-CREATE ROLE regress_readallstats; ++CREATE ROLE regress_readallstats PASSWORD NEON_PASSWORD_PLACEHOLDER; + + SELECT has_table_privilege('regress_readallstats','pg_backend_memory_contexts','SELECT'); -- no + SELECT has_table_privilege('regress_readallstats','pg_shmem_allocations','SELECT'); -- no +@@ -1871,10 +1871,10 @@ RESET ROLE; + DROP ROLE regress_readallstats; + + -- test role grantor machinery +-CREATE ROLE regress_group; +-CREATE ROLE regress_group_direct_manager; +-CREATE ROLE regress_group_indirect_manager; +-CREATE ROLE regress_group_member; ++CREATE ROLE regress_group PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_group_direct_manager PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_group_indirect_manager PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_group_member PASSWORD NEON_PASSWORD_PLACEHOLDER; + + GRANT regress_group TO regress_group_direct_manager WITH INHERIT FALSE, ADMIN TRUE; + GRANT regress_group_direct_manager TO regress_group_indirect_manager; +@@ -1896,9 +1896,9 @@ DROP ROLE regress_group_indirect_manager; + DROP ROLE regress_group_member; + + -- test SET and INHERIT options with object ownership changes +-CREATE ROLE regress_roleoption_protagonist; +-CREATE ROLE regress_roleoption_donor; +-CREATE ROLE regress_roleoption_recipient; ++CREATE ROLE regress_roleoption_protagonist PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_roleoption_donor PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_roleoption_recipient PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE SCHEMA regress_roleoption; + GRANT CREATE, USAGE ON SCHEMA regress_roleoption TO PUBLIC; + GRANT regress_roleoption_donor TO regress_roleoption_protagonist WITH INHERIT TRUE, SET FALSE; +@@ -1926,9 +1926,9 @@ DROP ROLE regress_roleoption_donor; + DROP ROLE regress_roleoption_recipient; + + -- MAINTAIN +-CREATE ROLE regress_no_maintain; +-CREATE ROLE regress_maintain; +-CREATE ROLE regress_maintain_all IN ROLE pg_maintain; ++CREATE ROLE regress_no_maintain PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_maintain PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_maintain_all IN ROLE pg_maintain PASSWORD NEON_PASSWORD_PLACEHOLDER; + CREATE TABLE maintain_test (a INT); + CREATE INDEX ON maintain_test (a); + GRANT MAINTAIN ON maintain_test TO regress_maintain; +diff --git a/src/test/regress/sql/psql.sql b/src/test/regress/sql/psql.sql +index 3b3c6f6e29..b09d6231f8 100644 +--- a/src/test/regress/sql/psql.sql ++++ b/src/test/regress/sql/psql.sql +@@ -500,7 +500,7 @@ select 1 where false; + \pset expanded off + + CREATE SCHEMA tableam_display; +-CREATE ROLE regress_display_role; ++CREATE ROLE regress_display_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER SCHEMA tableam_display OWNER TO regress_display_role; + SET search_path TO tableam_display; + CREATE ACCESS METHOD heap_psql TYPE TABLE HANDLER heap_tableam_handler; +@@ -1182,7 +1182,7 @@ reset debug_parallel_query; + \unset FETCH_COUNT + + create schema testpart; +-create role regress_partitioning_role; ++create role regress_partitioning_role PASSWORD NEON_PASSWORD_PLACEHOLDER; + + alter schema testpart owner to regress_partitioning_role; + +@@ -1293,7 +1293,7 @@ reset work_mem; + + -- check \df+ + -- we have to use functions with a predictable owner name, so make a role +-create role regress_psql_user superuser; ++create role regress_psql_user superuser PASSWORD NEON_PASSWORD_PLACEHOLDER; + begin; + set session authorization regress_psql_user; + +@@ -1439,11 +1439,14 @@ CREATE TEMPORARY TABLE reload_output( + ); + + SELECT 1 AS a \g :g_out_file +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + SELECT 2 AS b\; SELECT 3 AS c\; SELECT 4 AS d \g :g_out_file +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + COPY (SELECT 'foo') TO STDOUT \; COPY (SELECT 'bar') TO STDOUT \g :g_out_file +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + + SELECT line FROM reload_output ORDER BY lineno; + TRUNCATE TABLE reload_output; +@@ -1460,17 +1463,20 @@ SELECT 1 AS a\; SELECT 2 AS b\; SELECT 3 AS c; + -- COPY TO file + -- The data goes to :g_out_file and the status to :o_out_file + \set QUIET false +-COPY (SELECT unique1 FROM onek ORDER BY unique1 LIMIT 10) TO :'g_out_file'; ++\set command '\\COPY (SELECT unique1 FROM onek ORDER BY unique1 LIMIT 10) TO ' :'g_out_file'; ++:command + -- DML command status + UPDATE onek SET unique1 = unique1 WHERE false; + \set QUIET true + \o + + -- Check the contents of the files generated. +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + TRUNCATE TABLE reload_output; +-COPY reload_output(line) FROM :'o_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'o_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + TRUNCATE TABLE reload_output; + +@@ -1483,10 +1489,12 @@ COPY (SELECT 'foo2') TO STDOUT \; COPY (SELECT 'bar2') TO STDOUT \g :g_out_file + \o + + -- Check the contents of the files generated. +-COPY reload_output(line) FROM :'g_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'g_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + TRUNCATE TABLE reload_output; +-COPY reload_output(line) FROM :'o_out_file'; ++\set command '\\COPY reload_output(line) FROM ' :'o_out_file'; ++:command + SELECT line FROM reload_output ORDER BY lineno; + + DROP TABLE reload_output; +@@ -1834,10 +1842,10 @@ DROP FUNCTION psql_error; + \dX "no.such.database"."no.such.schema"."no.such.extended.statistics" + + -- check \drg and \du +-CREATE ROLE regress_du_role0; +-CREATE ROLE regress_du_role1; +-CREATE ROLE regress_du_role2; +-CREATE ROLE regress_du_admin; ++CREATE ROLE regress_du_role0 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_du_role1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_du_role2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_du_admin PASSWORD NEON_PASSWORD_PLACEHOLDER; + + GRANT regress_du_role0 TO regress_du_admin WITH ADMIN TRUE; + GRANT regress_du_role1 TO regress_du_admin WITH ADMIN TRUE; +diff --git a/src/test/regress/sql/publication.sql b/src/test/regress/sql/publication.sql +index 479d4f3264..6d348a93e7 100644 +--- a/src/test/regress/sql/publication.sql ++++ b/src/test/regress/sql/publication.sql +@@ -1,9 +1,9 @@ + -- + -- PUBLICATION + -- +-CREATE ROLE regress_publication_user LOGIN SUPERUSER; +-CREATE ROLE regress_publication_user2; +-CREATE ROLE regress_publication_user_dummy LOGIN NOSUPERUSER; ++CREATE ROLE regress_publication_user LOGIN SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_publication_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_publication_user_dummy LOGIN NOSUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION 'regress_publication_user'; + + -- suppress warning that depends on wal_level +@@ -810,7 +810,7 @@ DROP PUBLICATION testpub2; + DROP PUBLICATION testpub3; + + SET ROLE regress_publication_user; +-CREATE ROLE regress_publication_user3; ++CREATE ROLE regress_publication_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT regress_publication_user2 TO regress_publication_user3; + SET client_min_messages = 'ERROR'; + CREATE PUBLICATION testpub4 FOR TABLES IN SCHEMA pub_test; +diff --git a/src/test/regress/sql/regproc.sql b/src/test/regress/sql/regproc.sql +index 232289ac39..d967ef0cd3 100644 +--- a/src/test/regress/sql/regproc.sql ++++ b/src/test/regress/sql/regproc.sql +@@ -4,7 +4,7 @@ + + /* If objects exist, return oids */ + +-CREATE ROLE regress_regrole_test; ++CREATE ROLE regress_regrole_test PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- without schemaname + +diff --git a/src/test/regress/sql/roleattributes.sql b/src/test/regress/sql/roleattributes.sql +index c961b2d730..0859b89c4f 100644 +--- a/src/test/regress/sql/roleattributes.sql ++++ b/src/test/regress/sql/roleattributes.sql +@@ -1,83 +1,83 @@ + -- default for superuser is false +-CREATE ROLE regress_test_def_superuser; ++CREATE ROLE regress_test_def_superuser PASSWORD NEON_PASSWORD_PLACEHOLDER; + +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_superuser'; +-CREATE ROLE regress_test_superuser WITH SUPERUSER; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_superuser'; ++CREATE ROLE regress_test_superuser WITH SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; + ALTER ROLE regress_test_superuser WITH NOSUPERUSER; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; + ALTER ROLE regress_test_superuser WITH SUPERUSER; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_superuser'; + + -- default for inherit is true +-CREATE ROLE regress_test_def_inherit; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_inherit'; +-CREATE ROLE regress_test_inherit WITH NOINHERIT; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; ++CREATE ROLE regress_test_def_inherit PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_inherit'; ++CREATE ROLE regress_test_inherit WITH NOINHERIT PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; + ALTER ROLE regress_test_inherit WITH INHERIT; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; + ALTER ROLE regress_test_inherit WITH NOINHERIT; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_inherit'; + + -- default for create role is false +-CREATE ROLE regress_test_def_createrole; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createrole'; +-CREATE ROLE regress_test_createrole WITH CREATEROLE; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; ++CREATE ROLE regress_test_def_createrole PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createrole'; ++CREATE ROLE regress_test_createrole WITH CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; + ALTER ROLE regress_test_createrole WITH NOCREATEROLE; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; + ALTER ROLE regress_test_createrole WITH CREATEROLE; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createrole'; + + -- default for create database is false +-CREATE ROLE regress_test_def_createdb; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createdb'; +-CREATE ROLE regress_test_createdb WITH CREATEDB; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; ++CREATE ROLE regress_test_def_createdb PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_createdb'; ++CREATE ROLE regress_test_createdb WITH CREATEDB PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; + ALTER ROLE regress_test_createdb WITH NOCREATEDB; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; + ALTER ROLE regress_test_createdb WITH CREATEDB; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_createdb'; + + -- default for can login is false for role +-CREATE ROLE regress_test_def_role_canlogin; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_role_canlogin'; +-CREATE ROLE regress_test_role_canlogin WITH LOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; ++CREATE ROLE regress_test_def_role_canlogin PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_role_canlogin'; ++CREATE ROLE regress_test_role_canlogin WITH LOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; + ALTER ROLE regress_test_role_canlogin WITH NOLOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; + ALTER ROLE regress_test_role_canlogin WITH LOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_role_canlogin'; + + -- default for can login is true for user +-CREATE USER regress_test_def_user_canlogin; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_user_canlogin'; +-CREATE USER regress_test_user_canlogin WITH NOLOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; ++CREATE USER regress_test_def_user_canlogin PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_user_canlogin'; ++CREATE USER regress_test_user_canlogin WITH NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; + ALTER USER regress_test_user_canlogin WITH LOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; + ALTER USER regress_test_user_canlogin WITH NOLOGIN; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_user_canlogin'; + + -- default for replication is false +-CREATE ROLE regress_test_def_replication; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_replication'; +-CREATE ROLE regress_test_replication WITH REPLICATION; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; ++CREATE ROLE regress_test_def_replication PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_replication'; ++CREATE ROLE regress_test_replication WITH REPLICATION PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; + ALTER ROLE regress_test_replication WITH NOREPLICATION; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; + ALTER ROLE regress_test_replication WITH REPLICATION; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_replication'; + + -- default for bypassrls is false +-CREATE ROLE regress_test_def_bypassrls; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_bypassrls'; +-CREATE ROLE regress_test_bypassrls WITH BYPASSRLS; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; ++CREATE ROLE regress_test_def_bypassrls PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_def_bypassrls'; ++CREATE ROLE regress_test_bypassrls WITH BYPASSRLS PASSWORD NEON_PASSWORD_PLACEHOLDER; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; + ALTER ROLE regress_test_bypassrls WITH NOBYPASSRLS; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; + ALTER ROLE regress_test_bypassrls WITH BYPASSRLS; +-SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, rolpassword, rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; ++SELECT rolname, rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin, rolreplication, rolbypassrls, rolconnlimit, regexp_replace(rolpassword, '(SCRAM-SHA-256)\$(\d+):([a-zA-Z0-9+/=]+)\$([a-zA-Z0-9+=/]+):([a-zA-Z0-9+/=]+)', '\1$\2:$:'), rolvaliduntil FROM pg_authid WHERE rolname = 'regress_test_bypassrls'; + + -- clean up roles + DROP ROLE regress_test_def_superuser; +diff --git a/src/test/regress/sql/rowsecurity.sql b/src/test/regress/sql/rowsecurity.sql +index eab7d99003..0cf1139e01 100644 +--- a/src/test/regress/sql/rowsecurity.sql ++++ b/src/test/regress/sql/rowsecurity.sql +@@ -20,13 +20,13 @@ DROP SCHEMA IF EXISTS regress_rls_schema CASCADE; + RESET client_min_messages; + + -- initial setup +-CREATE USER regress_rls_alice NOLOGIN; +-CREATE USER regress_rls_bob NOLOGIN; +-CREATE USER regress_rls_carol NOLOGIN; +-CREATE USER regress_rls_dave NOLOGIN; +-CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN; +-CREATE ROLE regress_rls_group1 NOLOGIN; +-CREATE ROLE regress_rls_group2 NOLOGIN; ++CREATE USER regress_rls_alice NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_bob NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_carol NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_dave NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_rls_exempt_user BYPASSRLS NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_group1 NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_group2 NOLOGIN PASSWORD NEON_PASSWORD_PLACEHOLDER; + + GRANT regress_rls_group1 TO regress_rls_bob; + GRANT regress_rls_group2 TO regress_rls_carol; +@@ -2105,8 +2105,8 @@ SELECT count(*) = 0 FROM pg_depend + -- DROP OWNED BY testing + RESET SESSION AUTHORIZATION; + +-CREATE ROLE regress_rls_dob_role1; +-CREATE ROLE regress_rls_dob_role2; ++CREATE ROLE regress_rls_dob_role1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_rls_dob_role2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE TABLE dob_t1 (c1 int); + CREATE TABLE dob_t2 (c1 int) PARTITION BY RANGE (c1); +diff --git a/src/test/regress/sql/rules.sql b/src/test/regress/sql/rules.sql +index 4a5fa50585..a9e9eab77d 100644 +--- a/src/test/regress/sql/rules.sql ++++ b/src/test/regress/sql/rules.sql +@@ -1390,7 +1390,7 @@ DROP TABLE ruletest2; + -- Test non-SELECT rule on security invoker view. + -- Should use view owner's permissions. + -- +-CREATE USER regress_rule_user1; ++CREATE USER regress_rule_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE TABLE ruletest_t1 (x int); + CREATE TABLE ruletest_t2 (x int); +diff --git a/src/test/regress/sql/security_label.sql b/src/test/regress/sql/security_label.sql +index 98e6a5f211..68c868fef2 100644 +--- a/src/test/regress/sql/security_label.sql ++++ b/src/test/regress/sql/security_label.sql +@@ -10,8 +10,8 @@ DROP ROLE IF EXISTS regress_seclabel_user2; + + RESET client_min_messages; + +-CREATE USER regress_seclabel_user1 WITH CREATEROLE; +-CREATE USER regress_seclabel_user2; ++CREATE USER regress_seclabel_user1 WITH CREATEROLE PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_seclabel_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE TABLE seclabel_tbl1 (a int, b text); + CREATE TABLE seclabel_tbl2 (x int, y text); +diff --git a/src/test/regress/sql/select_into.sql b/src/test/regress/sql/select_into.sql +index 689c448cc2..223ceb1d75 100644 +--- a/src/test/regress/sql/select_into.sql ++++ b/src/test/regress/sql/select_into.sql +@@ -20,7 +20,7 @@ DROP TABLE sitmp1; + -- SELECT INTO and INSERT permission, if owner is not allowed to insert. + -- + CREATE SCHEMA selinto_schema; +-CREATE USER regress_selinto_user; ++CREATE USER regress_selinto_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + ALTER DEFAULT PRIVILEGES FOR ROLE regress_selinto_user + REVOKE INSERT ON TABLES FROM regress_selinto_user; + GRANT ALL ON SCHEMA selinto_schema TO public; +diff --git a/src/test/regress/sql/select_parallel.sql b/src/test/regress/sql/select_parallel.sql +index 3e4bfcb71f..99757eff3c 100644 +--- a/src/test/regress/sql/select_parallel.sql ++++ b/src/test/regress/sql/select_parallel.sql +@@ -498,7 +498,7 @@ SELECT 1 FROM tenk1_vw_sec + rollback; + + -- test that function option SET ROLE works in parallel workers. +-create role regress_parallel_worker; ++create role regress_parallel_worker PASSWORD NEON_PASSWORD_PLACEHOLDER; + + create function set_and_report_role() returns text as + $$ select current_setting('role') $$ language sql parallel safe +diff --git a/src/test/regress/sql/select_views.sql b/src/test/regress/sql/select_views.sql +index e742f13699..7bd0255df8 100644 +--- a/src/test/regress/sql/select_views.sql ++++ b/src/test/regress/sql/select_views.sql +@@ -12,7 +12,7 @@ SELECT * FROM toyemp WHERE name = 'sharon'; + -- + -- Test for Leaky view scenario + -- +-CREATE ROLE regress_alice; ++CREATE ROLE regress_alice PASSWORD NEON_PASSWORD_PLACEHOLDER; + + CREATE FUNCTION f_leak (text) + RETURNS bool LANGUAGE 'plpgsql' COST 0.0000001 +diff --git a/src/test/regress/sql/sequence.sql b/src/test/regress/sql/sequence.sql +index 793f1415f6..ec07c1f193 100644 +--- a/src/test/regress/sql/sequence.sql ++++ b/src/test/regress/sql/sequence.sql +@@ -293,7 +293,7 @@ ROLLBACK; + + -- privileges tests + +-CREATE USER regress_seq_user; ++CREATE USER regress_seq_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + + -- nextval + BEGIN; +diff --git a/src/test/regress/sql/stats.sql b/src/test/regress/sql/stats.sql +index d8ac0d06f4..c9cfcea208 100644 +--- a/src/test/regress/sql/stats.sql ++++ b/src/test/regress/sql/stats.sql +@@ -631,23 +631,6 @@ SELECT :io_sum_shared_after_writes > :io_sum_shared_before_writes; + SELECT current_setting('fsync') = 'off' + OR :io_sum_shared_after_fsyncs > :io_sum_shared_before_fsyncs; + +--- Change the tablespace so that the table is rewritten directly, then SELECT +--- from it to cause it to be read back into shared buffers. +-SELECT sum(reads) AS io_sum_shared_before_reads +- FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +--- Do this in a transaction to prevent spurious failures due to concurrent accesses to our newly +--- rewritten table, e.g. by autovacuum. +-BEGIN; +-ALTER TABLE test_io_shared SET TABLESPACE regress_tblspace; +--- SELECT from the table so that the data is read into shared buffers and +--- context 'normal', object 'relation' reads are counted. +-SELECT COUNT(*) FROM test_io_shared; +-COMMIT; +-SELECT pg_stat_force_next_flush(); +-SELECT sum(reads) AS io_sum_shared_after_reads +- FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset +-SELECT :io_sum_shared_after_reads > :io_sum_shared_before_reads; +- + SELECT sum(hits) AS io_sum_shared_before_hits + FROM pg_stat_io WHERE context = 'normal' AND object = 'relation' \gset + -- Select from the table again to count hits. +diff --git a/src/test/regress/sql/stats_ext.sql b/src/test/regress/sql/stats_ext.sql +index 0c08a6cc42..7a5b1036d8 100644 +--- a/src/test/regress/sql/stats_ext.sql ++++ b/src/test/regress/sql/stats_ext.sql +@@ -50,7 +50,7 @@ DROP TABLE ext_stats_test; + CREATE TABLE ab1 (a INTEGER, b INTEGER, c INTEGER); + CREATE STATISTICS IF NOT EXISTS ab1_a_b_stats ON a, b FROM ab1; + COMMENT ON STATISTICS ab1_a_b_stats IS 'new comment'; +-CREATE ROLE regress_stats_ext; ++CREATE ROLE regress_stats_ext PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION regress_stats_ext; + COMMENT ON STATISTICS ab1_a_b_stats IS 'changed comment'; + DROP STATISTICS ab1_a_b_stats; +@@ -1607,7 +1607,7 @@ drop statistics stts_t1_expr_expr_stat; + set search_path to public, stts_s1; + \dX + +-create role regress_stats_ext nosuperuser; ++create role regress_stats_ext nosuperuser PASSWORD NEON_PASSWORD_PLACEHOLDER; + set role regress_stats_ext; + \dX + reset role; +@@ -1618,7 +1618,7 @@ drop user regress_stats_ext; + reset search_path; + + -- User with no access +-CREATE USER regress_stats_user1; ++CREATE USER regress_stats_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT USAGE ON SCHEMA tststats TO regress_stats_user1; + SET SESSION AUTHORIZATION regress_stats_user1; + SELECT * FROM tststats.priv_test_tbl; -- Permission denied +diff --git a/src/test/regress/sql/subscription.sql b/src/test/regress/sql/subscription.sql +index 3e5ba4cb8c..a35f030908 100644 +--- a/src/test/regress/sql/subscription.sql ++++ b/src/test/regress/sql/subscription.sql +@@ -2,10 +2,10 @@ + -- SUBSCRIPTION + -- + +-CREATE ROLE regress_subscription_user LOGIN SUPERUSER; +-CREATE ROLE regress_subscription_user2; +-CREATE ROLE regress_subscription_user3 IN ROLE pg_create_subscription; +-CREATE ROLE regress_subscription_user_dummy LOGIN NOSUPERUSER; ++CREATE ROLE regress_subscription_user LOGIN SUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_subscription_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE ROLE regress_subscription_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER IN ROLE pg_create_subscription; ++CREATE ROLE regress_subscription_user_dummy LOGIN NOSUPERUSER PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET SESSION AUTHORIZATION 'regress_subscription_user'; + + -- fail - no publications +diff --git a/src/test/regress/sql/test_setup.sql b/src/test/regress/sql/test_setup.sql +index 06b0e2121f..01444f9426 100644 +--- a/src/test/regress/sql/test_setup.sql ++++ b/src/test/regress/sql/test_setup.sql +@@ -135,7 +135,8 @@ CREATE TABLE onek ( + ); + + \set filename :abs_srcdir '/data/onek.data' +-COPY onek FROM :'filename'; ++\set command '\\copy onek FROM ' :'filename'; ++:command + VACUUM ANALYZE onek; + + CREATE TABLE onek2 AS SELECT * FROM onek; +@@ -161,7 +162,8 @@ CREATE TABLE tenk1 ( + ); + + \set filename :abs_srcdir '/data/tenk.data' +-COPY tenk1 FROM :'filename'; ++\set command '\\copy tenk1 FROM ' :'filename'; ++:command + VACUUM ANALYZE tenk1; + + CREATE TABLE tenk2 AS SELECT * FROM tenk1; +@@ -174,7 +176,8 @@ CREATE TABLE person ( + ); + + \set filename :abs_srcdir '/data/person.data' +-COPY person FROM :'filename'; ++\set command '\\copy person FROM ' :'filename'; ++:command + VACUUM ANALYZE person; + + CREATE TABLE emp ( +@@ -183,7 +186,8 @@ CREATE TABLE emp ( + ) INHERITS (person); + + \set filename :abs_srcdir '/data/emp.data' +-COPY emp FROM :'filename'; ++\set command '\\copy emp FROM ' :'filename'; ++:command + VACUUM ANALYZE emp; + + CREATE TABLE student ( +@@ -191,7 +195,8 @@ CREATE TABLE student ( + ) INHERITS (person); + + \set filename :abs_srcdir '/data/student.data' +-COPY student FROM :'filename'; ++\set command '\\copy student FROM ' :'filename'; ++:command + VACUUM ANALYZE student; + + CREATE TABLE stud_emp ( +@@ -199,7 +204,8 @@ CREATE TABLE stud_emp ( + ) INHERITS (emp, student); + + \set filename :abs_srcdir '/data/stud_emp.data' +-COPY stud_emp FROM :'filename'; ++\set command '\\copy stud_emp FROM ' :'filename'; ++:command + VACUUM ANALYZE stud_emp; + + CREATE TABLE road ( +@@ -208,7 +214,8 @@ CREATE TABLE road ( + ); + + \set filename :abs_srcdir '/data/streets.data' +-COPY road FROM :'filename'; ++\set command '\\copy road FROM ' :'filename'; ++:command + VACUUM ANALYZE road; + + CREATE TABLE ihighway () INHERITS (road); +diff --git a/src/test/regress/sql/tsearch.sql b/src/test/regress/sql/tsearch.sql +index fbd26cdba4..7ec2d78eee 100644 +--- a/src/test/regress/sql/tsearch.sql ++++ b/src/test/regress/sql/tsearch.sql +@@ -49,7 +49,8 @@ CREATE TABLE test_tsvector( + ); + + \set filename :abs_srcdir '/data/tsearch.data' +-COPY test_tsvector FROM :'filename'; ++\set command '\\copy test_tsvector FROM ' :'filename'; ++:command + + ANALYZE test_tsvector; + +diff --git a/src/test/regress/sql/updatable_views.sql b/src/test/regress/sql/updatable_views.sql +index 93b693ae83..2983475265 100644 +--- a/src/test/regress/sql/updatable_views.sql ++++ b/src/test/regress/sql/updatable_views.sql +@@ -569,9 +569,9 @@ DROP TABLE base_tbl CASCADE; + + -- permissions checks + +-CREATE USER regress_view_user1; +-CREATE USER regress_view_user2; +-CREATE USER regress_view_user3; ++CREATE USER regress_view_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_view_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++CREATE USER regress_view_user3 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + SET SESSION AUTHORIZATION regress_view_user1; + CREATE TABLE base_tbl(a int, b text, c float); +@@ -1909,8 +1909,8 @@ drop view uv_iocu_view; + drop table uv_iocu_tab; + + -- ON CONFLICT DO UPDATE permissions checks +-create user regress_view_user1; +-create user regress_view_user2; ++create user regress_view_user1 PASSWORD NEON_PASSWORD_PLACEHOLDER; ++create user regress_view_user2 PASSWORD NEON_PASSWORD_PLACEHOLDER; + + set session authorization regress_view_user1; + create table base_tbl(a int unique, b text, c float); +diff --git a/src/test/regress/sql/update.sql b/src/test/regress/sql/update.sql +index 8b4707eb9c..b9041f8134 100644 +--- a/src/test/regress/sql/update.sql ++++ b/src/test/regress/sql/update.sql +@@ -342,7 +342,7 @@ DROP FUNCTION func_parted_mod_b(); + ----------------------------------------- + + ALTER TABLE range_parted ENABLE ROW LEVEL SECURITY; +-CREATE USER regress_range_parted_user; ++CREATE USER regress_range_parted_user PASSWORD NEON_PASSWORD_PLACEHOLDER; + GRANT ALL ON range_parted, mintab TO regress_range_parted_user; + CREATE POLICY seeall ON range_parted AS PERMISSIVE FOR SELECT USING (true); + CREATE POLICY policy_range_parted ON range_parted for UPDATE USING (true) WITH CHECK (c % 2 = 0); +diff --git a/src/test/regress/sql/vacuum.sql b/src/test/regress/sql/vacuum.sql +index 548cd7acca..5b15d4dab0 100644 +--- a/src/test/regress/sql/vacuum.sql ++++ b/src/test/regress/sql/vacuum.sql +@@ -335,7 +335,7 @@ CREATE TABLE vacowned (a int); + CREATE TABLE vacowned_parted (a int) PARTITION BY LIST (a); + CREATE TABLE vacowned_part1 PARTITION OF vacowned_parted FOR VALUES IN (1); + CREATE TABLE vacowned_part2 PARTITION OF vacowned_parted FOR VALUES IN (2); +-CREATE ROLE regress_vacuum; ++CREATE ROLE regress_vacuum PASSWORD NEON_PASSWORD_PLACEHOLDER; + SET ROLE regress_vacuum; + -- Simple table + VACUUM vacowned; diff --git a/test_runner/cloud_regress/README.md b/test_runner/cloud_regress/README.md new file mode 100644 index 0000000000..9c460e2764 --- /dev/null +++ b/test_runner/cloud_regress/README.md @@ -0,0 +1,21 @@ +# How to run the `pg_regress` tests on a cloud Neon instance. + +* Create a Neon project on staging. +* Grant the superuser privileges to the DB user. +* (Optional) create a branch for testing +* Configure the endpoint by updating the control-plane database with the following settings: + * `Timeone`: `America/Los_Angeles` + * `DateStyle`: `Postgres,MDY` + * `compute_query_id`: `off` +* Checkout the actual `Neon` sources +* Patch the sql and expected files for the specific PostgreSQL version, e.g. for v17: +```bash +$ cd vendor/postgres-v17 +$ patch -p1 <../../compute/patches/cloud_regress_pg17.patch +``` +* Set the environment variable `BENCHMARK_CONNSTR` to the connection URI of your project. +* Set the environment variable `PG_VERSION` to the version of your project. +* Run +```bash +$ pytest -m remote_cluster -k cloud_regress +``` \ No newline at end of file diff --git a/test_runner/cloud_regress/test_cloud_regress.py b/test_runner/cloud_regress/test_cloud_regress.py index 715d4a4881..63427c1912 100644 --- a/test_runner/cloud_regress/test_cloud_regress.py +++ b/test_runner/cloud_regress/test_cloud_regress.py @@ -5,68 +5,15 @@ Run the regression tests on the cloud instance of Neon from __future__ import annotations from pathlib import Path -from typing import Any -import psycopg2 import pytest -from fixtures.log_helper import log from fixtures.neon_fixtures import RemotePostgres from fixtures.pg_version import PgVersion -@pytest.fixture -def setup(remote_pg: RemotePostgres): - """ - Setup and teardown of the tests - """ - with psycopg2.connect(remote_pg.connstr()) as conn: - with conn.cursor() as cur: - log.info("Creating the extension") - cur.execute("CREATE EXTENSION IF NOT EXISTS regress_so") - conn.commit() - # TODO: Migrate to branches and remove this code - log.info("Looking for subscriptions in the regress database") - cur.execute( - "SELECT subname FROM pg_catalog.pg_subscription WHERE " - "subdbid = (SELECT oid FROM pg_catalog.pg_database WHERE datname='regression');" - ) - if cur.rowcount > 0: - with psycopg2.connect( - dbname="regression", - host=remote_pg.default_options["host"], - user=remote_pg.default_options["user"], - password=remote_pg.default_options["password"], - ) as regress_conn: - with regress_conn.cursor() as regress_cur: - for sub in cur: - regress_cur.execute(f"ALTER SUBSCRIPTION {sub[0]} DISABLE") - regress_cur.execute( - f"ALTER SUBSCRIPTION {sub[0]} SET (slot_name = NONE)" - ) - regress_cur.execute(f"DROP SUBSCRIPTION {sub[0]}") - regress_conn.commit() - - yield - # TODO: Migrate to branches and remove this code - log.info("Looking for extra roles...") - with psycopg2.connect(remote_pg.connstr()) as conn: - with conn.cursor() as cur: - cur.execute( - "SELECT rolname FROM pg_catalog.pg_roles WHERE oid > 16384 AND rolname <> 'neondb_owner'" - ) - roles: list[Any] = [] - for role in cur: - log.info("Role found: %s", role[0]) - roles.append(role[0]) - for role in roles: - cur.execute(f"DROP ROLE {role}") - conn.commit() - - @pytest.mark.timeout(7200) @pytest.mark.remote_cluster def test_cloud_regress( - setup, remote_pg: RemotePostgres, pg_version: PgVersion, pg_distrib_dir: Path, From 4c4cb80186de6d319716897117eaf5c434d5180f Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:06:06 -0500 Subject: [PATCH 03/44] fix(pageserver): fix gc-compaction racing with legacy gc (#10052) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem close https://github.com/neondatabase/neon/issues/10049, close https://github.com/neondatabase/neon/issues/10030, close https://github.com/neondatabase/neon/issues/8861 part of https://github.com/neondatabase/neon/issues/9114 The legacy gc process calls `get_latest_gc_cutoff`, which uses a Rcu different than the gc_info struct. In the gc_compaction_smoke test case, the "latest" cutoff could be lower than the gc_info struct, causing gc-compaction to collect data that could be accessed by `latest_gc_cutoff`. Technically speaking, there's nothing wrong with gc-compaction using gc_info without considering latest_gc_cutoff, because gc_info is the source of truth. But anyways, let's fix it. ## Summary of changes * gc-compaction uses `latest_gc_cutoff` instead of gc_info to determine the gc horizon. * if a gc-compaction is scheduled via tenant compaction iteration, it will take the gc_block lock to avoid racing with functionalities like detach ancestor (if it's triggered via manual compaction API without scheduling, then it won't take the lock) --------- Signed-off-by: Alex Chi Z Co-authored-by: Arpad Müller --- pageserver/src/http/routes.rs | 2 +- pageserver/src/tenant.rs | 96 ++++++++++++++++++-- pageserver/src/tenant/gc_block.rs | 12 +-- pageserver/src/tenant/timeline/compaction.rs | 16 +++- test_runner/regress/test_compaction.py | 33 +++---- 5 files changed, 121 insertions(+), 38 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 0f11bbc507..75d25d0a6a 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -2061,7 +2061,7 @@ async fn timeline_compact_handler( let tenant = state .tenant_manager .get_attached_tenant_shard(tenant_shard_id)?; - let rx = tenant.schedule_compaction(timeline_id, options).await; + let rx = tenant.schedule_compaction(timeline_id, options).await.map_err(ApiError::InternalServerError)?; if wait_until_scheduled_compaction_done { // It is possible that this will take a long time, dropping the HTTP request will not cancel the compaction. rx.await.ok(); diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 4a9c44aefd..e71a56ed40 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3028,14 +3028,23 @@ impl Tenant { let mut guard = self.scheduled_compaction_tasks.lock().unwrap(); let tline_pending_tasks = guard.entry(*timeline_id).or_default(); for (idx, job) in jobs.into_iter().enumerate() { - tline_pending_tasks.push_back(ScheduledCompactionTask { - options: job, - result_tx: if idx == jobs_len - 1 { - // The last compaction job sends the completion signal - next_scheduled_compaction_task.result_tx.take() - } else { - None - }, + tline_pending_tasks.push_back(if idx == jobs_len - 1 { + ScheduledCompactionTask { + options: job, + // The last job in the queue sends the signal and releases the gc guard + result_tx: next_scheduled_compaction_task + .result_tx + .take(), + gc_block: next_scheduled_compaction_task + .gc_block + .take(), + } + } else { + ScheduledCompactionTask { + options: job, + result_tx: None, + gc_block: None, + } }); } info!("scheduled enhanced gc bottom-most compaction with sub-compaction, split into {} jobs", jobs_len); @@ -3095,15 +3104,22 @@ impl Tenant { &self, timeline_id: TimelineId, options: CompactOptions, - ) -> tokio::sync::oneshot::Receiver<()> { + ) -> anyhow::Result> { + let gc_guard = match self.gc_block.start().await { + Ok(guard) => guard, + Err(e) => { + bail!("cannot run gc-compaction because gc is blocked: {}", e); + } + }; let (tx, rx) = tokio::sync::oneshot::channel(); let mut guard = self.scheduled_compaction_tasks.lock().unwrap(); let tline_pending_tasks = guard.entry(timeline_id).or_default(); tline_pending_tasks.push_back(ScheduledCompactionTask { options, result_tx: Some(tx), + gc_block: Some(gc_guard), }); - rx + Ok(rx) } // Call through to all timelines to freeze ephemeral layers if needed. Usually @@ -8150,6 +8166,12 @@ mod tests { ) .await?; { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x30)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); guard.cutoffs.time = Lsn(0x30); @@ -8252,6 +8274,12 @@ mod tests { // increase GC horizon and compact again { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x40)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); guard.cutoffs.time = Lsn(0x40); @@ -8632,6 +8660,12 @@ mod tests { .await? }; { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x30)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); *guard = GcInfo { @@ -8713,6 +8747,12 @@ mod tests { // increase GC horizon and compact again { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x40)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); guard.cutoffs.time = Lsn(0x40); @@ -9160,6 +9200,12 @@ mod tests { ) .await?; { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x30)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); *guard = GcInfo { @@ -9302,6 +9348,12 @@ mod tests { // increase GC horizon and compact again { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x38)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); guard.cutoffs.time = Lsn(0x38); @@ -9397,6 +9449,12 @@ mod tests { ) .await?; { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x30)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); *guard = GcInfo { @@ -9641,6 +9699,12 @@ mod tests { branch_tline.add_extra_test_dense_keyspace(KeySpace::single(get_key(0)..get_key(10))); { + parent_tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x10)) + .wait() + .await; // Update GC info let mut guard = parent_tline.gc_info.write().unwrap(); *guard = GcInfo { @@ -9655,6 +9719,12 @@ mod tests { } { + branch_tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x50)) + .wait() + .await; // Update GC info let mut guard = branch_tline.gc_info.write().unwrap(); *guard = GcInfo { @@ -9984,6 +10054,12 @@ mod tests { .await?; { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x30)) + .wait() + .await; // Update GC info let mut guard = tline.gc_info.write().unwrap(); *guard = GcInfo { diff --git a/pageserver/src/tenant/gc_block.rs b/pageserver/src/tenant/gc_block.rs index 373779ddb8..af73acb2be 100644 --- a/pageserver/src/tenant/gc_block.rs +++ b/pageserver/src/tenant/gc_block.rs @@ -1,4 +1,4 @@ -use std::collections::HashMap; +use std::{collections::HashMap, sync::Arc}; use utils::id::TimelineId; @@ -20,7 +20,7 @@ pub(crate) struct GcBlock { /// Do not add any more features taking and forbidding taking this lock. It should be /// `tokio::sync::Notify`, but that is rarely used. On the other side, [`GcBlock::insert`] /// synchronizes with gc attempts by locking and unlocking this mutex. - blocking: tokio::sync::Mutex<()>, + blocking: Arc>, } impl GcBlock { @@ -30,7 +30,7 @@ impl GcBlock { /// it's ending, or if not currently possible, a value describing the reasons why not. /// /// Cancellation safe. - pub(super) async fn start(&self) -> Result, BlockingReasons> { + pub(super) async fn start(&self) -> Result { let reasons = { let g = self.reasons.lock().unwrap(); @@ -44,7 +44,7 @@ impl GcBlock { Err(reasons) } else { Ok(Guard { - _inner: self.blocking.lock().await, + _inner: self.blocking.clone().lock_owned().await, }) } } @@ -170,8 +170,8 @@ impl GcBlock { } } -pub(super) struct Guard<'a> { - _inner: tokio::sync::MutexGuard<'a, ()>, +pub(crate) struct Guard { + _inner: tokio::sync::OwnedMutexGuard<()>, } #[derive(Debug)] diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index a18e157d37..fa924d23b0 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -41,7 +41,7 @@ use crate::tenant::storage_layer::{ use crate::tenant::timeline::ImageLayerCreationOutcome; use crate::tenant::timeline::{drop_rlock, DeltaLayerWriter, ImageLayerWriter}; use crate::tenant::timeline::{Layer, ResidentLayer}; -use crate::tenant::{DeltaLayer, MaybeOffloaded}; +use crate::tenant::{gc_block, DeltaLayer, MaybeOffloaded}; use crate::virtual_file::{MaybeFatalIo, VirtualFile}; use pageserver_api::config::tenant_conf_defaults::{ DEFAULT_CHECKPOINT_DISTANCE, DEFAULT_COMPACTION_THRESHOLD, @@ -63,9 +63,12 @@ use super::CompactionError; const COMPACTION_DELTA_THRESHOLD: usize = 5; /// A scheduled compaction task. -pub struct ScheduledCompactionTask { +pub(crate) struct ScheduledCompactionTask { pub options: CompactOptions, + /// The channel to send the compaction result. If this is a subcompaction, the last compaction job holds the sender. pub result_tx: Option>, + /// Hold the GC block. If this is a subcompaction, the last compaction job holds the gc block guard. + pub gc_block: Option, } pub struct GcCompactionJobDescription { @@ -1768,8 +1771,7 @@ impl Timeline { let compact_below_lsn = if let Some(compact_below_lsn) = options.compact_below_lsn { compact_below_lsn } else { - let gc_info = self.gc_info.read().unwrap(); - gc_info.cutoffs.select_min() // use the real gc cutoff + *self.get_latest_gc_cutoff_lsn() // use the real gc cutoff }; let mut compact_jobs = Vec::new(); // For now, we simply use the key partitioning information; we should do a more fine-grained partitioning @@ -1962,7 +1964,11 @@ impl Timeline { let gc_info = self.gc_info.read().unwrap(); let mut retain_lsns_below_horizon = Vec::new(); let gc_cutoff = { - let real_gc_cutoff = gc_info.cutoffs.select_min(); + // Currently, gc-compaction only kicks in after the legacy gc has updated the gc_cutoff. + // Therefore, it can only clean up data that cannot be cleaned up with legacy gc, instead of + // cleaning everything that theoritically it could. In the future, it should use `self.gc_info` + // to get the truth data. + let real_gc_cutoff = *self.get_latest_gc_cutoff_lsn(); // The compaction algorithm will keep all keys above the gc_cutoff while keeping only necessary keys below the gc_cutoff for // each of the retain_lsn. Therefore, if the user-provided `compact_below_lsn` is larger than the real gc cutoff, we will use // the real cutoff. diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index 881503046c..810a9723e0 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -121,9 +121,6 @@ page_cache_size=10 assert vectored_average < 8 -@pytest.mark.skip( - "This is being fixed and tracked in https://github.com/neondatabase/neon/issues/9114" -) @skip_in_debug_build("only run with release build") def test_pageserver_gc_compaction_smoke(neon_env_builder: NeonEnvBuilder): SMOKE_CONF = { @@ -156,20 +153,20 @@ def test_pageserver_gc_compaction_smoke(neon_env_builder: NeonEnvBuilder): if i % 10 == 0: log.info(f"Running churn round {i}/{churn_rounds} ...") - ps_http.timeline_compact( - tenant_id, - timeline_id, - enhanced_gc_bottom_most_compaction=True, - body={ - "scheduled": True, - "sub_compaction": True, - "compact_range": { - "start": "000000000000000000000000000000000000", - # skip the SLRU range for now -- it races with get-lsn-by-timestamp, TODO: fix this - "end": "010000000000000000000000000000000000", + # Run gc-compaction every 10 rounds to ensure the test doesn't take too long time. + ps_http.timeline_compact( + tenant_id, + timeline_id, + enhanced_gc_bottom_most_compaction=True, + body={ + "scheduled": True, + "sub_compaction": True, + "compact_range": { + "start": "000000000000000000000000000000000000", + "end": "030000000000000000000000000000000000", + }, }, - }, - ) + ) workload.churn_rows(row_count, env.pageserver.id) @@ -181,6 +178,10 @@ def test_pageserver_gc_compaction_smoke(neon_env_builder: NeonEnvBuilder): log.info("Validating at workload end ...") workload.validate(env.pageserver.id) + # Run a legacy compaction+gc to ensure gc-compaction can coexist with legacy compaction. + ps_http.timeline_checkpoint(tenant_id, timeline_id, wait_until_uploaded=True) + ps_http.timeline_gc(tenant_id, timeline_id, None) + # Stripe sizes in number of pages. TINY_STRIPES = 16 From b593e51eaea6ec53af22503332c49488ff9bb055 Mon Sep 17 00:00:00 2001 From: Evan Fleming Date: Mon, 9 Dec 2024 13:09:20 -0800 Subject: [PATCH 04/44] safekeeper: use arc for global timelines and config (#10051) Hello! I was interested in potentially making some contributions to Neon and looking through the issue backlog I found [8200](https://github.com/neondatabase/neon/issues/8200) which seemed like a good first issue to attempt to tackle. I see it was assigned a while ago so apologies if I'm stepping on any toes with this PR. I also apologize for the size of this PR. I'm not sure if there is a simple way to reduce it given the footprint of the components being changed. ## Problem This PR is attempting to address part of the problem outlined in issue [8200](https://github.com/neondatabase/neon/issues/8200). Namely to remove global static usage of timeline state in favour of `Arc` and to replace wasteful clones of `SafeKeeperConf` with `Arc`. I did not opt to tackle `RemoteStorage` in this PR to minimize the amount of changes as this PR is already quite large. I also did not opt to introduce an `SafekeeperApp` wrapper struct to similarly minimize changes but I can tackle either or both of these omissions in this PR if folks would like. ## Summary of changes - Remove static usage of `GlobalTimelines` in favour of `Arc` - Wrap `SafeKeeperConf` in `Arc` to avoid wasteful clones of the underlying struct ## Some additional thoughts - We seem to currently store `SafeKeeperConf` in `GlobalTimelines` and then expose it through a public`get_global_config` function which requires locking. This seems needlessly wasteful and based on observed usage we could remove this public accessor and force consumers to acquire `SafeKeeperConf` through the new Arc reference. --- safekeeper/benches/benchutils.rs | 10 +- safekeeper/src/bin/safekeeper.rs | 36 ++++--- safekeeper/src/broker.rs | 37 ++++--- safekeeper/src/copy_timeline.rs | 26 ++--- safekeeper/src/debug_dump.rs | 18 ++-- safekeeper/src/handler.rs | 14 ++- safekeeper/src/http/mod.rs | 8 +- safekeeper/src/http/routes.rs | 74 +++++++++----- safekeeper/src/json_ctrl.rs | 30 +++--- safekeeper/src/metrics.rs | 19 ++-- safekeeper/src/pull_timeline.rs | 12 ++- safekeeper/src/receive_wal.rs | 11 ++- safekeeper/src/send_wal.rs | 6 +- safekeeper/src/timeline.rs | 25 +++-- safekeeper/src/timelines_global_map.rs | 128 ++++++++++++------------- safekeeper/src/wal_service.rs | 22 +++-- 16 files changed, 283 insertions(+), 193 deletions(-) diff --git a/safekeeper/benches/benchutils.rs b/safekeeper/benches/benchutils.rs index 4e8dc58c49..48d796221b 100644 --- a/safekeeper/benches/benchutils.rs +++ b/safekeeper/benches/benchutils.rs @@ -83,14 +83,20 @@ impl Env { node_id: NodeId, ttid: TenantTimelineId, ) -> anyhow::Result> { - let conf = self.make_conf(node_id); + let conf = Arc::new(self.make_conf(node_id)); let timeline_dir = get_timeline_dir(&conf, &ttid); let remote_path = remote_timeline_path(&ttid)?; let safekeeper = self.make_safekeeper(node_id, ttid).await?; let shared_state = SharedState::new(StateSK::Loaded(safekeeper)); - let timeline = Timeline::new(ttid, &timeline_dir, &remote_path, shared_state); + let timeline = Timeline::new( + ttid, + &timeline_dir, + &remote_path, + shared_state, + conf.clone(), + ); timeline.bootstrap( &mut timeline.write_shared_state().await, &conf, diff --git a/safekeeper/src/bin/safekeeper.rs b/safekeeper/src/bin/safekeeper.rs index 4dc7edef37..13f6e34575 100644 --- a/safekeeper/src/bin/safekeeper.rs +++ b/safekeeper/src/bin/safekeeper.rs @@ -338,7 +338,7 @@ async fn main() -> anyhow::Result<()> { } }; - let conf = SafeKeeperConf { + let conf = Arc::new(SafeKeeperConf { workdir, my_id: id, listen_pg_addr: args.listen_pg, @@ -368,7 +368,7 @@ async fn main() -> anyhow::Result<()> { control_file_save_interval: args.control_file_save_interval, partial_backup_concurrency: args.partial_backup_concurrency, eviction_min_resident: args.eviction_min_resident, - }; + }); // initialize sentry if SENTRY_DSN is provided let _sentry_guard = init_sentry( @@ -382,7 +382,7 @@ async fn main() -> anyhow::Result<()> { /// complete, e.g. panicked, inner is error produced by task itself. type JoinTaskRes = Result, JoinError>; -async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { +async fn start_safekeeper(conf: Arc) -> Result<()> { // fsync the datadir to make sure we have a consistent state on disk. if !conf.no_sync { let dfd = File::open(&conf.workdir).context("open datadir for syncfs")?; @@ -428,9 +428,11 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { e })?; + let global_timelines = Arc::new(GlobalTimelines::new(conf.clone())); + // Register metrics collector for active timelines. It's important to do this // after daemonizing, otherwise process collector will be upset. - let timeline_collector = safekeeper::metrics::TimelineCollector::new(); + let timeline_collector = safekeeper::metrics::TimelineCollector::new(global_timelines.clone()); metrics::register_internal(Box::new(timeline_collector))?; wal_backup::init_remote_storage(&conf).await; @@ -447,9 +449,8 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { .then(|| Handle::try_current().expect("no runtime in main")); // Load all timelines from disk to memory. - GlobalTimelines::init(conf.clone()).await?; + global_timelines.init().await?; - let conf_ = conf.clone(); // Run everything in current thread rt, if asked. if conf.current_thread_runtime { info!("running in current thread runtime"); @@ -459,14 +460,16 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { .as_ref() .unwrap_or_else(|| WAL_SERVICE_RUNTIME.handle()) .spawn(wal_service::task_main( - conf_, + conf.clone(), pg_listener, Scope::SafekeeperData, + global_timelines.clone(), )) // wrap with task name for error reporting .map(|res| ("WAL service main".to_owned(), res)); tasks_handles.push(Box::pin(wal_service_handle)); + let global_timelines_ = global_timelines.clone(); let timeline_housekeeping_handle = current_thread_rt .as_ref() .unwrap_or_else(|| WAL_SERVICE_RUNTIME.handle()) @@ -474,40 +477,45 @@ async fn start_safekeeper(conf: SafeKeeperConf) -> Result<()> { const TOMBSTONE_TTL: Duration = Duration::from_secs(3600 * 24); loop { tokio::time::sleep(TOMBSTONE_TTL).await; - GlobalTimelines::housekeeping(&TOMBSTONE_TTL); + global_timelines_.housekeeping(&TOMBSTONE_TTL); } }) .map(|res| ("Timeline map housekeeping".to_owned(), res)); tasks_handles.push(Box::pin(timeline_housekeeping_handle)); if let Some(pg_listener_tenant_only) = pg_listener_tenant_only { - let conf_ = conf.clone(); let wal_service_handle = current_thread_rt .as_ref() .unwrap_or_else(|| WAL_SERVICE_RUNTIME.handle()) .spawn(wal_service::task_main( - conf_, + conf.clone(), pg_listener_tenant_only, Scope::Tenant, + global_timelines.clone(), )) // wrap with task name for error reporting .map(|res| ("WAL service tenant only main".to_owned(), res)); tasks_handles.push(Box::pin(wal_service_handle)); } - let conf_ = conf.clone(); let http_handle = current_thread_rt .as_ref() .unwrap_or_else(|| HTTP_RUNTIME.handle()) - .spawn(http::task_main(conf_, http_listener)) + .spawn(http::task_main( + conf.clone(), + http_listener, + global_timelines.clone(), + )) .map(|res| ("HTTP service main".to_owned(), res)); tasks_handles.push(Box::pin(http_handle)); - let conf_ = conf.clone(); let broker_task_handle = current_thread_rt .as_ref() .unwrap_or_else(|| BROKER_RUNTIME.handle()) - .spawn(broker::task_main(conf_).instrument(info_span!("broker"))) + .spawn( + broker::task_main(conf.clone(), global_timelines.clone()) + .instrument(info_span!("broker")), + ) .map(|res| ("broker main".to_owned(), res)); tasks_handles.push(Box::pin(broker_task_handle)); diff --git a/safekeeper/src/broker.rs b/safekeeper/src/broker.rs index 485816408f..4b091e2c29 100644 --- a/safekeeper/src/broker.rs +++ b/safekeeper/src/broker.rs @@ -39,14 +39,17 @@ const RETRY_INTERVAL_MSEC: u64 = 1000; const PUSH_INTERVAL_MSEC: u64 = 1000; /// Push once in a while data about all active timelines to the broker. -async fn push_loop(conf: SafeKeeperConf) -> anyhow::Result<()> { +async fn push_loop( + conf: Arc, + global_timelines: Arc, +) -> anyhow::Result<()> { if conf.disable_periodic_broker_push { info!("broker push_loop is disabled, doing nothing..."); futures::future::pending::<()>().await; // sleep forever return Ok(()); } - let active_timelines_set = GlobalTimelines::get_global_broker_active_set(); + let active_timelines_set = global_timelines.get_global_broker_active_set(); let mut client = storage_broker::connect(conf.broker_endpoint.clone(), conf.broker_keepalive_interval)?; @@ -87,8 +90,13 @@ async fn push_loop(conf: SafeKeeperConf) -> anyhow::Result<()> { /// Subscribe and fetch all the interesting data from the broker. #[instrument(name = "broker_pull", skip_all)] -async fn pull_loop(conf: SafeKeeperConf, stats: Arc) -> Result<()> { - let mut client = storage_broker::connect(conf.broker_endpoint, conf.broker_keepalive_interval)?; +async fn pull_loop( + conf: Arc, + global_timelines: Arc, + stats: Arc, +) -> Result<()> { + let mut client = + storage_broker::connect(conf.broker_endpoint.clone(), conf.broker_keepalive_interval)?; // TODO: subscribe only to local timelines instead of all let request = SubscribeSafekeeperInfoRequest { @@ -113,7 +121,7 @@ async fn pull_loop(conf: SafeKeeperConf, stats: Arc) -> Result<()> .as_ref() .ok_or_else(|| anyhow!("missing tenant_timeline_id"))?; let ttid = parse_proto_ttid(proto_ttid)?; - if let Ok(tli) = GlobalTimelines::get(ttid) { + if let Ok(tli) = global_timelines.get(ttid) { // Note that we also receive *our own* info. That's // important, as it is used as an indication of live // connection to the broker. @@ -135,7 +143,11 @@ async fn pull_loop(conf: SafeKeeperConf, stats: Arc) -> Result<()> /// Process incoming discover requests. This is done in a separate task to avoid /// interfering with the normal pull/push loops. -async fn discover_loop(conf: SafeKeeperConf, stats: Arc) -> Result<()> { +async fn discover_loop( + conf: Arc, + global_timelines: Arc, + stats: Arc, +) -> Result<()> { let mut client = storage_broker::connect(conf.broker_endpoint.clone(), conf.broker_keepalive_interval)?; @@ -171,7 +183,7 @@ async fn discover_loop(conf: SafeKeeperConf, stats: Arc) -> Result< .as_ref() .ok_or_else(|| anyhow!("missing tenant_timeline_id"))?; let ttid = parse_proto_ttid(proto_ttid)?; - if let Ok(tli) = GlobalTimelines::get(ttid) { + if let Ok(tli) = global_timelines.get(ttid) { // we received a discovery request for a timeline we know about discover_counter.inc(); @@ -210,7 +222,10 @@ async fn discover_loop(conf: SafeKeeperConf, stats: Arc) -> Result< bail!("end of stream"); } -pub async fn task_main(conf: SafeKeeperConf) -> anyhow::Result<()> { +pub async fn task_main( + conf: Arc, + global_timelines: Arc, +) -> anyhow::Result<()> { info!("started, broker endpoint {:?}", conf.broker_endpoint); let mut ticker = tokio::time::interval(Duration::from_millis(RETRY_INTERVAL_MSEC)); @@ -261,13 +276,13 @@ pub async fn task_main(conf: SafeKeeperConf) -> anyhow::Result<()> { }, _ = ticker.tick() => { if push_handle.is_none() { - push_handle = Some(tokio::spawn(push_loop(conf.clone()))); + push_handle = Some(tokio::spawn(push_loop(conf.clone(), global_timelines.clone()))); } if pull_handle.is_none() { - pull_handle = Some(tokio::spawn(pull_loop(conf.clone(), stats.clone()))); + pull_handle = Some(tokio::spawn(pull_loop(conf.clone(), global_timelines.clone(), stats.clone()))); } if discover_handle.is_none() { - discover_handle = Some(tokio::spawn(discover_loop(conf.clone(), stats.clone()))); + discover_handle = Some(tokio::spawn(discover_loop(conf.clone(), global_timelines.clone(), stats.clone()))); } }, _ = &mut stats_task => {} diff --git a/safekeeper/src/copy_timeline.rs b/safekeeper/src/copy_timeline.rs index 07fa98212f..28ef2b1d23 100644 --- a/safekeeper/src/copy_timeline.rs +++ b/safekeeper/src/copy_timeline.rs @@ -1,9 +1,7 @@ -use std::sync::Arc; - use anyhow::{bail, Result}; use camino::Utf8PathBuf; - use postgres_ffi::{MAX_SEND_SIZE, WAL_SEGMENT_SIZE}; +use std::sync::Arc; use tokio::{ fs::OpenOptions, io::{AsyncSeekExt, AsyncWriteExt}, @@ -14,7 +12,7 @@ use utils::{id::TenantTimelineId, lsn::Lsn}; use crate::{ control_file::FileStorage, state::TimelinePersistentState, - timeline::{Timeline, TimelineError, WalResidentTimeline}, + timeline::{TimelineError, WalResidentTimeline}, timelines_global_map::{create_temp_timeline_dir, validate_temp_timeline}, wal_backup::copy_s3_segments, wal_storage::{wal_file_paths, WalReader}, @@ -25,16 +23,19 @@ use crate::{ const MAX_BACKUP_LAG: u64 = 10 * WAL_SEGMENT_SIZE as u64; pub struct Request { - pub source: Arc, + pub source_ttid: TenantTimelineId, pub until_lsn: Lsn, pub destination_ttid: TenantTimelineId, } -pub async fn handle_request(request: Request) -> Result<()> { +pub async fn handle_request( + request: Request, + global_timelines: Arc, +) -> Result<()> { // TODO: request.until_lsn MUST be a valid LSN, and we cannot check it :( // if LSN will point to the middle of a WAL record, timeline will be in "broken" state - match GlobalTimelines::get(request.destination_ttid) { + match global_timelines.get(request.destination_ttid) { // timeline already exists. would be good to check that this timeline is the copy // of the source timeline, but it isn't obvious how to do that Ok(_) => return Ok(()), @@ -46,9 +47,10 @@ pub async fn handle_request(request: Request) -> Result<()> { } } - let source_tli = request.source.wal_residence_guard().await?; + let source = global_timelines.get(request.source_ttid)?; + let source_tli = source.wal_residence_guard().await?; - let conf = &GlobalTimelines::get_global_config(); + let conf = &global_timelines.get_global_config(); let ttid = request.destination_ttid; let (_tmp_dir, tli_dir_path) = create_temp_timeline_dir(conf, ttid).await?; @@ -127,7 +129,7 @@ pub async fn handle_request(request: Request) -> Result<()> { copy_s3_segments( wal_seg_size, - &request.source.ttid, + &request.source_ttid, &request.destination_ttid, first_segment, first_ondisk_segment, @@ -158,7 +160,9 @@ pub async fn handle_request(request: Request) -> Result<()> { // now we have a ready timeline in a temp directory validate_temp_timeline(conf, request.destination_ttid, &tli_dir_path).await?; - GlobalTimelines::load_temp_timeline(request.destination_ttid, &tli_dir_path, true).await?; + global_timelines + .load_temp_timeline(request.destination_ttid, &tli_dir_path, true) + .await?; Ok(()) } diff --git a/safekeeper/src/debug_dump.rs b/safekeeper/src/debug_dump.rs index a2d0c49768..93011eddec 100644 --- a/safekeeper/src/debug_dump.rs +++ b/safekeeper/src/debug_dump.rs @@ -207,23 +207,23 @@ pub struct FileInfo { } /// Build debug dump response, using the provided [`Args`] filters. -pub async fn build(args: Args) -> Result { +pub async fn build(args: Args, global_timelines: Arc) -> Result { let start_time = Utc::now(); - let timelines_count = GlobalTimelines::timelines_count(); - let config = GlobalTimelines::get_global_config(); + let timelines_count = global_timelines.timelines_count(); + let config = global_timelines.get_global_config(); let ptrs_snapshot = if args.tenant_id.is_some() && args.timeline_id.is_some() { // If both tenant_id and timeline_id are specified, we can just get the // timeline directly, without taking a snapshot of the whole list. let ttid = TenantTimelineId::new(args.tenant_id.unwrap(), args.timeline_id.unwrap()); - if let Ok(tli) = GlobalTimelines::get(ttid) { + if let Ok(tli) = global_timelines.get(ttid) { vec![tli] } else { vec![] } } else { // Otherwise, take a snapshot of the whole list. - GlobalTimelines::get_all() + global_timelines.get_all() }; let mut timelines = Vec::new(); @@ -344,12 +344,12 @@ fn get_wal_last_modified(path: &Utf8Path) -> Result>> { /// Converts SafeKeeperConf to Config, filtering out the fields that are not /// supposed to be exposed. -fn build_config(config: SafeKeeperConf) -> Config { +fn build_config(config: Arc) -> Config { Config { id: config.my_id, - workdir: config.workdir.into(), - listen_pg_addr: config.listen_pg_addr, - listen_http_addr: config.listen_http_addr, + workdir: config.workdir.clone().into(), + listen_pg_addr: config.listen_pg_addr.clone(), + listen_http_addr: config.listen_http_addr.clone(), no_sync: config.no_sync, max_offloader_lag_bytes: config.max_offloader_lag_bytes, wal_backup_enabled: config.wal_backup_enabled, diff --git a/safekeeper/src/handler.rs b/safekeeper/src/handler.rs index 8dd2929a03..2ca6333ba8 100644 --- a/safekeeper/src/handler.rs +++ b/safekeeper/src/handler.rs @@ -33,7 +33,7 @@ use utils::{ /// Safekeeper handler of postgres commands pub struct SafekeeperPostgresHandler { - pub conf: SafeKeeperConf, + pub conf: Arc, /// assigned application name pub appname: Option, pub tenant_id: Option, @@ -43,6 +43,7 @@ pub struct SafekeeperPostgresHandler { pub protocol: Option, /// Unique connection id is logged in spans for observability. pub conn_id: ConnectionId, + pub global_timelines: Arc, /// Auth scope allowed on the connections and public key used to check auth tokens. None if auth is not configured. auth: Option<(Scope, Arc)>, claims: Option, @@ -314,10 +315,11 @@ impl postgres_backend::Handler impl SafekeeperPostgresHandler { pub fn new( - conf: SafeKeeperConf, + conf: Arc, conn_id: u32, io_metrics: Option, auth: Option<(Scope, Arc)>, + global_timelines: Arc, ) -> Self { SafekeeperPostgresHandler { conf, @@ -331,6 +333,7 @@ impl SafekeeperPostgresHandler { claims: None, auth, io_metrics, + global_timelines, } } @@ -360,7 +363,7 @@ impl SafekeeperPostgresHandler { pgb: &mut PostgresBackend, ) -> Result<(), QueryError> { // Get timeline, handling "not found" error - let tli = match GlobalTimelines::get(self.ttid) { + let tli = match self.global_timelines.get(self.ttid) { Ok(tli) => Ok(Some(tli)), Err(TimelineError::NotFound(_)) => Ok(None), Err(e) => Err(QueryError::Other(e.into())), @@ -394,7 +397,10 @@ impl SafekeeperPostgresHandler { &mut self, pgb: &mut PostgresBackend, ) -> Result<(), QueryError> { - let tli = GlobalTimelines::get(self.ttid).map_err(|e| QueryError::Other(e.into()))?; + let tli = self + .global_timelines + .get(self.ttid) + .map_err(|e| QueryError::Other(e.into()))?; let lsn = if self.is_walproposer_recovery() { // walproposer should get all local WAL until flush_lsn diff --git a/safekeeper/src/http/mod.rs b/safekeeper/src/http/mod.rs index 52fb13ff5b..7229ccb739 100644 --- a/safekeeper/src/http/mod.rs +++ b/safekeeper/src/http/mod.rs @@ -3,14 +3,16 @@ pub mod routes; pub use routes::make_router; pub use safekeeper_api::models; +use std::sync::Arc; -use crate::SafeKeeperConf; +use crate::{GlobalTimelines, SafeKeeperConf}; pub async fn task_main( - conf: SafeKeeperConf, + conf: Arc, http_listener: std::net::TcpListener, + global_timelines: Arc, ) -> anyhow::Result<()> { - let router = make_router(conf) + let router = make_router(conf, global_timelines) .build() .map_err(|err| anyhow::anyhow!(err))?; let service = utils::http::RouterService::new(router).unwrap(); diff --git a/safekeeper/src/http/routes.rs b/safekeeper/src/http/routes.rs index 69b775fd76..71c36f1d46 100644 --- a/safekeeper/src/http/routes.rs +++ b/safekeeper/src/http/routes.rs @@ -66,6 +66,13 @@ fn get_conf(request: &Request) -> &SafeKeeperConf { .as_ref() } +fn get_global_timelines(request: &Request) -> Arc { + request + .data::>() + .expect("unknown state type") + .clone() +} + /// Same as TermLsn, but serializes LSN using display serializer /// in Postgres format, i.e. 0/FFFFFFFF. Used only for the API response. #[derive(Debug, Clone, Copy, Serialize, Deserialize)] @@ -123,9 +130,11 @@ async fn tenant_delete_handler(mut request: Request) -> Result) -> Result) -> Result) -> Result, ApiError> { check_permission(&request, None)?; - let res: Vec = GlobalTimelines::get_all() + let global_timelines = get_global_timelines(&request); + let res: Vec = global_timelines + .get_all() .iter() .map(|tli| tli.ttid) .collect(); @@ -182,7 +195,8 @@ async fn timeline_status_handler(request: Request) -> Result) -> Result) -> Result) -> Result, // so create the chan and write to it in another task. @@ -293,19 +311,19 @@ async fn timeline_copy_handler(mut request: Request) -> Result) -> Result) -> Result = parse_query_param(&request, "from_lsn")?; let until_lsn: Option = parse_query_param(&request, "until_lsn")?; @@ -371,7 +392,7 @@ async fn timeline_digest_handler(request: Request) -> Result) -> Result) -> Result) -> Result let dump_term_history = dump_term_history.unwrap_or(true); let dump_wal_last_modified = dump_wal_last_modified.unwrap_or(dump_all); + let global_timelines = get_global_timelines(&request); + let args = debug_dump::Args { dump_all, dump_control_file, @@ -517,7 +543,7 @@ async fn dump_debug_handler(mut request: Request) -> Result timeline_id, }; - let resp = debug_dump::build(args) + let resp = debug_dump::build(args, global_timelines) .await .map_err(ApiError::InternalServerError)?; @@ -570,7 +596,10 @@ async fn dump_debug_handler(mut request: Request) -> Result } /// Safekeeper http router. -pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder { +pub fn make_router( + conf: Arc, + global_timelines: Arc, +) -> RouterBuilder { let mut router = endpoint::make_router(); if conf.http_auth.is_some() { router = router.middleware(auth_middleware(|request| { @@ -592,7 +621,8 @@ pub fn make_router(conf: SafeKeeperConf) -> RouterBuilder // located nearby (/safekeeper/src/http/openapi_spec.yaml). let auth = conf.http_auth.clone(); router - .data(Arc::new(conf)) + .data(conf) + .data(global_timelines) .data(auth) .get("/metrics", |r| request_span(r, prometheus_metrics_handler)) .get("/profile/cpu", |r| request_span(r, profile_cpu_handler)) diff --git a/safekeeper/src/json_ctrl.rs b/safekeeper/src/json_ctrl.rs index 0573ea81e7..dc4ad3706e 100644 --- a/safekeeper/src/json_ctrl.rs +++ b/safekeeper/src/json_ctrl.rs @@ -11,7 +11,6 @@ use postgres_backend::QueryError; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncRead, AsyncWrite}; use tracing::*; -use utils::id::TenantTimelineId; use crate::handler::SafekeeperPostgresHandler; use crate::safekeeper::{AcceptorProposerMessage, AppendResponse, ServerInfo}; @@ -21,7 +20,6 @@ use crate::safekeeper::{ use crate::safekeeper::{Term, TermHistory, TermLsn}; use crate::state::TimelinePersistentState; use crate::timeline::WalResidentTimeline; -use crate::GlobalTimelines; use postgres_backend::PostgresBackend; use postgres_ffi::encode_logical_message; use postgres_ffi::WAL_SEGMENT_SIZE; @@ -70,7 +68,7 @@ pub async fn handle_json_ctrl( info!("JSON_CTRL request: {append_request:?}"); // need to init safekeeper state before AppendRequest - let tli = prepare_safekeeper(spg.ttid, append_request.pg_version).await?; + let tli = prepare_safekeeper(spg, append_request.pg_version).await?; // if send_proposer_elected is true, we need to update local history if append_request.send_proposer_elected { @@ -99,20 +97,22 @@ pub async fn handle_json_ctrl( /// Prepare safekeeper to process append requests without crashes, /// by sending ProposerGreeting with default server.wal_seg_size. async fn prepare_safekeeper( - ttid: TenantTimelineId, + spg: &SafekeeperPostgresHandler, pg_version: u32, ) -> anyhow::Result { - let tli = GlobalTimelines::create( - ttid, - ServerInfo { - pg_version, - wal_seg_size: WAL_SEGMENT_SIZE as u32, - system_id: 0, - }, - Lsn::INVALID, - Lsn::INVALID, - ) - .await?; + let tli = spg + .global_timelines + .create( + spg.ttid, + ServerInfo { + pg_version, + wal_seg_size: WAL_SEGMENT_SIZE as u32, + system_id: 0, + }, + Lsn::INVALID, + Lsn::INVALID, + ) + .await?; tli.wal_residence_guard().await } diff --git a/safekeeper/src/metrics.rs b/safekeeper/src/metrics.rs index bbd2f86898..5883f402c7 100644 --- a/safekeeper/src/metrics.rs +++ b/safekeeper/src/metrics.rs @@ -455,6 +455,7 @@ pub struct FullTimelineInfo { /// Collects metrics for all active timelines. pub struct TimelineCollector { + global_timelines: Arc, descs: Vec, commit_lsn: GenericGaugeVec, backup_lsn: GenericGaugeVec, @@ -478,14 +479,8 @@ pub struct TimelineCollector { active_timelines_count: IntGauge, } -impl Default for TimelineCollector { - fn default() -> Self { - Self::new() - } -} - impl TimelineCollector { - pub fn new() -> TimelineCollector { + pub fn new(global_timelines: Arc) -> TimelineCollector { let mut descs = Vec::new(); let commit_lsn = GenericGaugeVec::new( @@ -676,6 +671,7 @@ impl TimelineCollector { descs.extend(active_timelines_count.desc().into_iter().cloned()); TimelineCollector { + global_timelines, descs, commit_lsn, backup_lsn, @@ -728,17 +724,18 @@ impl Collector for TimelineCollector { self.written_wal_seconds.reset(); self.flushed_wal_seconds.reset(); - let timelines_count = GlobalTimelines::get_all().len(); + let timelines_count = self.global_timelines.get_all().len(); let mut active_timelines_count = 0; // Prometheus Collector is sync, and data is stored under async lock. To // bridge the gap with a crutch, collect data in spawned thread with // local tokio runtime. + let global_timelines = self.global_timelines.clone(); let infos = std::thread::spawn(|| { let rt = tokio::runtime::Builder::new_current_thread() .build() .expect("failed to create rt"); - rt.block_on(collect_timeline_metrics()) + rt.block_on(collect_timeline_metrics(global_timelines)) }) .join() .expect("collect_timeline_metrics thread panicked"); @@ -857,9 +854,9 @@ impl Collector for TimelineCollector { } } -async fn collect_timeline_metrics() -> Vec { +async fn collect_timeline_metrics(global_timelines: Arc) -> Vec { let mut res = vec![]; - let active_timelines = GlobalTimelines::get_global_broker_active_set().get_all(); + let active_timelines = global_timelines.get_global_broker_active_set().get_all(); for tli in active_timelines { if let Some(info) = tli.info_for_metrics().await { diff --git a/safekeeper/src/pull_timeline.rs b/safekeeper/src/pull_timeline.rs index c700e18cc7..f58a9dca1d 100644 --- a/safekeeper/src/pull_timeline.rs +++ b/safekeeper/src/pull_timeline.rs @@ -409,8 +409,9 @@ pub struct DebugDumpResponse { pub async fn handle_request( request: Request, sk_auth_token: Option, + global_timelines: Arc, ) -> Result { - let existing_tli = GlobalTimelines::get(TenantTimelineId::new( + let existing_tli = global_timelines.get(TenantTimelineId::new( request.tenant_id, request.timeline_id, )); @@ -453,13 +454,14 @@ pub async fn handle_request( assert!(status.tenant_id == request.tenant_id); assert!(status.timeline_id == request.timeline_id); - pull_timeline(status, safekeeper_host, sk_auth_token).await + pull_timeline(status, safekeeper_host, sk_auth_token, global_timelines).await } async fn pull_timeline( status: TimelineStatus, host: String, sk_auth_token: Option, + global_timelines: Arc, ) -> Result { let ttid = TenantTimelineId::new(status.tenant_id, status.timeline_id); info!( @@ -472,7 +474,7 @@ async fn pull_timeline( status.acceptor_state.epoch ); - let conf = &GlobalTimelines::get_global_config(); + let conf = &global_timelines.get_global_config(); let (_tmp_dir, tli_dir_path) = create_temp_timeline_dir(conf, ttid).await?; @@ -531,7 +533,9 @@ async fn pull_timeline( assert!(status.commit_lsn <= status.flush_lsn); // Finally, load the timeline. - let _tli = GlobalTimelines::load_temp_timeline(ttid, &tli_dir_path, false).await?; + let _tli = global_timelines + .load_temp_timeline(ttid, &tli_dir_path, false) + .await?; Ok(Response { safekeeper_host: host, diff --git a/safekeeper/src/receive_wal.rs b/safekeeper/src/receive_wal.rs index bfa1764abf..2a49890d61 100644 --- a/safekeeper/src/receive_wal.rs +++ b/safekeeper/src/receive_wal.rs @@ -267,6 +267,7 @@ impl SafekeeperPostgresHandler { pgb_reader: &mut pgb_reader, peer_addr, acceptor_handle: &mut acceptor_handle, + global_timelines: self.global_timelines.clone(), }; // Read first message and create timeline if needed. @@ -331,6 +332,7 @@ struct NetworkReader<'a, IO> { // WalAcceptor is spawned when we learn server info from walproposer and // create timeline; handle is put here. acceptor_handle: &'a mut Option>>, + global_timelines: Arc, } impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { @@ -350,10 +352,11 @@ impl<'a, IO: AsyncRead + AsyncWrite + Unpin> NetworkReader<'a, IO> { system_id: greeting.system_id, wal_seg_size: greeting.wal_seg_size, }; - let tli = - GlobalTimelines::create(self.ttid, server_info, Lsn::INVALID, Lsn::INVALID) - .await - .context("create timeline")?; + let tli = self + .global_timelines + .create(self.ttid, server_info, Lsn::INVALID, Lsn::INVALID) + .await + .context("create timeline")?; tli.wal_residence_guard().await? } _ => { diff --git a/safekeeper/src/send_wal.rs b/safekeeper/src/send_wal.rs index 225b7f4c05..0887cf7264 100644 --- a/safekeeper/src/send_wal.rs +++ b/safekeeper/src/send_wal.rs @@ -10,7 +10,6 @@ use crate::timeline::WalResidentTimeline; use crate::wal_reader_stream::WalReaderStreamBuilder; use crate::wal_service::ConnectionId; use crate::wal_storage::WalReader; -use crate::GlobalTimelines; use anyhow::{bail, Context as AnyhowContext}; use bytes::Bytes; use futures::future::Either; @@ -400,7 +399,10 @@ impl SafekeeperPostgresHandler { start_pos: Lsn, term: Option, ) -> Result<(), QueryError> { - let tli = GlobalTimelines::get(self.ttid).map_err(|e| QueryError::Other(e.into()))?; + let tli = self + .global_timelines + .get(self.ttid) + .map_err(|e| QueryError::Other(e.into()))?; let residence_guard = tli.wal_residence_guard().await?; if let Err(end) = self diff --git a/safekeeper/src/timeline.rs b/safekeeper/src/timeline.rs index ef928f7633..94d6ef1061 100644 --- a/safekeeper/src/timeline.rs +++ b/safekeeper/src/timeline.rs @@ -44,8 +44,8 @@ use crate::wal_backup_partial::PartialRemoteSegment; use crate::metrics::{FullTimelineInfo, WalStorageMetrics, MISC_OPERATION_SECONDS}; use crate::wal_storage::{Storage as wal_storage_iface, WalReader}; +use crate::SafeKeeperConf; use crate::{debug_dump, timeline_manager, wal_storage}; -use crate::{GlobalTimelines, SafeKeeperConf}; /// Things safekeeper should know about timeline state on peers. #[derive(Debug, Clone, Serialize, Deserialize)] @@ -467,6 +467,7 @@ pub struct Timeline { walreceivers: Arc, timeline_dir: Utf8PathBuf, manager_ctl: ManagerCtl, + conf: Arc, /// Hold this gate from code that depends on the Timeline's non-shut-down state. While holding /// this gate, you must respect [`Timeline::cancel`] @@ -489,6 +490,7 @@ impl Timeline { timeline_dir: &Utf8Path, remote_path: &RemotePath, shared_state: SharedState, + conf: Arc, ) -> Arc { let (commit_lsn_watch_tx, commit_lsn_watch_rx) = watch::channel(shared_state.sk.state().commit_lsn); @@ -516,6 +518,7 @@ impl Timeline { gate: Default::default(), cancel: CancellationToken::default(), manager_ctl: ManagerCtl::new(), + conf, broker_active: AtomicBool::new(false), wal_backup_active: AtomicBool::new(false), last_removed_segno: AtomicU64::new(0), @@ -524,11 +527,14 @@ impl Timeline { } /// Load existing timeline from disk. - pub fn load_timeline(conf: &SafeKeeperConf, ttid: TenantTimelineId) -> Result> { + pub fn load_timeline( + conf: Arc, + ttid: TenantTimelineId, + ) -> Result> { let _enter = info_span!("load_timeline", timeline = %ttid.timeline_id).entered(); - let shared_state = SharedState::restore(conf, &ttid)?; - let timeline_dir = get_timeline_dir(conf, &ttid); + let shared_state = SharedState::restore(conf.as_ref(), &ttid)?; + let timeline_dir = get_timeline_dir(conf.as_ref(), &ttid); let remote_path = remote_timeline_path(&ttid)?; Ok(Timeline::new( @@ -536,6 +542,7 @@ impl Timeline { &timeline_dir, &remote_path, shared_state, + conf, )) } @@ -604,8 +611,7 @@ impl Timeline { // it is cancelled, so WAL storage won't be opened again. shared_state.sk.close_wal_store(); - let conf = GlobalTimelines::get_global_config(); - if !only_local && conf.is_wal_backup_enabled() { + if !only_local && self.conf.is_wal_backup_enabled() { // Note: we concurrently delete remote storage data from multiple // safekeepers. That's ok, s3 replies 200 if object doesn't exist and we // do some retries anyway. @@ -951,7 +957,7 @@ impl WalResidentTimeline { pub async fn get_walreader(&self, start_lsn: Lsn) -> Result { let (_, persisted_state) = self.get_state().await; - let enable_remote_read = GlobalTimelines::get_global_config().is_wal_backup_enabled(); + let enable_remote_read = self.conf.is_wal_backup_enabled(); WalReader::new( &self.ttid, @@ -1061,7 +1067,6 @@ impl ManagerTimeline { /// Try to switch state Offloaded->Present. pub(crate) async fn switch_to_present(&self) -> anyhow::Result<()> { - let conf = GlobalTimelines::get_global_config(); let mut shared = self.write_shared_state().await; // trying to restore WAL storage @@ -1069,7 +1074,7 @@ impl ManagerTimeline { &self.ttid, &self.timeline_dir, shared.sk.state(), - conf.no_sync, + self.conf.no_sync, )?; // updating control file @@ -1096,7 +1101,7 @@ impl ManagerTimeline { // now we can switch shared.sk to Present, shouldn't fail let prev_sk = std::mem::replace(&mut shared.sk, StateSK::Empty); let cfile_state = prev_sk.take_state(); - shared.sk = StateSK::Loaded(SafeKeeper::new(cfile_state, wal_store, conf.my_id)?); + shared.sk = StateSK::Loaded(SafeKeeper::new(cfile_state, wal_store, self.conf.my_id)?); Ok(()) } diff --git a/safekeeper/src/timelines_global_map.rs b/safekeeper/src/timelines_global_map.rs index 067945fd5f..e1241ceb9b 100644 --- a/safekeeper/src/timelines_global_map.rs +++ b/safekeeper/src/timelines_global_map.rs @@ -13,7 +13,6 @@ use crate::{control_file, wal_storage, SafeKeeperConf}; use anyhow::{bail, Context, Result}; use camino::Utf8PathBuf; use camino_tempfile::Utf8TempDir; -use once_cell::sync::Lazy; use serde::Serialize; use std::collections::HashMap; use std::str::FromStr; @@ -42,23 +41,16 @@ struct GlobalTimelinesState { // this map is dropped on restart. tombstones: HashMap, - conf: Option, + conf: Arc, broker_active_set: Arc, global_rate_limiter: RateLimiter, } impl GlobalTimelinesState { - /// Get configuration, which must be set once during init. - fn get_conf(&self) -> &SafeKeeperConf { - self.conf - .as_ref() - .expect("GlobalTimelinesState conf is not initialized") - } - /// Get dependencies for a timeline constructor. - fn get_dependencies(&self) -> (SafeKeeperConf, Arc, RateLimiter) { + fn get_dependencies(&self) -> (Arc, Arc, RateLimiter) { ( - self.get_conf().clone(), + self.conf.clone(), self.broker_active_set.clone(), self.global_rate_limiter.clone(), ) @@ -82,35 +74,39 @@ impl GlobalTimelinesState { } } -static TIMELINES_STATE: Lazy> = Lazy::new(|| { - Mutex::new(GlobalTimelinesState { - timelines: HashMap::new(), - tombstones: HashMap::new(), - conf: None, - broker_active_set: Arc::new(TimelinesSet::default()), - global_rate_limiter: RateLimiter::new(1, 1), - }) -}); - -/// A zero-sized struct used to manage access to the global timelines map. -pub struct GlobalTimelines; +/// A struct used to manage access to the global timelines map. +pub struct GlobalTimelines { + state: Mutex, +} impl GlobalTimelines { + /// Create a new instance of the global timelines map. + pub fn new(conf: Arc) -> Self { + Self { + state: Mutex::new(GlobalTimelinesState { + timelines: HashMap::new(), + tombstones: HashMap::new(), + conf, + broker_active_set: Arc::new(TimelinesSet::default()), + global_rate_limiter: RateLimiter::new(1, 1), + }), + } + } + /// Inject dependencies needed for the timeline constructors and load all timelines to memory. - pub async fn init(conf: SafeKeeperConf) -> Result<()> { + pub async fn init(&self) -> Result<()> { // clippy isn't smart enough to understand that drop(state) releases the // lock, so use explicit block let tenants_dir = { - let mut state = TIMELINES_STATE.lock().unwrap(); + let mut state = self.state.lock().unwrap(); state.global_rate_limiter = RateLimiter::new( - conf.partial_backup_concurrency, + state.conf.partial_backup_concurrency, DEFAULT_EVICTION_CONCURRENCY, ); - state.conf = Some(conf); // Iterate through all directories and load tenants for all directories // named as a valid tenant_id. - state.get_conf().workdir.clone() + state.conf.workdir.clone() }; let mut tenant_count = 0; for tenants_dir_entry in std::fs::read_dir(&tenants_dir) @@ -122,7 +118,7 @@ impl GlobalTimelines { TenantId::from_str(tenants_dir_entry.file_name().to_str().unwrap_or("")) { tenant_count += 1; - GlobalTimelines::load_tenant_timelines(tenant_id).await?; + self.load_tenant_timelines(tenant_id).await?; } } Err(e) => error!( @@ -135,7 +131,7 @@ impl GlobalTimelines { info!( "found {} tenants directories, successfully loaded {} timelines", tenant_count, - TIMELINES_STATE.lock().unwrap().timelines.len() + self.state.lock().unwrap().timelines.len() ); Ok(()) } @@ -143,13 +139,13 @@ impl GlobalTimelines { /// Loads all timelines for the given tenant to memory. Returns fs::read_dir /// errors if any. /// - /// It is async, but TIMELINES_STATE lock is sync and there is no important + /// It is async, but self.state lock is sync and there is no important /// reason to make it async (it is always held for a short while), so we /// just lock and unlock it for each timeline -- this function is called /// during init when nothing else is running, so this is fine. - async fn load_tenant_timelines(tenant_id: TenantId) -> Result<()> { + async fn load_tenant_timelines(&self, tenant_id: TenantId) -> Result<()> { let (conf, broker_active_set, partial_backup_rate_limiter) = { - let state = TIMELINES_STATE.lock().unwrap(); + let state = self.state.lock().unwrap(); state.get_dependencies() }; @@ -163,10 +159,10 @@ impl GlobalTimelines { TimelineId::from_str(timeline_dir_entry.file_name().to_str().unwrap_or("")) { let ttid = TenantTimelineId::new(tenant_id, timeline_id); - match Timeline::load_timeline(&conf, ttid) { + match Timeline::load_timeline(conf.clone(), ttid) { Ok(tli) => { let mut shared_state = tli.write_shared_state().await; - TIMELINES_STATE + self.state .lock() .unwrap() .timelines @@ -200,29 +196,30 @@ impl GlobalTimelines { } /// Get the number of timelines in the map. - pub fn timelines_count() -> usize { - TIMELINES_STATE.lock().unwrap().timelines.len() + pub fn timelines_count(&self) -> usize { + self.state.lock().unwrap().timelines.len() } /// Get the global safekeeper config. - pub fn get_global_config() -> SafeKeeperConf { - TIMELINES_STATE.lock().unwrap().get_conf().clone() + pub fn get_global_config(&self) -> Arc { + self.state.lock().unwrap().conf.clone() } - pub fn get_global_broker_active_set() -> Arc { - TIMELINES_STATE.lock().unwrap().broker_active_set.clone() + pub fn get_global_broker_active_set(&self) -> Arc { + self.state.lock().unwrap().broker_active_set.clone() } /// Create a new timeline with the given id. If the timeline already exists, returns /// an existing timeline. pub(crate) async fn create( + &self, ttid: TenantTimelineId, server_info: ServerInfo, commit_lsn: Lsn, local_start_lsn: Lsn, ) -> Result> { let (conf, _, _) = { - let state = TIMELINES_STATE.lock().unwrap(); + let state = self.state.lock().unwrap(); if let Ok(timeline) = state.get(&ttid) { // Timeline already exists, return it. return Ok(timeline); @@ -245,7 +242,7 @@ impl GlobalTimelines { let state = TimelinePersistentState::new(&ttid, server_info, vec![], commit_lsn, local_start_lsn)?; control_file::FileStorage::create_new(&tmp_dir_path, state, conf.no_sync).await?; - let timeline = GlobalTimelines::load_temp_timeline(ttid, &tmp_dir_path, true).await?; + let timeline = self.load_temp_timeline(ttid, &tmp_dir_path, true).await?; Ok(timeline) } @@ -261,13 +258,14 @@ impl GlobalTimelines { /// 2) move the directory and load the timeline /// 3) take lock again and insert the timeline into the global map. pub async fn load_temp_timeline( + &self, ttid: TenantTimelineId, tmp_path: &Utf8PathBuf, check_tombstone: bool, ) -> Result> { // Check for existence and mark that we're creating it. let (conf, broker_active_set, partial_backup_rate_limiter) = { - let mut state = TIMELINES_STATE.lock().unwrap(); + let mut state = self.state.lock().unwrap(); match state.timelines.get(&ttid) { Some(GlobalMapTimeline::CreationInProgress) => { bail!(TimelineError::CreationInProgress(ttid)); @@ -295,10 +293,10 @@ impl GlobalTimelines { }; // Do the actual move and reflect the result in the map. - match GlobalTimelines::install_temp_timeline(ttid, tmp_path, &conf).await { + match GlobalTimelines::install_temp_timeline(ttid, tmp_path, conf.clone()).await { Ok(timeline) => { let mut timeline_shared_state = timeline.write_shared_state().await; - let mut state = TIMELINES_STATE.lock().unwrap(); + let mut state = self.state.lock().unwrap(); assert!(matches!( state.timelines.get(&ttid), Some(GlobalMapTimeline::CreationInProgress) @@ -319,7 +317,7 @@ impl GlobalTimelines { } Err(e) => { // Init failed, remove the marker from the map - let mut state = TIMELINES_STATE.lock().unwrap(); + let mut state = self.state.lock().unwrap(); assert!(matches!( state.timelines.get(&ttid), Some(GlobalMapTimeline::CreationInProgress) @@ -334,10 +332,10 @@ impl GlobalTimelines { async fn install_temp_timeline( ttid: TenantTimelineId, tmp_path: &Utf8PathBuf, - conf: &SafeKeeperConf, + conf: Arc, ) -> Result> { - let tenant_path = get_tenant_dir(conf, &ttid.tenant_id); - let timeline_path = get_timeline_dir(conf, &ttid); + let tenant_path = get_tenant_dir(conf.as_ref(), &ttid.tenant_id); + let timeline_path = get_timeline_dir(conf.as_ref(), &ttid); // We must have already checked that timeline doesn't exist in the map, // but there might be existing datadir: if timeline is corrupted it is @@ -382,9 +380,9 @@ impl GlobalTimelines { /// Get a timeline from the global map. If it's not present, it doesn't exist on disk, /// or was corrupted and couldn't be loaded on startup. Returned timeline is always valid, /// i.e. loaded in memory and not cancelled. - pub(crate) fn get(ttid: TenantTimelineId) -> Result, TimelineError> { + pub(crate) fn get(&self, ttid: TenantTimelineId) -> Result, TimelineError> { let tli_res = { - let state = TIMELINES_STATE.lock().unwrap(); + let state = self.state.lock().unwrap(); state.get(&ttid) }; match tli_res { @@ -399,8 +397,8 @@ impl GlobalTimelines { } /// Returns all timelines. This is used for background timeline processes. - pub fn get_all() -> Vec> { - let global_lock = TIMELINES_STATE.lock().unwrap(); + pub fn get_all(&self) -> Vec> { + let global_lock = self.state.lock().unwrap(); global_lock .timelines .values() @@ -419,8 +417,8 @@ impl GlobalTimelines { /// Returns all timelines belonging to a given tenant. Used for deleting all timelines of a tenant, /// and that's why it can return cancelled timelines, to retry deleting them. - fn get_all_for_tenant(tenant_id: TenantId) -> Vec> { - let global_lock = TIMELINES_STATE.lock().unwrap(); + fn get_all_for_tenant(&self, tenant_id: TenantId) -> Vec> { + let global_lock = self.state.lock().unwrap(); global_lock .timelines .values() @@ -435,11 +433,12 @@ impl GlobalTimelines { /// Cancels timeline, then deletes the corresponding data directory. /// If only_local, doesn't remove WAL segments in remote storage. pub(crate) async fn delete( + &self, ttid: &TenantTimelineId, only_local: bool, ) -> Result { let tli_res = { - let state = TIMELINES_STATE.lock().unwrap(); + let state = self.state.lock().unwrap(); if state.tombstones.contains_key(ttid) { // Presence of a tombstone guarantees that a previous deletion has completed and there is no work to do. @@ -472,7 +471,7 @@ impl GlobalTimelines { } Err(_) => { // Timeline is not memory, but it may still exist on disk in broken state. - let dir_path = get_timeline_dir(TIMELINES_STATE.lock().unwrap().get_conf(), ttid); + let dir_path = get_timeline_dir(self.state.lock().unwrap().conf.as_ref(), ttid); let dir_existed = delete_dir(dir_path)?; Ok(TimelineDeleteForceResult { @@ -485,7 +484,7 @@ impl GlobalTimelines { // Finalize deletion, by dropping Timeline objects and storing smaller tombstones. The tombstones // are used to prevent still-running computes from re-creating the same timeline when they send data, // and to speed up repeated deletion calls by avoiding re-listing objects. - TIMELINES_STATE.lock().unwrap().delete(*ttid); + self.state.lock().unwrap().delete(*ttid); result } @@ -497,17 +496,18 @@ impl GlobalTimelines { /// /// If only_local, doesn't remove WAL segments in remote storage. pub async fn delete_force_all_for_tenant( + &self, tenant_id: &TenantId, only_local: bool, ) -> Result> { info!("deleting all timelines for tenant {}", tenant_id); - let to_delete = Self::get_all_for_tenant(*tenant_id); + let to_delete = self.get_all_for_tenant(*tenant_id); let mut err = None; let mut deleted = HashMap::new(); for tli in &to_delete { - match Self::delete(&tli.ttid, only_local).await { + match self.delete(&tli.ttid, only_local).await { Ok(result) => { deleted.insert(tli.ttid, result); } @@ -529,15 +529,15 @@ impl GlobalTimelines { // so the directory may be not empty. In this case timelines will have bad state // and timeline background jobs can panic. delete_dir(get_tenant_dir( - TIMELINES_STATE.lock().unwrap().get_conf(), + self.state.lock().unwrap().conf.as_ref(), tenant_id, ))?; Ok(deleted) } - pub fn housekeeping(tombstone_ttl: &Duration) { - let mut state = TIMELINES_STATE.lock().unwrap(); + pub fn housekeeping(&self, tombstone_ttl: &Duration) { + let mut state = self.state.lock().unwrap(); // We keep tombstones long enough to have a good chance of preventing rogue computes from re-creating deleted // timelines. If a compute kept running for longer than this TTL (or across a safekeeper restart) then they diff --git a/safekeeper/src/wal_service.rs b/safekeeper/src/wal_service.rs index 5248d545db..1ff83918a7 100644 --- a/safekeeper/src/wal_service.rs +++ b/safekeeper/src/wal_service.rs @@ -4,6 +4,7 @@ //! use anyhow::{Context, Result}; use postgres_backend::QueryError; +use std::sync::Arc; use std::time::Duration; use tokio::net::TcpStream; use tokio_io_timeout::TimeoutReader; @@ -11,9 +12,9 @@ use tokio_util::sync::CancellationToken; use tracing::*; use utils::{auth::Scope, measured_stream::MeasuredStream}; -use crate::handler::SafekeeperPostgresHandler; use crate::metrics::TrafficMetrics; use crate::SafeKeeperConf; +use crate::{handler::SafekeeperPostgresHandler, GlobalTimelines}; use postgres_backend::{AuthType, PostgresBackend}; /// Accept incoming TCP connections and spawn them into a background thread. @@ -22,9 +23,10 @@ use postgres_backend::{AuthType, PostgresBackend}; /// to any tenant are allowed) or Tenant (only tokens giving access to specific /// tenant are allowed). Doesn't matter if auth is disabled in conf. pub async fn task_main( - conf: SafeKeeperConf, + conf: Arc, pg_listener: std::net::TcpListener, allowed_auth_scope: Scope, + global_timelines: Arc, ) -> anyhow::Result<()> { // Tokio's from_std won't do this for us, per its comment. pg_listener.set_nonblocking(true)?; @@ -37,10 +39,10 @@ pub async fn task_main( debug!("accepted connection from {}", peer_addr); let conf = conf.clone(); let conn_id = issue_connection_id(&mut connection_count); - + let global_timelines = global_timelines.clone(); tokio::spawn( async move { - if let Err(err) = handle_socket(socket, conf, conn_id, allowed_auth_scope).await { + if let Err(err) = handle_socket(socket, conf, conn_id, allowed_auth_scope, global_timelines).await { error!("connection handler exited: {}", err); } } @@ -53,9 +55,10 @@ pub async fn task_main( /// async fn handle_socket( socket: TcpStream, - conf: SafeKeeperConf, + conf: Arc, conn_id: ConnectionId, allowed_auth_scope: Scope, + global_timelines: Arc, ) -> Result<(), QueryError> { socket.set_nodelay(true)?; let peer_addr = socket.peer_addr()?; @@ -96,8 +99,13 @@ async fn handle_socket( Some(_) => AuthType::NeonJWT, }; let auth_pair = auth_key.map(|key| (allowed_auth_scope, key)); - let mut conn_handler = - SafekeeperPostgresHandler::new(conf, conn_id, Some(traffic_metrics.clone()), auth_pair); + let mut conn_handler = SafekeeperPostgresHandler::new( + conf, + conn_id, + Some(traffic_metrics.clone()), + auth_pair, + global_timelines, + ); let pgbackend = PostgresBackend::new_from_io(socket, peer_addr, auth_type, None)?; // libpq protocol between safekeeper and walproposer / pageserver // We don't use shutdown. From 34c1295594c70bf124c0a6a85fa21d51dff7967b Mon Sep 17 00:00:00 2001 From: Ivan Efremov Date: Tue, 10 Dec 2024 12:14:28 +0200 Subject: [PATCH 05/44] [proxy] impr: Additional logging for cancellation queries (#10039) ## Problem Since cancellation tasks spawned in the background sometimes logs missing context. https://neondb.slack.com/archives/C060N3SEF9D/p1733427801527419?thread_ts=1733419882.560159&cid=C060N3SEF9D ## Summary of changes Add `session_id` and change loglevel for cancellation queries --- proxy/src/cancellation.rs | 3 +- proxy/src/console_redirect_proxy.rs | 54 ++++++++++++++++------------- proxy/src/proxy/mod.rs | 54 ++++++++++++++++------------- proxy/src/redis/notifications.rs | 4 +++ 4 files changed, 64 insertions(+), 51 deletions(-) diff --git a/proxy/src/cancellation.rs b/proxy/src/cancellation.rs index 7bc5587a25..ed717507ee 100644 --- a/proxy/src/cancellation.rs +++ b/proxy/src/cancellation.rs @@ -115,7 +115,8 @@ impl CancellationHandler

{ IpAddr::V6(ip) => IpNet::V6(Ipv6Net::new_assert(ip, 64).trunc()), }; if !self.limiter.lock().unwrap().check(subnet_key, 1) { - tracing::debug!("Rate limit exceeded. Skipping cancellation message"); + // log only the subnet part of the IP address to know which subnet is rate limited + tracing::warn!("Rate limit exceeded. Skipping cancellation message, {subnet_key}"); Metrics::get() .proxy .cancellation_requests_total diff --git a/proxy/src/console_redirect_proxy.rs b/proxy/src/console_redirect_proxy.rs index 7db1179eea..65702e0e4c 100644 --- a/proxy/src/console_redirect_proxy.rs +++ b/proxy/src/console_redirect_proxy.rs @@ -163,32 +163,36 @@ pub(crate) async fn handle_client( let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Client); let do_handshake = handshake(ctx, stream, tls, record_handshake_error); - let (mut stream, params) = - match tokio::time::timeout(config.handshake_timeout, do_handshake).await?? { - HandshakeData::Startup(stream, params) => (stream, params), - HandshakeData::Cancel(cancel_key_data) => { - // spawn a task to cancel the session, but don't wait for it - cancellations.spawn({ - let cancellation_handler_clone = Arc::clone(&cancellation_handler); - let session_id = ctx.session_id(); - let peer_ip = ctx.peer_addr(); - async move { - drop( - cancellation_handler_clone - .cancel_session( - cancel_key_data, - session_id, - peer_ip, - config.authentication_config.ip_allowlist_check_enabled, - ) - .await, - ); - } - }); + let (mut stream, params) = match tokio::time::timeout(config.handshake_timeout, do_handshake) + .await?? + { + HandshakeData::Startup(stream, params) => (stream, params), + HandshakeData::Cancel(cancel_key_data) => { + // spawn a task to cancel the session, but don't wait for it + cancellations.spawn({ + let cancellation_handler_clone = Arc::clone(&cancellation_handler); + let session_id = ctx.session_id(); + let peer_ip = ctx.peer_addr(); + let cancel_span = tracing::span!(parent: None, tracing::Level::INFO, "cancel_session", session_id = ?session_id); + cancel_span.follows_from(tracing::Span::current()); + async move { + drop( + cancellation_handler_clone + .cancel_session( + cancel_key_data, + session_id, + peer_ip, + config.authentication_config.ip_allowlist_check_enabled, + ) + .instrument(cancel_span) + .await, + ); + } + }); - return Ok(None); - } - }; + return Ok(None); + } + }; drop(pause); ctx.set_db_options(params.clone()); diff --git a/proxy/src/proxy/mod.rs b/proxy/src/proxy/mod.rs index f74eb5940f..cc04bc5e5c 100644 --- a/proxy/src/proxy/mod.rs +++ b/proxy/src/proxy/mod.rs @@ -272,32 +272,36 @@ pub(crate) async fn handle_client( let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Client); let do_handshake = handshake(ctx, stream, mode.handshake_tls(tls), record_handshake_error); - let (mut stream, params) = - match tokio::time::timeout(config.handshake_timeout, do_handshake).await?? { - HandshakeData::Startup(stream, params) => (stream, params), - HandshakeData::Cancel(cancel_key_data) => { - // spawn a task to cancel the session, but don't wait for it - cancellations.spawn({ - let cancellation_handler_clone = Arc::clone(&cancellation_handler); - let session_id = ctx.session_id(); - let peer_ip = ctx.peer_addr(); - async move { - drop( - cancellation_handler_clone - .cancel_session( - cancel_key_data, - session_id, - peer_ip, - config.authentication_config.ip_allowlist_check_enabled, - ) - .await, - ); - } - }); + let (mut stream, params) = match tokio::time::timeout(config.handshake_timeout, do_handshake) + .await?? + { + HandshakeData::Startup(stream, params) => (stream, params), + HandshakeData::Cancel(cancel_key_data) => { + // spawn a task to cancel the session, but don't wait for it + cancellations.spawn({ + let cancellation_handler_clone = Arc::clone(&cancellation_handler); + let session_id = ctx.session_id(); + let peer_ip = ctx.peer_addr(); + let cancel_span = tracing::span!(parent: None, tracing::Level::INFO, "cancel_session", session_id = ?session_id); + cancel_span.follows_from(tracing::Span::current()); + async move { + drop( + cancellation_handler_clone + .cancel_session( + cancel_key_data, + session_id, + peer_ip, + config.authentication_config.ip_allowlist_check_enabled, + ) + .instrument(cancel_span) + .await, + ); + } + }); - return Ok(None); - } - }; + return Ok(None); + } + }; drop(pause); ctx.set_db_options(params.clone()); diff --git a/proxy/src/redis/notifications.rs b/proxy/src/redis/notifications.rs index 9ac07b7e90..f3aa97c032 100644 --- a/proxy/src/redis/notifications.rs +++ b/proxy/src/redis/notifications.rs @@ -13,6 +13,7 @@ use crate::cache::project_info::ProjectInfoCache; use crate::cancellation::{CancelMap, CancellationHandler}; use crate::intern::{ProjectIdInt, RoleNameInt}; use crate::metrics::{Metrics, RedisErrors, RedisEventsCount}; +use tracing::Instrument; const CPLANE_CHANNEL_NAME: &str = "neondb-proxy-ws-updates"; pub(crate) const PROXY_CHANNEL_NAME: &str = "neondb-proxy-to-proxy-updates"; @@ -143,6 +144,8 @@ impl MessageHandler { let peer_addr = cancel_session .peer_addr .unwrap_or(std::net::IpAddr::V4(std::net::Ipv4Addr::UNSPECIFIED)); + let cancel_span = tracing::span!(parent: None, tracing::Level::INFO, "cancel_session", session_id = ?cancel_session.session_id); + cancel_span.follows_from(tracing::Span::current()); // This instance of cancellation_handler doesn't have a RedisPublisherClient so it can't publish the message. match self .cancellation_handler @@ -152,6 +155,7 @@ impl MessageHandler { peer_addr, cancel_session.peer_addr.is_some(), ) + .instrument(cancel_span) .await { Ok(()) => {} From c51db1db61c7c1513cb6fc69563e80c4110abd9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Tue, 10 Dec 2024 12:29:38 +0100 Subject: [PATCH 06/44] Replace MAX_KEYS_PER_DELETE constant with function (#10061) Azure has a different per-request limit of 256 items for bulk deletion compared to the number of 1000 on AWS. Therefore, we need to support multiple values. Due to `GenericRemoteStorage`, we can't add an associated constant, but it has to be a function. The PR replaces the `MAX_KEYS_PER_DELETE` constant with a function of the same name, implemented on both the `RemoteStorage` trait as well as on `GenericRemoteStorage`. The value serves as hint of how many objects to pass to the `delete_objects` function. Reading: * https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch * https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html Part of #7931 --- libs/remote_storage/src/azure_blob.rs | 4 +++ libs/remote_storage/src/lib.rs | 27 +++++++++++++++++++- libs/remote_storage/src/local_fs.rs | 4 +++ libs/remote_storage/src/s3_bucket.rs | 8 ++++-- libs/remote_storage/src/simulate_failures.rs | 4 +++ pageserver/src/deletion_queue/deleter.rs | 10 ++++---- storage_scrubber/src/garbage.rs | 11 ++++---- 7 files changed, 54 insertions(+), 14 deletions(-) diff --git a/libs/remote_storage/src/azure_blob.rs b/libs/remote_storage/src/azure_blob.rs index a1d7569140..32c51bc2ad 100644 --- a/libs/remote_storage/src/azure_blob.rs +++ b/libs/remote_storage/src/azure_blob.rs @@ -624,6 +624,10 @@ impl RemoteStorage for AzureBlobStorage { res } + fn max_keys_per_delete(&self) -> usize { + super::MAX_KEYS_PER_DELETE_AZURE + } + async fn copy( &self, from: &RemotePath, diff --git a/libs/remote_storage/src/lib.rs b/libs/remote_storage/src/lib.rs index 0ece29d99e..2a3468f986 100644 --- a/libs/remote_storage/src/lib.rs +++ b/libs/remote_storage/src/lib.rs @@ -70,7 +70,14 @@ pub const DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT: usize = 100; pub const DEFAULT_MAX_KEYS_PER_LIST_RESPONSE: Option = None; /// As defined in S3 docs -pub const MAX_KEYS_PER_DELETE: usize = 1000; +/// +/// +pub const MAX_KEYS_PER_DELETE_S3: usize = 1000; + +/// As defined in Azure docs +/// +/// +pub const MAX_KEYS_PER_DELETE_AZURE: usize = 256; const REMOTE_STORAGE_PREFIX_SEPARATOR: char = '/'; @@ -340,6 +347,14 @@ pub trait RemoteStorage: Send + Sync + 'static { cancel: &CancellationToken, ) -> anyhow::Result<()>; + /// Returns the maximum number of keys that a call to [`Self::delete_objects`] can delete without chunking + /// + /// The value returned is only an optimization hint, One can pass larger number of objects to + /// `delete_objects` as well. + /// + /// The value is guaranteed to be >= 1. + fn max_keys_per_delete(&self) -> usize; + /// Deletes all objects matching the given prefix. /// /// NB: this uses NoDelimiter and will match partial prefixes. For example, the prefix /a/b will @@ -533,6 +548,16 @@ impl GenericRemoteStorage> { } } + /// [`RemoteStorage::max_keys_per_delete`] + pub fn max_keys_per_delete(&self) -> usize { + match self { + Self::LocalFs(s) => s.max_keys_per_delete(), + Self::AwsS3(s) => s.max_keys_per_delete(), + Self::AzureBlob(s) => s.max_keys_per_delete(), + Self::Unreliable(s) => s.max_keys_per_delete(), + } + } + /// See [`RemoteStorage::delete_prefix`] pub async fn delete_prefix( &self, diff --git a/libs/remote_storage/src/local_fs.rs b/libs/remote_storage/src/local_fs.rs index ee2fc9d6e2..1a2d421c66 100644 --- a/libs/remote_storage/src/local_fs.rs +++ b/libs/remote_storage/src/local_fs.rs @@ -573,6 +573,10 @@ impl RemoteStorage for LocalFs { Ok(()) } + fn max_keys_per_delete(&self) -> usize { + super::MAX_KEYS_PER_DELETE_S3 + } + async fn copy( &self, from: &RemotePath, diff --git a/libs/remote_storage/src/s3_bucket.rs b/libs/remote_storage/src/s3_bucket.rs index cde32df402..2891f92d07 100644 --- a/libs/remote_storage/src/s3_bucket.rs +++ b/libs/remote_storage/src/s3_bucket.rs @@ -48,7 +48,7 @@ use crate::{ metrics::{start_counting_cancelled_wait, start_measuring_requests}, support::PermitCarrying, ConcurrencyLimiter, Download, DownloadError, DownloadOpts, Listing, ListingMode, ListingObject, - RemotePath, RemoteStorage, TimeTravelError, TimeoutOrCancel, MAX_KEYS_PER_DELETE, + RemotePath, RemoteStorage, TimeTravelError, TimeoutOrCancel, MAX_KEYS_PER_DELETE_S3, REMOTE_STORAGE_PREFIX_SEPARATOR, }; @@ -355,7 +355,7 @@ impl S3Bucket { let kind = RequestKind::Delete; let mut cancel = std::pin::pin!(cancel.cancelled()); - for chunk in delete_objects.chunks(MAX_KEYS_PER_DELETE) { + for chunk in delete_objects.chunks(MAX_KEYS_PER_DELETE_S3) { let started_at = start_measuring_requests(kind); let req = self @@ -832,6 +832,10 @@ impl RemoteStorage for S3Bucket { self.delete_oids(&permit, &delete_objects, cancel).await } + fn max_keys_per_delete(&self) -> usize { + MAX_KEYS_PER_DELETE_S3 + } + async fn delete(&self, path: &RemotePath, cancel: &CancellationToken) -> anyhow::Result<()> { let paths = std::array::from_ref(path); self.delete_objects(paths, cancel).await diff --git a/libs/remote_storage/src/simulate_failures.rs b/libs/remote_storage/src/simulate_failures.rs index 10db53971c..51833c1fe6 100644 --- a/libs/remote_storage/src/simulate_failures.rs +++ b/libs/remote_storage/src/simulate_failures.rs @@ -203,6 +203,10 @@ impl RemoteStorage for UnreliableWrapper { Ok(()) } + fn max_keys_per_delete(&self) -> usize { + self.inner.max_keys_per_delete() + } + async fn copy( &self, from: &RemotePath, diff --git a/pageserver/src/deletion_queue/deleter.rs b/pageserver/src/deletion_queue/deleter.rs index 3d02387c98..ef1dfbac19 100644 --- a/pageserver/src/deletion_queue/deleter.rs +++ b/pageserver/src/deletion_queue/deleter.rs @@ -9,7 +9,6 @@ use remote_storage::GenericRemoteStorage; use remote_storage::RemotePath; use remote_storage::TimeoutOrCancel; -use remote_storage::MAX_KEYS_PER_DELETE; use std::time::Duration; use tokio_util::sync::CancellationToken; use tracing::info; @@ -131,7 +130,8 @@ impl Deleter { } pub(super) async fn background(&mut self) -> Result<(), DeletionQueueError> { - self.accumulator.reserve(MAX_KEYS_PER_DELETE); + let max_keys_per_delete = self.remote_storage.max_keys_per_delete(); + self.accumulator.reserve(max_keys_per_delete); loop { if self.cancel.is_cancelled() { @@ -156,14 +156,14 @@ impl Deleter { match msg { DeleterMessage::Delete(mut list) => { - while !list.is_empty() || self.accumulator.len() == MAX_KEYS_PER_DELETE { - if self.accumulator.len() == MAX_KEYS_PER_DELETE { + while !list.is_empty() || self.accumulator.len() == max_keys_per_delete { + if self.accumulator.len() == max_keys_per_delete { self.flush().await?; // If we have received this number of keys, proceed with attempting to execute assert_eq!(self.accumulator.len(), 0); } - let available_slots = MAX_KEYS_PER_DELETE - self.accumulator.len(); + let available_slots = max_keys_per_delete - self.accumulator.len(); let take_count = std::cmp::min(available_slots, list.len()); for path in list.drain(list.len() - take_count..) { self.accumulator.push(path); diff --git a/storage_scrubber/src/garbage.rs b/storage_scrubber/src/garbage.rs index b026efbc3b..a4e5107e3d 100644 --- a/storage_scrubber/src/garbage.rs +++ b/storage_scrubber/src/garbage.rs @@ -459,12 +459,10 @@ pub async fn get_timeline_objects( Ok(list.keys) } -const MAX_KEYS_PER_DELETE: usize = 1000; - /// Drain a buffer of keys into DeleteObjects requests /// /// If `drain` is true, drains keys completely; otherwise stops when < -/// MAX_KEYS_PER_DELETE keys are left. +/// `max_keys_per_delete`` keys are left. /// `num_deleted` returns number of deleted keys. async fn do_delete( remote_client: &GenericRemoteStorage, @@ -474,9 +472,10 @@ async fn do_delete( progress_tracker: &mut DeletionProgressTracker, ) -> anyhow::Result<()> { let cancel = CancellationToken::new(); - while (!keys.is_empty() && drain) || (keys.len() >= MAX_KEYS_PER_DELETE) { + let max_keys_per_delete = remote_client.max_keys_per_delete(); + while (!keys.is_empty() && drain) || (keys.len() >= max_keys_per_delete) { let request_keys = - keys.split_off(keys.len() - (std::cmp::min(MAX_KEYS_PER_DELETE, keys.len()))); + keys.split_off(keys.len() - (std::cmp::min(max_keys_per_delete, keys.len()))); let request_keys: Vec = request_keys.into_iter().map(|o| o.key).collect(); @@ -617,7 +616,7 @@ pub async fn purge_garbage( } objects_to_delete.append(&mut object_list); - if objects_to_delete.len() >= MAX_KEYS_PER_DELETE { + if objects_to_delete.len() >= remote_client.max_keys_per_delete() { do_delete( &remote_client, &mut objects_to_delete, From ad472bd4a18a6fb03310b631fc565cfbc28c0a82 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Tue, 10 Dec 2024 13:07:00 +0100 Subject: [PATCH 07/44] test_runner: add visibility map test (#9940) Verifies that visibility map pages are correctly maintained across shards. Touches #9914. --- test_runner/regress/test_vm_bits.py | 75 ++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/test_runner/regress/test_vm_bits.py b/test_runner/regress/test_vm_bits.py index f93fc6bd8b..46e90852a6 100644 --- a/test_runner/regress/test_vm_bits.py +++ b/test_runner/regress/test_vm_bits.py @@ -4,7 +4,7 @@ import time from contextlib import closing from fixtures.log_helper import log -from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, fork_at_current_lsn +from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, PgBin, fork_at_current_lsn from fixtures.utils import query_scalar @@ -292,3 +292,76 @@ def test_vm_bit_clear_on_heap_lock_blackbox(neon_env_builder: NeonEnvBuilder): tup = cur.fetchall() log.info(f"tuple = {tup}") cur.execute("commit transaction") + + +def test_check_visibility_map(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): + """ + Runs pgbench across a few databases on a sharded tenant, then performs a visibility map + consistency check. Regression test for https://github.com/neondatabase/neon/issues/9914. + """ + + # Use a large number of shards with small stripe sizes, to ensure the visibility + # map will end up on non-zero shards. + SHARD_COUNT = 8 + STRIPE_SIZE = 32 # in 8KB pages + PGBENCH_RUNS = 4 + + env = neon_env_builder.init_start( + initial_tenant_shard_count=SHARD_COUNT, initial_tenant_shard_stripe_size=STRIPE_SIZE + ) + endpoint = env.endpoints.create_start( + "main", + config_lines=[ + "shared_buffers = 64MB", + ], + ) + + # Run pgbench in 4 different databases, to exercise different shards. + dbnames = [f"pgbench{i}" for i in range(PGBENCH_RUNS)] + for i, dbname in enumerate(dbnames): + log.info(f"pgbench run {i+1}/{PGBENCH_RUNS}") + endpoint.safe_psql(f"create database {dbname}") + connstr = endpoint.connstr(dbname=dbname) + # pgbench -i will automatically vacuum the tables. This creates the visibility map. + pg_bin.run(["pgbench", "-i", "-s", "10", connstr]) + # Freeze the tuples to set the initial frozen bit. + endpoint.safe_psql("vacuum freeze", dbname=dbname) + # Run pgbench. + pg_bin.run(["pgbench", "-c", "32", "-j", "8", "-T", "10", connstr]) + + # Restart the endpoint to flush the compute page cache. We want to make sure we read VM pages + # from storage, not cache. + endpoint.stop() + endpoint.start() + + # Check that the visibility map matches the heap contents for pg_accounts (the main table). + for dbname in dbnames: + log.info(f"Checking visibility map for {dbname}") + with endpoint.cursor(dbname=dbname) as cur: + cur.execute("create extension pg_visibility") + + cur.execute("select count(*) from pg_check_visible('pgbench_accounts')") + row = cur.fetchone() + assert row is not None + assert row[0] == 0, f"{row[0]} inconsistent VM pages (visible)" + + cur.execute("select count(*) from pg_check_frozen('pgbench_accounts')") + row = cur.fetchone() + assert row is not None + assert row[0] == 0, f"{row[0]} inconsistent VM pages (frozen)" + + # Vacuum and freeze the tables, and check that the visibility map is still accurate. + for dbname in dbnames: + log.info(f"Vacuuming and checking visibility map for {dbname}") + with endpoint.cursor(dbname=dbname) as cur: + cur.execute("vacuum freeze") + + cur.execute("select count(*) from pg_check_visible('pgbench_accounts')") + row = cur.fetchone() + assert row is not None + assert row[0] == 0, f"{row[0]} inconsistent VM pages (visible)" + + cur.execute("select count(*) from pg_check_frozen('pgbench_accounts')") + row = cur.fetchone() + assert row is not None + assert row[0] == 0, f"{row[0]} inconsistent VM pages (frozen)" From 311ee793b972e6dfc856f560a273734dd57a0c77 Mon Sep 17 00:00:00 2001 From: Konstantin Knizhnik Date: Tue, 10 Dec 2024 17:01:40 +0200 Subject: [PATCH 08/44] Fix handling in-flight requersts in prefetch buffer resize (#9968) ## Problem See https://github.com/neondatabase/neon/issues/9961 Current implementation of prefetch buffer resize doesn't correctly handle in-flight requests ## Summary of changes 1. Fix index of entry we should wait for if new prefetch buffer size is smaller than number of in-flight requests. 2. Correctly set flush position Co-authored-by: Konstantin Knizhnik --- pgxn/neon/pagestore_smgr.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pgxn/neon/pagestore_smgr.c b/pgxn/neon/pagestore_smgr.c index 880c0de64e..385905d9ce 100644 --- a/pgxn/neon/pagestore_smgr.c +++ b/pgxn/neon/pagestore_smgr.c @@ -423,7 +423,11 @@ readahead_buffer_resize(int newsize, void *extra) * ensuring we have received all but the last n requests (n = newsize). */ if (MyPState->n_requests_inflight > newsize) - prefetch_wait_for(MyPState->ring_unused - newsize); + { + Assert(MyPState->ring_unused >= MyPState->n_requests_inflight - newsize); + prefetch_wait_for(MyPState->ring_unused - (MyPState->n_requests_inflight - newsize)); + Assert(MyPState->n_requests_inflight <= newsize); + } /* construct the new PrefetchState, and copy over the memory contexts */ newPState = MemoryContextAllocZero(TopMemoryContext, newprfs_size); @@ -438,7 +442,6 @@ readahead_buffer_resize(int newsize, void *extra) newPState->ring_last = newsize; newPState->ring_unused = newsize; newPState->ring_receive = newsize; - newPState->ring_flush = newsize; newPState->max_shard_no = MyPState->max_shard_no; memcpy(newPState->shard_bitmap, MyPState->shard_bitmap, sizeof(MyPState->shard_bitmap)); @@ -489,6 +492,7 @@ readahead_buffer_resize(int newsize, void *extra) } newPState->n_unused -= 1; } + newPState->ring_flush = newPState->ring_receive; MyNeonCounters->getpage_prefetches_buffered = MyPState->n_responses_buffered; @@ -498,6 +502,7 @@ readahead_buffer_resize(int newsize, void *extra) for (; end >= MyPState->ring_last && end != UINT64_MAX; end -= 1) { PrefetchRequest *slot = GetPrfSlot(end); + Assert(slot->status != PRFS_REQUESTED); if (slot->status == PRFS_RECEIVED) { pfree(slot->response); From 6ad99826c1d175e12dea50120041bc0822830be6 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Tue, 10 Dec 2024 10:23:26 -0500 Subject: [PATCH 09/44] fix(pageserver): refresh_gc_info should always increase cutoff (#9862) ## Problem close https://github.com/neondatabase/cloud/issues/19671 ``` Timeline ----------------------------- ^ last GC happened LSN ^ original retention period setting = 24hr > refresh-gc-info updates the gc_info ^ planned cutoff (gc_info) ^ customer set retention to 48hr, and it's still within the last GC LSN ^1 ^2 we have two choices: (1) update the planned cutoff to move backwards, or (2) keep the current one ``` In this patch, we decided to keep the current cutoff instead of moving back the gc_info to avoid races. In the future, we could allow the planned gc cutoff to go back once cplane sends a retention_history tenant config update, but this requires a careful revisit of the code. ## Summary of changes Ensure that GC cutoffs never go back if retention settings get changed. Signed-off-by: Alex Chi Z --- pageserver/src/tenant.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index e71a56ed40..54fa95fc47 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -4506,7 +4506,12 @@ impl Tenant { // - this timeline was created while we were finding cutoffs // - lsn for timestamp search fails for this timeline repeatedly if let Some(cutoffs) = gc_cutoffs.get(&timeline.timeline_id) { - target.cutoffs = cutoffs.clone(); + let original_cutoffs = target.cutoffs.clone(); + // GC cutoffs should never go back + target.cutoffs = GcCutoffs { + space: Lsn(cutoffs.space.0.max(original_cutoffs.space.0)), + time: Lsn(cutoffs.time.0.max(original_cutoffs.time.0)), + } } } From b853f7813606a02af226d2ef80a92d10dafca527 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Tue, 10 Dec 2024 18:26:56 +0200 Subject: [PATCH 10/44] Print a log message if GetPage response takes too long (#10046) We have metrics for GetPage request latencies, but this is an extra measure to capture requests that take way too long in the logs. The log message is printed every 10 s, until the response is received: ``` PG:2024-12-09 16:02:07.715 GMT [1782845] LOG: [NEON_SMGR] [shard 0] no response received from pageserver for 10.000 s, still waiting (sent 10613 requests, received 10612 responses) PG:2024-12-09 16:02:17.723 GMT [1782845] LOG: [NEON_SMGR] [shard 0] no response received from pageserver for 20.008 s, still waiting (sent 10613 requests, received 10612 responses) PG:2024-12-09 16:02:19.719 GMT [1782845] LOG: [NEON_SMGR] [shard 0] received response from pageserver after 22.006 s ``` --- pgxn/neon/libpagestore.c | 67 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 66 insertions(+), 1 deletion(-) diff --git a/pgxn/neon/libpagestore.c b/pgxn/neon/libpagestore.c index b60ae41af3..6513ba4dd6 100644 --- a/pgxn/neon/libpagestore.c +++ b/pgxn/neon/libpagestore.c @@ -22,6 +22,7 @@ #include "libpq/pqformat.h" #include "miscadmin.h" #include "pgstat.h" +#include "portability/instr_time.h" #include "postmaster/interrupt.h" #include "storage/buf_internals.h" #include "storage/ipc.h" @@ -118,6 +119,11 @@ typedef struct */ PSConnectionState state; PGconn *conn; + + /* request / response counters for debugging */ + uint64 nrequests_sent; + uint64 nresponses_received; + /*--- * WaitEventSet containing: * - WL_SOCKET_READABLE on 'conn' @@ -628,6 +634,8 @@ pageserver_connect(shardno_t shard_no, int elevel) } shard->state = PS_Connected; + shard->nrequests_sent = 0; + shard->nresponses_received = 0; } /* FALLTHROUGH */ case PS_Connected: @@ -656,6 +664,27 @@ call_PQgetCopyData(shardno_t shard_no, char **buffer) int ret; PageServer *shard = &page_servers[shard_no]; PGconn *pageserver_conn = shard->conn; + instr_time now, + start_ts, + since_start, + last_log_ts, + since_last_log; + bool logged = false; + + /* + * As a debugging aid, if we don't get a response for a long time, print a + * log message. + * + * 10 s is a very generous threshold, normally we expect a response in a + * few milliseconds. We have metrics to track latencies in normal ranges, + * but in the cases that take exceptionally long, it's useful to log the + * exact timestamps. + */ +#define LOG_INTERVAL_US UINT64CONST(10 * 1000000) + + INSTR_TIME_SET_CURRENT(now); + start_ts = last_log_ts = now; + INSTR_TIME_SET_ZERO(since_last_log); retry: ret = PQgetCopyData(pageserver_conn, buffer, 1 /* async */ ); @@ -663,9 +692,12 @@ retry: if (ret == 0) { WaitEvent event; + long timeout; + + timeout = Min(0, LOG_INTERVAL_US - INSTR_TIME_GET_MICROSEC(since_last_log)); /* Sleep until there's something to do */ - (void) WaitEventSetWait(shard->wes_read, -1L, &event, 1, + (void) WaitEventSetWait(shard->wes_read, timeout, &event, 1, WAIT_EVENT_NEON_PS_READ); ResetLatch(MyLatch); @@ -684,9 +716,40 @@ retry: } } + /* + * Print a message to the log if a long time has passed with no + * response. + */ + INSTR_TIME_SET_CURRENT(now); + since_last_log = now; + INSTR_TIME_SUBTRACT(since_last_log, last_log_ts); + if (INSTR_TIME_GET_MICROSEC(since_last_log) >= LOG_INTERVAL_US) + { + since_start = now; + INSTR_TIME_SUBTRACT(since_start, start_ts); + neon_shard_log(shard_no, LOG, "no response received from pageserver for %0.3f s, still waiting (sent " UINT64_FORMAT " requests, received " UINT64_FORMAT " responses)", + INSTR_TIME_GET_DOUBLE(since_start), + shard->nrequests_sent, shard->nresponses_received); + last_log_ts = now; + logged = true; + } + goto retry; } + /* + * If we logged earlier that the response is taking a long time, log + * another message when the response is finally received. + */ + if (logged) + { + INSTR_TIME_SET_CURRENT(now); + since_start = now; + INSTR_TIME_SUBTRACT(since_start, start_ts); + neon_shard_log(shard_no, LOG, "received response from pageserver after %0.3f s", + INSTR_TIME_GET_DOUBLE(since_start)); + } + return ret; } @@ -786,6 +849,7 @@ pageserver_send(shardno_t shard_no, NeonRequest *request) * PGRES_POLLING_WRITING state. It's kinda dirty to disconnect at this * point, but on the grand scheme of things it's only a small issue. */ + shard->nrequests_sent++; if (PQputCopyData(pageserver_conn, req_buff.data, req_buff.len) <= 0) { char *msg = pchomp(PQerrorMessage(pageserver_conn)); @@ -878,6 +942,7 @@ pageserver_receive(shardno_t shard_no) neon_shard_log(shard_no, ERROR, "pageserver_receive disconnect: unexpected PQgetCopyData return value: %d", rc); } + shard->nresponses_received++; return (NeonResponse *) resp; } From aa0554fd1ef003ddd6a51e5c40944351caa130ac Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Tue, 10 Dec 2024 12:00:47 -0500 Subject: [PATCH 11/44] feat(test_runner): allowed_errors in storage scrubber (#10062) ## Problem resolve https://github.com/neondatabase/neon/issues/9988#issuecomment-2528239437 ## Summary of changes * New verbose mode for storage scrubber scan metadata (pageserver) that contains the error messages. * Filter allowed_error list from the JSON output to determine the healthy flag status. --------- Signed-off-by: Alex Chi Z --- storage_scrubber/src/main.rs | 8 ++- .../src/scan_pageserver_metadata.rs | 39 +++++++++---- test_runner/fixtures/neon_fixtures.py | 58 ++++++++++++++++++- test_runner/regress/test_storage_scrubber.py | 8 ++- 4 files changed, 98 insertions(+), 15 deletions(-) diff --git a/storage_scrubber/src/main.rs b/storage_scrubber/src/main.rs index 92979d609e..fa6ee90b66 100644 --- a/storage_scrubber/src/main.rs +++ b/storage_scrubber/src/main.rs @@ -86,6 +86,8 @@ enum Command { /// For safekeeper node_kind only, json list of timelines and their lsn info #[arg(long, default_value = None)] timeline_lsns: Option, + #[arg(long, default_value_t = false)] + verbose: bool, }, TenantSnapshot { #[arg(long = "tenant-id")] @@ -166,6 +168,7 @@ async fn main() -> anyhow::Result<()> { dump_db_connstr, dump_db_table, timeline_lsns, + verbose, } => { if let NodeKind::Safekeeper = node_kind { let db_or_list = match (timeline_lsns, dump_db_connstr) { @@ -203,6 +206,7 @@ async fn main() -> anyhow::Result<()> { tenant_ids, json, post_to_storcon, + verbose, cli.exit_code, ) .await @@ -313,6 +317,7 @@ pub async fn run_cron_job( Vec::new(), true, post_to_storcon, + false, // default to non-verbose mode exit_code, ) .await?; @@ -362,12 +367,13 @@ pub async fn scan_pageserver_metadata_cmd( tenant_shard_ids: Vec, json: bool, post_to_storcon: bool, + verbose: bool, exit_code: bool, ) -> anyhow::Result<()> { if controller_client.is_none() && post_to_storcon { return Err(anyhow!("Posting pageserver scan health status to storage controller requires `--controller-api` and `--controller-jwt` to run")); } - match scan_pageserver_metadata(bucket_config.clone(), tenant_shard_ids).await { + match scan_pageserver_metadata(bucket_config.clone(), tenant_shard_ids, verbose).await { Err(e) => { tracing::error!("Failed: {e}"); Err(e) diff --git a/storage_scrubber/src/scan_pageserver_metadata.rs b/storage_scrubber/src/scan_pageserver_metadata.rs index cb3299d413..c8de6e46b3 100644 --- a/storage_scrubber/src/scan_pageserver_metadata.rs +++ b/storage_scrubber/src/scan_pageserver_metadata.rs @@ -21,8 +21,12 @@ pub struct MetadataSummary { tenant_count: usize, timeline_count: usize, timeline_shard_count: usize, - with_errors: HashSet, - with_warnings: HashSet, + /// Tenant-shard timeline (key) mapping to errors. The key has to be a string because it will be serialized to a JSON. + /// The key is generated using `TenantShardTimelineId::to_string()`. + with_errors: HashMap>, + /// Tenant-shard timeline (key) mapping to warnings. The key has to be a string because it will be serialized to a JSON. + /// The key is generated using `TenantShardTimelineId::to_string()`. + with_warnings: HashMap>, with_orphans: HashSet, indices_by_version: HashMap, @@ -52,7 +56,12 @@ impl MetadataSummary { } } - fn update_analysis(&mut self, id: &TenantShardTimelineId, analysis: &TimelineAnalysis) { + fn update_analysis( + &mut self, + id: &TenantShardTimelineId, + analysis: &TimelineAnalysis, + verbose: bool, + ) { if analysis.is_healthy() { self.healthy_tenant_shards.insert(id.tenant_shard_id); } else { @@ -61,11 +70,17 @@ impl MetadataSummary { } if !analysis.errors.is_empty() { - self.with_errors.insert(*id); + let entry = self.with_errors.entry(id.to_string()).or_default(); + if verbose { + entry.extend(analysis.errors.iter().cloned()); + } } if !analysis.warnings.is_empty() { - self.with_warnings.insert(*id); + let entry = self.with_warnings.entry(id.to_string()).or_default(); + if verbose { + entry.extend(analysis.warnings.iter().cloned()); + } } } @@ -120,6 +135,7 @@ Index versions: {version_summary} pub async fn scan_pageserver_metadata( bucket_config: BucketConfig, tenant_ids: Vec, + verbose: bool, ) -> anyhow::Result { let (remote_client, target) = init_remote(bucket_config, NodeKind::Pageserver).await?; @@ -164,6 +180,7 @@ pub async fn scan_pageserver_metadata( mut tenant_objects: TenantObjectListing, timelines: Vec<(TenantShardTimelineId, RemoteTimelineBlobData)>, highest_shard_count: ShardCount, + verbose: bool, ) { summary.tenant_count += 1; @@ -203,7 +220,7 @@ pub async fn scan_pageserver_metadata( Some(data), ) .await; - summary.update_analysis(&ttid, &analysis); + summary.update_analysis(&ttid, &analysis, verbose); timeline_ids.insert(ttid.timeline_id); } else { @@ -271,10 +288,6 @@ pub async fn scan_pageserver_metadata( summary.update_data(&data); match tenant_id { - None => { - tenant_id = Some(ttid.tenant_shard_id.tenant_id); - highest_shard_count = highest_shard_count.max(ttid.tenant_shard_id.shard_count); - } Some(prev_tenant_id) => { if prev_tenant_id != ttid.tenant_shard_id.tenant_id { // New tenant: analyze this tenant's timelines, clear accumulated tenant_timeline_results @@ -287,6 +300,7 @@ pub async fn scan_pageserver_metadata( tenant_objects, timelines, highest_shard_count, + verbose, ) .instrument(info_span!("analyze-tenant", tenant = %prev_tenant_id)) .await; @@ -296,6 +310,10 @@ pub async fn scan_pageserver_metadata( highest_shard_count = highest_shard_count.max(ttid.tenant_shard_id.shard_count); } } + None => { + tenant_id = Some(ttid.tenant_shard_id.tenant_id); + highest_shard_count = highest_shard_count.max(ttid.tenant_shard_id.shard_count); + } } match &data.blob_data { @@ -326,6 +344,7 @@ pub async fn scan_pageserver_metadata( tenant_objects, tenant_timeline_results, highest_shard_count, + verbose, ) .instrument(info_span!("analyze-tenant", tenant = %tenant_id)) .await; diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 60c4a23936..8354432c0c 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -4556,6 +4556,7 @@ class StorageScrubber: def __init__(self, env: NeonEnv, log_dir: Path): self.env = env self.log_dir = log_dir + self.allowed_errors: list[str] = [] def scrubber_cli( self, args: list[str], timeout, extra_env: dict[str, str] | None = None @@ -4633,19 +4634,70 @@ class StorageScrubber: if timeline_lsns is not None: args.append("--timeline-lsns") args.append(json.dumps(timeline_lsns)) + if node_kind == NodeKind.PAGESERVER: + args.append("--verbose") stdout = self.scrubber_cli(args, timeout=30, extra_env=extra_env) try: summary = json.loads(stdout) - # summary does not contain "with_warnings" if node_kind is the safekeeper - no_warnings = "with_warnings" not in summary or not summary["with_warnings"] - healthy = not summary["with_errors"] and no_warnings + healthy = self._check_run_healthy(summary) return healthy, summary except: log.error("Failed to decode JSON output from `scan-metadata`. Dumping stdout:") log.error(stdout) raise + def _check_line_allowed(self, line: str) -> bool: + for a in self.allowed_errors: + try: + if re.match(a, line): + return True + except re.error: + log.error(f"Invalid regex: '{a}'") + raise + return False + + def _check_line_list_allowed(self, lines: list[str]) -> bool: + for line in lines: + if not self._check_line_allowed(line): + return False + return True + + def _check_run_healthy(self, summary: dict[str, Any]) -> bool: + # summary does not contain "with_warnings" if node_kind is the safekeeper + healthy = True + with_warnings = summary.get("with_warnings", None) + if with_warnings is not None: + if isinstance(with_warnings, list): + if len(with_warnings) > 0: + # safekeeper scan_metadata output is a list of tenants + healthy = False + else: + for _, warnings in with_warnings.items(): + assert ( + len(warnings) > 0 + ), "with_warnings value should not be empty, running without verbose mode?" + if not self._check_line_list_allowed(warnings): + healthy = False + break + if not healthy: + return healthy + with_errors = summary.get("with_errors", None) + if with_errors is not None: + if isinstance(with_errors, list): + if len(with_errors) > 0: + # safekeeper scan_metadata output is a list of tenants + healthy = False + else: + for _, errors in with_errors.items(): + assert ( + len(errors) > 0 + ), "with_errors value should not be empty, running without verbose mode?" + if not self._check_line_list_allowed(errors): + healthy = False + break + return healthy + def tenant_snapshot(self, tenant_id: TenantId, output_path: Path): stdout = self.scrubber_cli( ["tenant-snapshot", "--tenant-id", str(tenant_id), "--output-path", str(output_path)], diff --git a/test_runner/regress/test_storage_scrubber.py b/test_runner/regress/test_storage_scrubber.py index b16dc54c24..198e4f0460 100644 --- a/test_runner/regress/test_storage_scrubber.py +++ b/test_runner/regress/test_storage_scrubber.py @@ -572,4 +572,10 @@ def test_scrubber_scan_pageserver_metadata( unhealthy = env.storage_controller.metadata_health_list_unhealthy()["unhealthy_tenant_shards"] assert len(unhealthy) == 1 and unhealthy[0] == str(tenant_shard_id) - neon_env_builder.disable_scrub_on_exit() + healthy, _ = env.storage_scrubber.scan_metadata() + assert not healthy + env.storage_scrubber.allowed_errors.append(".*not present in remote storage.*") + healthy, _ = env.storage_scrubber.scan_metadata() + assert healthy + + neon_env_builder.disable_scrub_on_exit() # We already ran scrubber, no need to do an extra run From e71d20d3928dd03e1159ec5a08cfa902cf85cd31 Mon Sep 17 00:00:00 2001 From: Matthias van de Meent Date: Tue, 10 Dec 2024 20:42:52 +0100 Subject: [PATCH 12/44] Emit nbtree vacuum cycle id in nbtree xlog through forced FPIs (#9932) This fixes neondatabase/neon#9929. ## Postgres repo PRS: - PG17: https://github.com/neondatabase/postgres/pull/538 - PG16: https://github.com/neondatabase/postgres/pull/539 - PG15: https://github.com/neondatabase/postgres/pull/540 - PG14: https://github.com/neondatabase/postgres/pull/541 ## Problem see #9929 ## Summary of changes We update the split code to force the code to emit an FPI whenever the cycle ID might be interesting for concurrent btree vacuum. --- .../regress/test_nbtree_pagesplit_cycleid.py | 124 ++++++++++++++++++ vendor/postgres-v14 | 2 +- vendor/postgres-v15 | 2 +- vendor/postgres-v16 | 2 +- vendor/postgres-v17 | 2 +- vendor/revisions.json | 8 +- 6 files changed, 132 insertions(+), 8 deletions(-) create mode 100644 test_runner/regress/test_nbtree_pagesplit_cycleid.py diff --git a/test_runner/regress/test_nbtree_pagesplit_cycleid.py b/test_runner/regress/test_nbtree_pagesplit_cycleid.py new file mode 100644 index 0000000000..558557aeba --- /dev/null +++ b/test_runner/regress/test_nbtree_pagesplit_cycleid.py @@ -0,0 +1,124 @@ +import threading +import time + +from fixtures.neon_fixtures import NeonEnv + +BTREE_NUM_CYCLEID_PAGES = """ + WITH raw_pages AS ( + SELECT blkno, get_raw_page_at_lsn('t_uidx', 'main', blkno, NULL, NULL) page + FROM generate_series(1, pg_relation_size('t_uidx'::regclass) / 8192) blkno + ), + parsed_pages AS ( + /* cycle ID is the last 2 bytes of the btree page */ + SELECT blkno, SUBSTRING(page FROM 8191 FOR 2) as cycle_id + FROM raw_pages + ) + SELECT count(*), + encode(cycle_id, 'hex') + FROM parsed_pages + WHERE encode(cycle_id, 'hex') != '0000' + GROUP BY encode(cycle_id, 'hex'); + """ + + +def test_nbtree_pagesplit_cycleid(neon_simple_env: NeonEnv): + env = neon_simple_env + endpoint = env.endpoints.create_start("main") + + ses1 = endpoint.connect().cursor() + ses1.execute("ALTER SYSTEM SET autovacuum = off;") + ses1.execute("ALTER SYSTEM SET enable_seqscan = off;") + ses1.execute("ALTER SYSTEM SET full_page_writes = off;") + ses1.execute("SELECT pg_reload_conf();") + ses1.execute("CREATE EXTENSION neon_test_utils;") + # prepare a large index + ses1.execute("CREATE TABLE t(id integer GENERATED ALWAYS AS IDENTITY, txt text);") + ses1.execute("CREATE UNIQUE INDEX t_uidx ON t(id);") + ses1.execute("INSERT INTO t (txt) SELECT i::text FROM generate_series(1, 2035) i;") + + ses1.execute("SELECT neon_xlogflush();") + ses1.execute(BTREE_NUM_CYCLEID_PAGES) + pages = ses1.fetchall() + assert ( + len(pages) == 0 + ), f"0 back splits with cycle ID expected, real {len(pages)} first {pages[0]}" + # Delete enough tuples to clear the first index page. + # (there are up to 407 rows per 8KiB page; 406 for non-rightmost leafs. + ses1.execute("DELETE FROM t WHERE id <= 406;") + # Make sure the page is cleaned up + ses1.execute("VACUUM (FREEZE, INDEX_CLEANUP ON) t;") + + # Do another delete-then-indexcleanup cycle, to move the pages from + # "dead" to "reusable" + ses1.execute("DELETE FROM t WHERE id <= 446;") + ses1.execute("VACUUM (FREEZE, INDEX_CLEANUP ON) t;") + + # Make sure the vacuum we're about to trigger in s3 has cleanup work to do + ses1.execute("DELETE FROM t WHERE id <= 610;") + + # Flush wal, for checking purposes + ses1.execute("SELECT neon_xlogflush();") + ses1.execute(BTREE_NUM_CYCLEID_PAGES) + pages = ses1.fetchall() + assert len(pages) == 0, f"No back splits with cycle ID expected, got batches of {pages} instead" + + ses2 = endpoint.connect().cursor() + ses3 = endpoint.connect().cursor() + + # Session 2 pins a btree page, which prevents vacuum from processing that + # page, thus allowing us to reliably split pages while a concurrent vacuum + # is running. + ses2.execute("BEGIN;") + ses2.execute( + "DECLARE foo NO SCROLL CURSOR FOR SELECT row_number() over () FROM t ORDER BY id ASC" + ) + ses2.execute("FETCH FROM foo;") # pins the leaf page with id 611 + wait_evt = threading.Event() + + # Session 3 runs the VACUUM command. Note that this will block, and + # therefore must run on another thread. + # We rely on this running quickly enough to hit the pinned page from + # session 2 by the time we start other work again in session 1, but + # technically there is a race where the thread (and/or PostgreSQL process) + # don't get to that pinned page with vacuum until >2s after evt.set() was + # called, and session 1 thus might already have split pages. + def vacuum_freeze_t(ses3, evt: threading.Event): + # Begin parallel vacuum that should hit the index + evt.set() + # this'll hang until s2 fetches enough new data from its cursor. + # this is technically a race with the time.sleep(2) below, but if this + # command doesn't hit + ses3.execute("VACUUM (FREEZE, INDEX_CLEANUP on, DISABLE_PAGE_SKIPPING on) t;") + + ses3t = threading.Thread(target=vacuum_freeze_t, args=(ses3, wait_evt)) + ses3t.start() + wait_evt.wait() + # Make extra sure we got the thread started and vacuum is stuck, by waiting + # some time even after wait_evt got set. This isn't truly reliable (it is + # possible + time.sleep(2) + + # Insert 2 pages worth of new data. + # This should reuse the one empty page, plus another page at the end of + # the index relation; with split ordering + # old_blk -> blkno=1 -> old_blk + 1. + # As this is run while vacuum in session 3 is happening, these splits + # should receive cycle IDs where applicable. + ses1.execute("INSERT INTO t (txt) SELECT i::text FROM generate_series(1, 812) i;") + # unpin the btree page, allowing s3's vacuum to complete + ses2.execute("FETCH ALL FROM foo;") + ses2.execute("ROLLBACK;") + # flush WAL to make sure PS is up-to-date + ses1.execute("SELECT neon_xlogflush();") + # check that our expectations are correct + ses1.execute(BTREE_NUM_CYCLEID_PAGES) + pages = ses1.fetchall() + assert ( + len(pages) == 1 and pages[0][0] == 3 + ), f"3 page splits with cycle ID expected; actual {pages}" + + # final cleanup + ses3t.join() + ses1.close() + ses2.close() + ses3.close() diff --git a/vendor/postgres-v14 b/vendor/postgres-v14 index 373f9decad..13ff324150 160000 --- a/vendor/postgres-v14 +++ b/vendor/postgres-v14 @@ -1 +1 @@ -Subproject commit 373f9decad933d2d46f321231032ae8b0da81acd +Subproject commit 13ff324150fceaac72920e01742addc053db9462 diff --git a/vendor/postgres-v15 b/vendor/postgres-v15 index 972e325e62..8736b10c1d 160000 --- a/vendor/postgres-v15 +++ b/vendor/postgres-v15 @@ -1 +1 @@ -Subproject commit 972e325e62b455957adbbdd8580e31275bb5b8c9 +Subproject commit 8736b10c1d93d11b9c0489872dd529c4c0f5338f diff --git a/vendor/postgres-v16 b/vendor/postgres-v16 index dff6615a8e..81428621f7 160000 --- a/vendor/postgres-v16 +++ b/vendor/postgres-v16 @@ -1 +1 @@ -Subproject commit dff6615a8e48a10bb17a03fa3c00635f1ace7a92 +Subproject commit 81428621f7c04aed03671cf80a928e0a36d92505 diff --git a/vendor/postgres-v17 b/vendor/postgres-v17 index a10d95be67..471c449ab8 160000 --- a/vendor/postgres-v17 +++ b/vendor/postgres-v17 @@ -1 +1 @@ -Subproject commit a10d95be67265e0f10a422ba0457f5a7af01de71 +Subproject commit 471c449ab8f8ff5988b6bfb9eafa0a79772ad562 diff --git a/vendor/revisions.json b/vendor/revisions.json index 8a73e14dcf..ba0f34e23e 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,18 +1,18 @@ { "v17": [ "17.2", - "a10d95be67265e0f10a422ba0457f5a7af01de71" + "471c449ab8f8ff5988b6bfb9eafa0a79772ad562" ], "v16": [ "16.6", - "dff6615a8e48a10bb17a03fa3c00635f1ace7a92" + "81428621f7c04aed03671cf80a928e0a36d92505" ], "v15": [ "15.10", - "972e325e62b455957adbbdd8580e31275bb5b8c9" + "8736b10c1d93d11b9c0489872dd529c4c0f5338f" ], "v14": [ "14.15", - "373f9decad933d2d46f321231032ae8b0da81acd" + "13ff324150fceaac72920e01742addc053db9462" ] } From 597125e124b3d92ef3a0ca243722aa0f99238037 Mon Sep 17 00:00:00 2001 From: Matthias van de Meent Date: Wed, 11 Dec 2024 01:51:05 +0100 Subject: [PATCH 13/44] Disable readstream's reliance on seqscan readahead (#9860) Neon doesn't have seqscan detection of its own, so stop read_stream from trying to utilize that readahead, and instead make it issue readahead of its own. ## Problem @knizhnik noticed that we didn't issue smgrprefetch[v] calls for seqscans in PG17 due to the move to the read_stream API, which assumes that the underlying IO facilities do seqscan detection for readahead. That is a wrong assumption when Neon is involved, so let's remove the code that applies that assumption. ## Summary of changes Remove the cases where seqscans are detected and prefetch is disabled as a consequence, and instead don't do that detection. PG PR: https://github.com/neondatabase/postgres/pull/532 --- vendor/postgres-v17 | 2 +- vendor/revisions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vendor/postgres-v17 b/vendor/postgres-v17 index 471c449ab8..01fa3c4866 160000 --- a/vendor/postgres-v17 +++ b/vendor/postgres-v17 @@ -1 +1 @@ -Subproject commit 471c449ab8f8ff5988b6bfb9eafa0a79772ad562 +Subproject commit 01fa3c48664ca030cfb69bb4a350aa9df4691d88 diff --git a/vendor/revisions.json b/vendor/revisions.json index ba0f34e23e..7329aa437f 100644 --- a/vendor/revisions.json +++ b/vendor/revisions.json @@ -1,7 +1,7 @@ { "v17": [ "17.2", - "471c449ab8f8ff5988b6bfb9eafa0a79772ad562" + "01fa3c48664ca030cfb69bb4a350aa9df4691d88" ], "v16": [ "16.6", From 38415a9816dab1ccd1fadea77857aafb369abdf6 Mon Sep 17 00:00:00 2001 From: John Spray Date: Wed, 11 Dec 2024 09:16:11 +0000 Subject: [PATCH 14/44] pageserver: fix ingest handling of CLog truncate (#10080) ## Problem In #9786 we stop storing SLRUs on non-zero shards. However, there was one code path during ingest that still tries to enumerate SLRU relations on all shards. This fails if it sees a tenant who has never seen any write to an SLRU, or who has done such thorough compaction+GC that it has dropped its SLRU directory key. ## Summary of changes - Avoid trying to list SLRU relations on nonzero shards --- pageserver/src/walingest.rs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index 30c8965d51..b7712cfac7 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -877,22 +877,24 @@ impl WalIngest { // will block waiting for the last valid LSN to advance up to // it. So we use the previous record's LSN in the get calls // instead. - for segno in modification - .tline - .list_slru_segments(SlruKind::Clog, Version::Modified(modification), ctx) - .await? - { - let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT; + if modification.tline.get_shard_identity().is_shard_zero() { + for segno in modification + .tline + .list_slru_segments(SlruKind::Clog, Version::Modified(modification), ctx) + .await? + { + let segpage = segno * pg_constants::SLRU_PAGES_PER_SEGMENT; - let may_delete = dispatch_pgversion!(modification.tline.pg_version, { - pgv::nonrelfile_utils::slru_may_delete_clogsegment(segpage, pageno) - }); + let may_delete = dispatch_pgversion!(modification.tline.pg_version, { + pgv::nonrelfile_utils::slru_may_delete_clogsegment(segpage, pageno) + }); - if may_delete { - modification - .drop_slru_segment(SlruKind::Clog, segno, ctx) - .await?; - trace!("Drop CLOG segment {:>04X}", segno); + if may_delete { + modification + .drop_slru_segment(SlruKind::Clog, segno, ctx) + .await?; + trace!("Drop CLOG segment {:>04X}", segno); + } } } From d7aeca2f343675f17f2adee05082f89644715469 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JC=20Gr=C3=BCnhage?= Date: Wed, 11 Dec 2024 10:41:34 +0100 Subject: [PATCH 15/44] CI(deploy): create git tags/releases before triggering deploy workflows (#10022) ## Problem When dev deployments are disabled (or fail), the tags for releases aren't created. It makes more sense to have tag and release creation before the deployment to prevent situations like [this](https://github.com/neondatabase/neon/pull/9959). It is not enough to move the tag creation before the deployment. If the deployment fails, re-running the job isn't possible because the API call to create the tag will fail. ## Summary of changes - Tag/Release creation now happens before the deployment - The two steps for tag and release have been merged into a bigger one - There's new checks to ensure the that if the tags/releases already exist as expected, things will continue just fine. --- .github/workflows/build_and_test.yml | 93 +++++++++++++++++++--------- 1 file changed, 64 insertions(+), 29 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index cb966f292e..6023d1bb6f 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1066,6 +1066,70 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Create git tag and GitHub release + if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' + uses: actions/github-script@v7 + with: + retries: 5 + script: | + const tag = "${{ needs.tag.outputs.build-tag }}"; + + try { + const existingRef = await github.rest.git.getRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `tags/${tag}`, + }); + + if (existingRef.data.object.sha !== context.sha) { + throw new Error(`Tag ${tag} already exists but points to a different commit (expected: ${context.sha}, actual: ${existingRef.data.object.sha}).`); + } + + console.log(`Tag ${tag} already exists and points to ${context.sha} as expected.`); + } catch (error) { + if (error.status !== 404) { + throw error; + } + + console.log(`Tag ${tag} does not exist. Creating it...`); + await github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/tags/${tag}`, + sha: context.sha, + }); + console.log(`Tag ${tag} created successfully.`); + } + + # TODO: check how GitHub releases looks for proxy/compute releases and enable them if they're ok + if (context.ref !== 'refs/heads/release') { + console.log(`GitHub release skipped for ${context.ref}.`); + return; + } + + try { + const existingRelease = await github.rest.repos.getReleaseByTag({ + owner: context.repo.owner, + repo: context.repo.repo, + tag: tag, + }); + + console.log(`Release for tag ${tag} already exists (ID: ${existingRelease.data.id}).`); + } catch (error) { + if (error.status !== 404) { + throw error; + } + + console.log(`Release for tag ${tag} does not exist. Creating it...`); + await github.rest.repos.createRelease({ + owner: context.repo.owner, + repo: context.repo.repo, + tag_name: tag, + generate_release_notes: true, + }); + console.log(`Release for tag ${tag} created successfully.`); + } + - name: Trigger deploy workflow env: GH_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }} @@ -1115,35 +1179,6 @@ jobs: exit 1 fi - - name: Create git tag - if: github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute' - uses: actions/github-script@v7 - with: - # Retry script for 5XX server errors: https://github.com/actions/github-script#retries - retries: 5 - script: | - await github.rest.git.createRef({ - owner: context.repo.owner, - repo: context.repo.repo, - ref: "refs/tags/${{ needs.tag.outputs.build-tag }}", - sha: context.sha, - }) - - # TODO: check how GitHub releases looks for proxy releases and enable it if it's ok - - name: Create GitHub release - if: github.ref_name == 'release' - uses: actions/github-script@v7 - with: - # Retry script for 5XX server errors: https://github.com/actions/github-script#retries - retries: 5 - script: | - await github.rest.repos.createRelease({ - owner: context.repo.owner, - repo: context.repo.repo, - tag_name: "${{ needs.tag.outputs.build-tag }}", - generate_release_notes: true, - }) - # The job runs on `release` branch and copies compatibility data and Neon artifact from the last *release PR* to the latest directory promote-compatibility-data: needs: [ deploy ] From 665369c439d6ec1d107dea7ccc80ce64b080297a Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Wed, 11 Dec 2024 12:35:02 +0000 Subject: [PATCH 16/44] wal_decoder: fix compact key protobuf encoding (#10074) ## Problem Protobuf doesn't support 128 bit integers, so we encode the keys as two 64 bit integers. Issue is that when we split the 128 bit compact key we use signed 64 bit integers to represent the two halves. This may result in a negative lower half when relnode is larger than `0x00800000`. When we convert the lower half to an i128 we get a negative `CompactKey`. ## Summary of Changes Use unsigned integers when encoding into Protobuf. ## Deployment * Prod: We disabled the interpreted proto, so no compat concerns. * Staging: Disable the interpreted proto, do one release, and then release the fixed version. We do this because a negative int32 will convert to a large uint32 value and could give a key in the actual pageserver space. In production we would around this by adding new fields to the proto and deprecating the old ones, but we can make our lives easy here. * Pre-prod: Same as staging --- libs/pageserver_api/src/key.rs | 2 +- libs/wal_decoder/proto/interpreted_wal.proto | 4 +- libs/wal_decoder/src/wire_format.rs | 65 +++++++++++++++++++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/libs/pageserver_api/src/key.rs b/libs/pageserver_api/src/key.rs index 37dff6fe46..373329c9b4 100644 --- a/libs/pageserver_api/src/key.rs +++ b/libs/pageserver_api/src/key.rs @@ -24,7 +24,7 @@ pub struct Key { /// When working with large numbers of Keys in-memory, it is more efficient to handle them as i128 than as /// a struct of fields. -#[derive(Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize)] +#[derive(Clone, Copy, Hash, PartialEq, Eq, Ord, PartialOrd, Serialize, Deserialize, Debug)] pub struct CompactKey(i128); /// The storage key size. diff --git a/libs/wal_decoder/proto/interpreted_wal.proto b/libs/wal_decoder/proto/interpreted_wal.proto index 0393392c1a..d68484d30f 100644 --- a/libs/wal_decoder/proto/interpreted_wal.proto +++ b/libs/wal_decoder/proto/interpreted_wal.proto @@ -37,7 +37,7 @@ message ValueMeta { } message CompactKey { - int64 high = 1; - int64 low = 2; + uint64 high = 1; + uint64 low = 2; } diff --git a/libs/wal_decoder/src/wire_format.rs b/libs/wal_decoder/src/wire_format.rs index 5a343054c3..944ee5c919 100644 --- a/libs/wal_decoder/src/wire_format.rs +++ b/libs/wal_decoder/src/wire_format.rs @@ -236,8 +236,8 @@ impl From for proto::ValueMeta { impl From for proto::CompactKey { fn from(value: CompactKey) -> Self { proto::CompactKey { - high: (value.raw() >> 64) as i64, - low: value.raw() as i64, + high: (value.raw() >> 64) as u64, + low: value.raw() as u64, } } } @@ -354,3 +354,64 @@ impl From for CompactKey { (((value.high as i128) << 64) | (value.low as i128)).into() } } + +#[test] +fn test_compact_key_with_large_relnode() { + use pageserver_api::key::Key; + + let inputs = vec![ + Key { + field1: 0, + field2: 0x100, + field3: 0x200, + field4: 0, + field5: 0x10, + field6: 0x5, + }, + Key { + field1: 0, + field2: 0x100, + field3: 0x200, + field4: 0x007FFFFF, + field5: 0x10, + field6: 0x5, + }, + Key { + field1: 0, + field2: 0x100, + field3: 0x200, + field4: 0x00800000, + field5: 0x10, + field6: 0x5, + }, + Key { + field1: 0, + field2: 0x100, + field3: 0x200, + field4: 0x00800001, + field5: 0x10, + field6: 0x5, + }, + Key { + field1: 0, + field2: 0xFFFFFFFF, + field3: 0xFFFFFFFF, + field4: 0xFFFFFFFF, + field5: 0x0, + field6: 0x0, + }, + ]; + + for input in inputs { + assert!(input.is_valid_key_on_write_path()); + let compact = input.to_compact(); + let proto: proto::CompactKey = compact.into(); + let from_proto: CompactKey = proto.into(); + + assert_eq!( + compact, from_proto, + "Round trip failed for key with relnode={:#x}", + input.field4 + ); + } +} From 9ae980bf4f320c20dc48adce90e92e74fd2ea45c Mon Sep 17 00:00:00 2001 From: Christian Schwarz Date: Wed, 11 Dec 2024 14:37:08 +0100 Subject: [PATCH 17/44] page_service: don't count time spent in Batcher towards smgr latency metrics (#10075) ## Problem With pipelining enabled, the time a request spends in the batcher stage counts towards the smgr op latency. If pipelining is disabled, that time is not accounted for. In practice, this results in a jump in smgr getpage latencies in various dashboards and degrades the internal SLO. ## Solution In a similar vein to #10042 and with a similar rationale, this PR stops counting the time spent in batcher stage towards smgr op latency. The smgr op latency metric is reduced to the actual execution time. Time spent in batcher stage is tracked in a separate histogram. I expect to remove that histogram after batching rollout is complete, but it will be helpful in the meantime to reason about the rollout. --- pageserver/src/metrics.rs | 168 +++++++++++++++++++++--------- pageserver/src/page_service.rs | 13 ++- pageserver/src/tenant/throttle.rs | 19 ++-- test_runner/fixtures/metrics.py | 1 + 4 files changed, 143 insertions(+), 58 deletions(-) diff --git a/pageserver/src/metrics.rs b/pageserver/src/metrics.rs index 96ee157856..b4e20cb8b9 100644 --- a/pageserver/src/metrics.rs +++ b/pageserver/src/metrics.rs @@ -16,7 +16,6 @@ use postgres_backend::{is_expected_io_error, QueryError}; use pq_proto::framed::ConnectionError; use strum::{EnumCount, VariantNames}; use strum_macros::{IntoStaticStr, VariantNames}; -use tracing::warn; use utils::id::TimelineId; /// Prometheus histogram buckets (in seconds) for operations in the critical @@ -1225,32 +1224,58 @@ pub(crate) mod virtual_file_io_engine { pub(crate) struct SmgrOpTimer(Option); pub(crate) struct SmgrOpTimerInner { - global_latency_histo: Histogram, + global_execution_latency_histo: Histogram, + per_timeline_execution_latency_histo: Option, - // Optional because not all op types are tracked per-timeline - per_timeline_latency_histo: Option, + global_batch_wait_time: Histogram, + per_timeline_batch_wait_time: Histogram, global_flush_in_progress_micros: IntCounter, per_timeline_flush_in_progress_micros: IntCounter, - start: Instant, - throttled: Duration, - op: SmgrQueryType, + timings: SmgrOpTimerState, +} + +#[derive(Debug)] +enum SmgrOpTimerState { + Received { + received_at: Instant, + }, + ThrottleDoneExecutionStarting { + received_at: Instant, + throttle_started_at: Instant, + started_execution_at: Instant, + }, } pub(crate) struct SmgrOpFlushInProgress { - base: Instant, + flush_started_at: Instant, global_micros: IntCounter, per_timeline_micros: IntCounter, } impl SmgrOpTimer { - pub(crate) fn deduct_throttle(&mut self, throttle: &Option) { - let Some(throttle) = throttle else { - return; - }; + pub(crate) fn observe_throttle_done_execution_starting(&mut self, throttle: &ThrottleResult) { let inner = self.0.as_mut().expect("other public methods consume self"); - inner.throttled += *throttle; + match (&mut inner.timings, throttle) { + (SmgrOpTimerState::Received { received_at }, throttle) => match throttle { + ThrottleResult::NotThrottled { start } => { + inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting { + received_at: *received_at, + throttle_started_at: *start, + started_execution_at: *start, + }; + } + ThrottleResult::Throttled { start, end } => { + inner.timings = SmgrOpTimerState::ThrottleDoneExecutionStarting { + received_at: *start, + throttle_started_at: *start, + started_execution_at: *end, + }; + } + }, + (x, _) => panic!("called in unexpected state: {x:?}"), + } } pub(crate) fn observe_smgr_op_completion_and_start_flushing(mut self) -> SmgrOpFlushInProgress { @@ -1263,7 +1288,7 @@ impl SmgrOpTimer { .. } = inner; SmgrOpFlushInProgress { - base: flush_start, + flush_started_at: flush_start, global_micros: global_flush_in_progress_micros, per_timeline_micros: per_timeline_flush_in_progress_micros, } @@ -1274,32 +1299,42 @@ impl SmgrOpTimer { let inner = self.0.take()?; let now = Instant::now(); - let elapsed = now - inner.start; - let elapsed = match elapsed.checked_sub(inner.throttled) { - Some(elapsed) => elapsed, - None => { - use utils::rate_limit::RateLimit; - static LOGGED: Lazy>> = - Lazy::new(|| { - Mutex::new(enum_map::EnumMap::from_array(std::array::from_fn(|_| { - RateLimit::new(Duration::from_secs(10)) - }))) - }); - let mut guard = LOGGED.lock().unwrap(); - let rate_limit = &mut guard[inner.op]; - rate_limit.call(|| { - warn!(op=?inner.op, ?elapsed, ?inner.throttled, "implementation error: time spent throttled exceeds total request wall clock time"); - }); - elapsed // un-throttled time, more info than just saturating to 0 + let batch; + let execution; + let throttle; + match inner.timings { + SmgrOpTimerState::Received { received_at } => { + batch = (now - received_at).as_secs_f64(); + // TODO: use label for dropped requests. + // This is quite rare in practice, only during tenant/pageservers shutdown. + throttle = Duration::ZERO; + execution = Duration::ZERO.as_secs_f64(); } - }; + SmgrOpTimerState::ThrottleDoneExecutionStarting { + received_at, + throttle_started_at, + started_execution_at, + } => { + batch = (throttle_started_at - received_at).as_secs_f64(); + throttle = started_execution_at - throttle_started_at; + execution = (now - started_execution_at).as_secs_f64(); + } + } - let elapsed = elapsed.as_secs_f64(); + // update time spent in batching + inner.global_batch_wait_time.observe(batch); + inner.per_timeline_batch_wait_time.observe(batch); - inner.global_latency_histo.observe(elapsed); - if let Some(per_timeline_getpage_histo) = &inner.per_timeline_latency_histo { - per_timeline_getpage_histo.observe(elapsed); + // time spent in throttle metric is updated by throttle impl + let _ = throttle; + + // update metrics for execution latency + inner.global_execution_latency_histo.observe(execution); + if let Some(per_timeline_execution_latency_histo) = + &inner.per_timeline_execution_latency_histo + { + per_timeline_execution_latency_histo.observe(execution); } Some((now, inner)) @@ -1325,12 +1360,12 @@ impl SmgrOpFlushInProgress { // Last call is tracked in `now`. let mut observe_guard = scopeguard::guard( || { - let elapsed = now - self.base; + let elapsed = now - self.flush_started_at; self.global_micros .inc_by(u64::try_from(elapsed.as_micros()).unwrap()); self.per_timeline_micros .inc_by(u64::try_from(elapsed.as_micros()).unwrap()); - self.base = now; + self.flush_started_at = now; }, |mut observe| { observe(); @@ -1377,6 +1412,8 @@ pub(crate) struct SmgrQueryTimePerTimeline { per_timeline_batch_size: Histogram, global_flush_in_progress_micros: IntCounter, per_timeline_flush_in_progress_micros: IntCounter, + global_batch_wait_time: Histogram, + per_timeline_batch_wait_time: Histogram, } static SMGR_QUERY_STARTED_GLOBAL: Lazy = Lazy::new(|| { @@ -1399,12 +1436,15 @@ static SMGR_QUERY_STARTED_PER_TENANT_TIMELINE: Lazy = Lazy::new(| .expect("failed to define a metric") }); +// Alias so all histograms recording per-timeline smgr timings use the same buckets. +static SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS: &[f64] = CRITICAL_OP_BUCKETS; + static SMGR_QUERY_TIME_PER_TENANT_TIMELINE: Lazy = Lazy::new(|| { register_histogram_vec!( "pageserver_smgr_query_seconds", - "Time spent on smgr query handling, aggegated by query type and tenant/timeline.", + "Time spent _executing_ smgr query handling, excluding batch and throttle delays.", &["smgr_query_type", "tenant_id", "shard_id", "timeline_id"], - CRITICAL_OP_BUCKETS.into(), + SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(), ) .expect("failed to define a metric") }); @@ -1462,7 +1502,7 @@ static SMGR_QUERY_TIME_GLOBAL_BUCKETS: Lazy> = Lazy::new(|| { static SMGR_QUERY_TIME_GLOBAL: Lazy = Lazy::new(|| { register_histogram_vec!( "pageserver_smgr_query_seconds_global", - "Time spent on smgr query handling, aggregated by query type.", + "Like pageserver_smgr_query_seconds, but aggregated to instance level.", &["smgr_query_type"], SMGR_QUERY_TIME_GLOBAL_BUCKETS.clone(), ) @@ -1559,6 +1599,25 @@ static PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL: Lazy = Lazy .expect("failed to define a metric") }); +static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME: Lazy = Lazy::new(|| { + register_histogram_vec!( + "pageserver_page_service_pagestream_batch_wait_time_seconds", + "Time a request spent waiting in its batch until the batch moved to throttle&execution.", + &["tenant_id", "shard_id", "timeline_id"], + SMGR_QUERY_TIME_PER_TENANT_TIMELINE_BUCKETS.into(), + ) + .expect("failed to define a metric") +}); + +static PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL: Lazy = Lazy::new(|| { + register_histogram!( + "pageserver_page_service_pagestream_batch_wait_time_seconds_global", + "Like pageserver_page_service_pagestream_batch_wait_time_seconds, but aggregated to instance level.", + SMGR_QUERY_TIME_GLOBAL_BUCKETS.to_vec(), + ) + .expect("failed to define a metric") +}); + impl SmgrQueryTimePerTimeline { pub(crate) fn new(tenant_shard_id: &TenantShardId, timeline_id: &TimelineId) -> Self { let tenant_id = tenant_shard_id.tenant_id.to_string(); @@ -1599,6 +1658,11 @@ impl SmgrQueryTimePerTimeline { .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id]) .unwrap(); + let global_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL.clone(); + let per_timeline_batch_wait_time = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME + .get_metric_with_label_values(&[&tenant_id, &shard_slug, &timeline_id]) + .unwrap(); + let global_flush_in_progress_micros = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL.clone(); let per_timeline_flush_in_progress_micros = PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS @@ -1614,9 +1678,11 @@ impl SmgrQueryTimePerTimeline { per_timeline_batch_size, global_flush_in_progress_micros, per_timeline_flush_in_progress_micros, + global_batch_wait_time, + per_timeline_batch_wait_time, } } - pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, started_at: Instant) -> SmgrOpTimer { + pub(crate) fn start_smgr_op(&self, op: SmgrQueryType, received_at: Instant) -> SmgrOpTimer { self.global_started[op as usize].inc(); let per_timeline_latency_histo = if matches!(op, SmgrQueryType::GetPageAtLsn) { @@ -1627,15 +1693,15 @@ impl SmgrQueryTimePerTimeline { }; SmgrOpTimer(Some(SmgrOpTimerInner { - global_latency_histo: self.global_latency[op as usize].clone(), - per_timeline_latency_histo, - start: started_at, - op, - throttled: Duration::ZERO, + global_execution_latency_histo: self.global_latency[op as usize].clone(), + per_timeline_execution_latency_histo: per_timeline_latency_histo, + timings: SmgrOpTimerState::Received { received_at }, global_flush_in_progress_micros: self.global_flush_in_progress_micros.clone(), per_timeline_flush_in_progress_micros: self .per_timeline_flush_in_progress_micros .clone(), + global_batch_wait_time: self.global_batch_wait_time.clone(), + per_timeline_batch_wait_time: self.per_timeline_batch_wait_time.clone(), })) } @@ -2889,6 +2955,11 @@ impl TimelineMetrics { shard_id, timeline_id, ]); + let _ = PAGE_SERVICE_SMGR_BATCH_WAIT_TIME.remove_label_values(&[ + tenant_id, + shard_id, + timeline_id, + ]); } } @@ -2919,6 +2990,7 @@ use crate::context::{PageContentKind, RequestContext}; use crate::task_mgr::TaskKind; use crate::tenant::mgr::TenantSlot; use crate::tenant::tasks::BackgroundLoopKind; +use crate::tenant::throttle::ThrottleResult; use crate::tenant::Timeline; /// Maintain a per timeline gauge in addition to the global gauge. @@ -3773,6 +3845,7 @@ pub fn preinitialize_metrics(conf: &'static PageServerConf) { &REMOTE_ONDEMAND_DOWNLOADED_BYTES, &CIRCUIT_BREAKERS_BROKEN, &CIRCUIT_BREAKERS_UNBROKEN, + &PAGE_SERVICE_SMGR_FLUSH_INPROGRESS_MICROS_GLOBAL, ] .into_iter() .for_each(|c| { @@ -3820,6 +3893,7 @@ pub fn preinitialize_metrics(conf: &'static PageServerConf) { &WAL_REDO_BYTES_HISTOGRAM, &WAL_REDO_PROCESS_LAUNCH_DURATION_HISTOGRAM, &PAGE_SERVICE_BATCH_SIZE_GLOBAL, + &PAGE_SERVICE_SMGR_BATCH_WAIT_TIME_GLOBAL, ] .into_iter() .for_each(|h| { diff --git a/pageserver/src/page_service.rs b/pageserver/src/page_service.rs index 97d94bbe7f..d00ec11a76 100644 --- a/pageserver/src/page_service.rs +++ b/pageserver/src/page_service.rs @@ -575,7 +575,10 @@ enum BatchedFeMessage { } impl BatchedFeMessage { - async fn throttle(&mut self, cancel: &CancellationToken) -> Result<(), QueryError> { + async fn throttle_and_record_start_processing( + &mut self, + cancel: &CancellationToken, + ) -> Result<(), QueryError> { let (shard, tokens, timers) = match self { BatchedFeMessage::Exists { shard, timer, .. } | BatchedFeMessage::Nblocks { shard, timer, .. } @@ -603,7 +606,7 @@ impl BatchedFeMessage { } }; for timer in timers { - timer.deduct_throttle(&throttled); + timer.observe_throttle_done_execution_starting(&throttled); } Ok(()) } @@ -1230,7 +1233,7 @@ impl PageServerHandler { } }; - if let Err(cancelled) = msg.throttle(&self.cancel).await { + if let Err(cancelled) = msg.throttle_and_record_start_processing(&self.cancel).await { break cancelled; } @@ -1397,7 +1400,9 @@ impl PageServerHandler { return Err(e); } }; - batch.throttle(&self.cancel).await?; + batch + .throttle_and_record_start_processing(&self.cancel) + .await?; self.pagesteam_handle_batched_message(pgb_writer, batch, &cancel, &ctx) .await?; } diff --git a/pageserver/src/tenant/throttle.rs b/pageserver/src/tenant/throttle.rs index 54c0e59daa..8ab6a0e060 100644 --- a/pageserver/src/tenant/throttle.rs +++ b/pageserver/src/tenant/throttle.rs @@ -58,6 +58,11 @@ pub struct Stats { pub sum_throttled_usecs: u64, } +pub enum ThrottleResult { + NotThrottled { start: Instant }, + Throttled { start: Instant, end: Instant }, +} + impl Throttle where M: Metric, @@ -122,15 +127,15 @@ where self.inner.load().rate_limiter.steady_rps() } - pub async fn throttle(&self, key_count: usize) -> Option { + pub async fn throttle(&self, key_count: usize) -> ThrottleResult { let inner = self.inner.load_full(); // clones the `Inner` Arc - if !inner.enabled { - return None; - } - let start = std::time::Instant::now(); + if !inner.enabled { + return ThrottleResult::NotThrottled { start }; + } + self.metric.accounting_start(); self.count_accounted_start.fetch_add(1, Ordering::Relaxed); let did_throttle = inner.rate_limiter.acquire(key_count).await; @@ -145,9 +150,9 @@ where .fetch_add(wait_time.as_micros() as u64, Ordering::Relaxed); let observation = Observation { wait_time }; self.metric.observe_throttling(&observation); - Some(wait_time) + ThrottleResult::Throttled { start, end: now } } else { - None + ThrottleResult::NotThrottled { start } } } } diff --git a/test_runner/fixtures/metrics.py b/test_runner/fixtures/metrics.py index a591e088ef..c5295360c3 100644 --- a/test_runner/fixtures/metrics.py +++ b/test_runner/fixtures/metrics.py @@ -178,6 +178,7 @@ PAGESERVER_PER_TENANT_METRICS: tuple[str, ...] = ( counter("pageserver_timeline_wal_records_received"), counter("pageserver_page_service_pagestream_flush_in_progress_micros"), *histogram("pageserver_page_service_batch_size"), + *histogram("pageserver_page_service_pagestream_batch_wait_time_seconds"), *PAGESERVER_PER_TENANT_REMOTE_TIMELINE_CLIENT_METRICS, # "pageserver_directory_entries_count", -- only used if above a certain threshold # "pageserver_broken_tenants_count" -- used only for broken From a53db7385151d3bc9b18d7e70b110cee36d5d32c Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Wed, 11 Dec 2024 14:28:18 +0000 Subject: [PATCH 18/44] pageserver: don't drop multixact slrus on non zero shards (#10086) ## Problem We get slru truncation commands on non-zero shards. Compaction will drop the slru dir keys and ingest will fail when receiving such records. https://github.com/neondatabase/neon/pull/10080 fixed it for clog, but not for multixact. ## Summary of changes Only truncate multixact slrus on shard zero. I audited the rest of the ingest code and it looks fine from this pov. --- pageserver/src/walingest.rs | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/pageserver/src/walingest.rs b/pageserver/src/walingest.rs index b7712cfac7..e5b23fed51 100644 --- a/pageserver/src/walingest.rs +++ b/pageserver/src/walingest.rs @@ -1049,16 +1049,18 @@ impl WalIngest { // Delete all the segments except the last one. The last segment can still // contain, possibly partially, valid data. - while segment != endsegment { - modification - .drop_slru_segment(SlruKind::MultiXactMembers, segment as u32, ctx) - .await?; + if modification.tline.get_shard_identity().is_shard_zero() { + while segment != endsegment { + modification + .drop_slru_segment(SlruKind::MultiXactMembers, segment as u32, ctx) + .await?; - /* move to next segment, handling wraparound correctly */ - if segment == maxsegment { - segment = 0; - } else { - segment += 1; + /* move to next segment, handling wraparound correctly */ + if segment == maxsegment { + segment = 0; + } else { + segment += 1; + } } } From c79c1dd8e90a8d813c0e02d8dcbfaaae99779b8e Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Wed, 11 Dec 2024 15:03:11 +0000 Subject: [PATCH 19/44] compute_ctl: don't panic if control plane can't be reached (#10078) ## Problem If the control plane cannot be reached for some reason, compute_ctl panics ## Summary of changes panic is removed in favour of returning an error. Code is reformatted a bit for more flat control flow Resolves: #5391 --- compute_tools/src/bin/compute_ctl.rs | 79 ++++++++++++++-------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/compute_tools/src/bin/compute_ctl.rs b/compute_tools/src/bin/compute_ctl.rs index e73ccd908e..bb248734a8 100644 --- a/compute_tools/src/bin/compute_ctl.rs +++ b/compute_tools/src/bin/compute_ctl.rs @@ -246,47 +246,48 @@ fn try_spec_from_cli( let compute_id = matches.get_one::("compute-id"); let control_plane_uri = matches.get_one::("control-plane-uri"); - let spec; - let mut live_config_allowed = false; - match spec_json { - // First, try to get cluster spec from the cli argument - Some(json) => { - info!("got spec from cli argument {}", json); - spec = Some(serde_json::from_str(json)?); - } - None => { - // Second, try to read it from the file if path is provided - if let Some(sp) = spec_path { - let path = Path::new(sp); - let file = File::open(path)?; - spec = Some(serde_json::from_reader(file)?); - live_config_allowed = true; - } else if let Some(id) = compute_id { - if let Some(cp_base) = control_plane_uri { - live_config_allowed = true; - spec = match get_spec_from_control_plane(cp_base, id) { - Ok(s) => s, - Err(e) => { - error!("cannot get response from control plane: {}", e); - panic!("neither spec nor confirmation that compute is in the Empty state was received"); - } - }; - } else { - panic!("must specify both --control-plane-uri and --compute-id or none"); - } - } else { - panic!( - "compute spec should be provided by one of the following ways: \ - --spec OR --spec-path OR --control-plane-uri and --compute-id" - ); - } - } + // First, try to get cluster spec from the cli argument + if let Some(spec_json) = spec_json { + info!("got spec from cli argument {}", spec_json); + return Ok(CliSpecParams { + spec: Some(serde_json::from_str(spec_json)?), + live_config_allowed: false, + }); + } + + // Second, try to read it from the file if path is provided + if let Some(spec_path) = spec_path { + let file = File::open(Path::new(spec_path))?; + return Ok(CliSpecParams { + spec: Some(serde_json::from_reader(file)?), + live_config_allowed: true, + }); + } + + let Some(compute_id) = compute_id else { + panic!( + "compute spec should be provided by one of the following ways: \ + --spec OR --spec-path OR --control-plane-uri and --compute-id" + ); + }; + let Some(control_plane_uri) = control_plane_uri else { + panic!("must specify both --control-plane-uri and --compute-id or none"); }; - Ok(CliSpecParams { - spec, - live_config_allowed, - }) + match get_spec_from_control_plane(control_plane_uri, compute_id) { + Ok(spec) => Ok(CliSpecParams { + spec, + live_config_allowed: true, + }), + Err(e) => { + error!( + "cannot get response from control plane: {}\n\ + neither spec nor confirmation that compute is in the Empty state was received", + e + ); + Err(e) + } + } } struct CliSpecParams { From b987648e713b7842be1a41ab2eca898a9c90eefb Mon Sep 17 00:00:00 2001 From: a-masterov <72613290+a-masterov@users.noreply.github.com> Date: Wed, 11 Dec 2024 16:28:10 +0100 Subject: [PATCH 20/44] Enable LFC for all the PG versions. (#10068) ## Problem We added support for LFC for tests but are still using it only for the PG17 release. ## Summary of changes LFC is enabled for all PG versions. Errors in tests with LFC enabled now block merging as usual. We keep tests with disabled LFC for PG17 release. Tests on debug builds with LFC enabled still don't affect permission to merge. --- .github/workflows/_build-and-test-locally.yml | 2 +- .github/workflows/build_and_test.yml | 18 ++++++++++-------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index 42c32a23e3..7d47f78d6b 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -283,7 +283,7 @@ jobs: submodules: true - name: Pytest regression tests - continue-on-error: ${{ matrix.lfc_state == 'with-lfc' }} + continue-on-error: ${{ matrix.lfc_state == 'with-lfc' && inputs.build-type == 'debug' }} uses: ./.github/actions/run-python-test-set timeout-minutes: 60 with: diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 6023d1bb6f..ee22f2ff54 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -255,15 +255,17 @@ jobs: build-tools-image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm build-tag: ${{ needs.tag.outputs.build-tag }} build-type: ${{ matrix.build-type }} - # Run tests on all Postgres versions in release builds and only on the latest version in debug builds - # run without LFC on v17 release only + # Run tests on all Postgres versions in release builds and only on the latest version in debug builds. + # Run without LFC on v17 release and debug builds only. For all the other cases LFC is enabled. Failure on the + # debug build with LFC enabled doesn't block merging. test-cfg: | - ${{ matrix.build-type == 'release' && '[{"pg_version":"v14", "lfc_state": "without-lfc"}, - {"pg_version":"v15", "lfc_state": "without-lfc"}, - {"pg_version":"v16", "lfc_state": "without-lfc"}, - {"pg_version":"v17", "lfc_state": "without-lfc"}, - {"pg_version":"v17", "lfc_state": "with-lfc"}]' - || '[{"pg_version":"v17", "lfc_state": "without-lfc"}]' }} + ${{ matrix.build-type == 'release' && '[{"pg_version":"v14", "lfc_state": "with-lfc"}, + {"pg_version":"v15", "lfc_state": "with-lfc"}, + {"pg_version":"v16", "lfc_state": "with-lfc"}, + {"pg_version":"v17", "lfc_state": "with-lfc"}, + {"pg_version":"v17", "lfc_state": "without-lfc"}]' + || '[{"pg_version":"v17", "lfc_state": "without-lfc"}, + {"pg_version":"v17", "lfc_state": "with-lfc" }]' }} secrets: inherit # Keep `benchmarks` job outside of `build-and-test-locally` workflow to make job failures non-blocking From e4bb1ca7d82ed1d6663459fcfea3f6be78be00ea Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Wed, 11 Dec 2024 18:46:50 +0300 Subject: [PATCH 21/44] Increase neon_local http client to compute timeout in reconfigure. (#10088) Seems like 30s sometimes not enough when CI runners are overloaded, causing pull_timeline flakiness. ref https://github.com/neondatabase/neon/issues/9731#issuecomment-2535946443 --- control_plane/src/endpoint.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control_plane/src/endpoint.rs b/control_plane/src/endpoint.rs index 35067c95b6..1fdf326051 100644 --- a/control_plane/src/endpoint.rs +++ b/control_plane/src/endpoint.rs @@ -810,7 +810,7 @@ impl Endpoint { } let client = reqwest::Client::builder() - .timeout(Duration::from_secs(30)) + .timeout(Duration::from_secs(120)) .build() .unwrap(); let response = client From dee2041cd3a7bd74b732933890345cd354fd1c80 Mon Sep 17 00:00:00 2001 From: Mikhail Kot Date: Wed, 11 Dec 2024 16:23:59 +0000 Subject: [PATCH 22/44] walproposer: fix link error on debian 12 / ubuntu 22 (#10090) ## Problem Linking walproposer library (e.g. `cargo t`) produces linker errors: /home/myrrc/neon/pgxn/neon/walproposer_compat.c:169: undefined reference to `pg_snprintf' The library with these symbols (libpgcommon.a) is present ## Summary of changes Changed order of libraries resolution for linker --- libs/walproposer/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/walproposer/build.rs b/libs/walproposer/build.rs index 3f549889b8..8d5b1ade35 100644 --- a/libs/walproposer/build.rs +++ b/libs/walproposer/build.rs @@ -30,9 +30,9 @@ fn main() -> anyhow::Result<()> { let pgxn_neon = std::fs::canonicalize(pgxn_neon)?; let pgxn_neon = pgxn_neon.to_str().ok_or(anyhow!("Bad non-UTF path"))?; + println!("cargo:rustc-link-lib=static=walproposer"); println!("cargo:rustc-link-lib=static=pgport"); println!("cargo:rustc-link-lib=static=pgcommon"); - println!("cargo:rustc-link-lib=static=walproposer"); println!("cargo:rustc-link-search={walproposer_lib_search_str}"); // Rebuild crate when libwalproposer.a changes From ef233e91ef7446cbf70898aafdc98b8335c53c59 Mon Sep 17 00:00:00 2001 From: Anastasia Lubennikova Date: Wed, 11 Dec 2024 16:43:26 +0000 Subject: [PATCH 23/44] Update compute_installed_extensions metric: (#9891) add owned_by_superuser field to filter out system extensions. While on it, also correct related code: - fix the metric setting: use set() instead of inc() in a loop. inc() is not idempotent and can lead to incorrect results if the function called multiple times. Currently it is only called at compute start, but this will change soon. - fix the return type of the installed_extensions endpoint to match the metric. Currently it is only used in the test. --- compute_tools/src/http/openapi_spec.yaml | 6 +- compute_tools/src/installed_extensions.rs | 55 ++++++++++++------- libs/compute_api/src/responses.rs | 4 +- .../regress/test_installed_extensions.py | 28 +++++----- 4 files changed, 56 insertions(+), 37 deletions(-) diff --git a/compute_tools/src/http/openapi_spec.yaml b/compute_tools/src/http/openapi_spec.yaml index 7b9a62c545..24a67cac71 100644 --- a/compute_tools/src/http/openapi_spec.yaml +++ b/compute_tools/src/http/openapi_spec.yaml @@ -537,12 +537,14 @@ components: properties: extname: type: string - versions: - type: array + version: + type: string items: type: string n_databases: type: integer + owned_by_superuser: + type: integer SetRoleGrantsRequest: type: object diff --git a/compute_tools/src/installed_extensions.rs b/compute_tools/src/installed_extensions.rs index 5f62f08858..0ab259ddf1 100644 --- a/compute_tools/src/installed_extensions.rs +++ b/compute_tools/src/installed_extensions.rs @@ -1,7 +1,6 @@ use compute_api::responses::{InstalledExtension, InstalledExtensions}; use metrics::proto::MetricFamily; use std::collections::HashMap; -use std::collections::HashSet; use anyhow::Result; use postgres::{Client, NoTls}; @@ -38,61 +37,77 @@ fn list_dbs(client: &mut Client) -> Result> { /// Connect to every database (see list_dbs above) and get the list of installed extensions. /// /// Same extension can be installed in multiple databases with different versions, -/// we only keep the highest and lowest version across all databases. +/// so we report a separate metric (number of databases where it is installed) +/// for each extension version. pub fn get_installed_extensions(mut conf: postgres::config::Config) -> Result { conf.application_name("compute_ctl:get_installed_extensions"); let mut client = conf.connect(NoTls)?; - let databases: Vec = list_dbs(&mut client)?; - let mut extensions_map: HashMap = HashMap::new(); + let mut extensions_map: HashMap<(String, String, String), InstalledExtension> = HashMap::new(); for db in databases.iter() { conf.dbname(db); let mut db_client = conf.connect(NoTls)?; - let extensions: Vec<(String, String)> = db_client + let extensions: Vec<(String, String, i32)> = db_client .query( - "SELECT extname, extversion FROM pg_catalog.pg_extension;", + "SELECT extname, extversion, extowner::integer FROM pg_catalog.pg_extension", &[], )? .iter() - .map(|row| (row.get("extname"), row.get("extversion"))) + .map(|row| { + ( + row.get("extname"), + row.get("extversion"), + row.get("extowner"), + ) + }) .collect(); - for (extname, v) in extensions.iter() { + for (extname, v, extowner) in extensions.iter() { let version = v.to_string(); - // increment the number of databases where the version of extension is installed - INSTALLED_EXTENSIONS - .with_label_values(&[extname, &version]) - .inc(); + // check if the extension is owned by superuser + // 10 is the oid of superuser + let owned_by_superuser = if *extowner == 10 { "1" } else { "0" }; extensions_map - .entry(extname.to_string()) + .entry(( + extname.to_string(), + version.clone(), + owned_by_superuser.to_string(), + )) .and_modify(|e| { - e.versions.insert(version.clone()); // count the number of databases where the extension is installed e.n_databases += 1; }) .or_insert(InstalledExtension { extname: extname.to_string(), - versions: HashSet::from([version.clone()]), + version: version.clone(), n_databases: 1, + owned_by_superuser: owned_by_superuser.to_string(), }); } } - let res = InstalledExtensions { - extensions: extensions_map.into_values().collect(), - }; + for (key, ext) in extensions_map.iter() { + let (extname, version, owned_by_superuser) = key; + let n_databases = ext.n_databases as u64; - Ok(res) + INSTALLED_EXTENSIONS + .with_label_values(&[extname, version, owned_by_superuser]) + .set(n_databases); + } + + Ok(InstalledExtensions { + extensions: extensions_map.into_values().collect(), + }) } static INSTALLED_EXTENSIONS: Lazy = Lazy::new(|| { register_uint_gauge_vec!( "compute_installed_extensions", "Number of databases where the version of extension is installed", - &["extension_name", "version"] + &["extension_name", "version", "owned_by_superuser"] ) .expect("failed to define a metric") }); diff --git a/libs/compute_api/src/responses.rs b/libs/compute_api/src/responses.rs index 79234be720..0d65f6a38d 100644 --- a/libs/compute_api/src/responses.rs +++ b/libs/compute_api/src/responses.rs @@ -1,6 +1,5 @@ //! Structs representing the JSON formats used in the compute_ctl's HTTP API. -use std::collections::HashSet; use std::fmt::Display; use chrono::{DateTime, Utc}; @@ -163,8 +162,9 @@ pub enum ControlPlaneComputeStatus { #[derive(Clone, Debug, Default, Serialize)] pub struct InstalledExtension { pub extname: String, - pub versions: HashSet, + pub version: String, pub n_databases: u32, // Number of databases using this extension + pub owned_by_superuser: String, } #[derive(Clone, Debug, Default, Serialize)] diff --git a/test_runner/regress/test_installed_extensions.py b/test_runner/regress/test_installed_extensions.py index 04ccec5875..4e51e7e10c 100644 --- a/test_runner/regress/test_installed_extensions.py +++ b/test_runner/regress/test_installed_extensions.py @@ -30,7 +30,7 @@ def test_installed_extensions(neon_simple_env: NeonEnv): info("Extensions: %s", res["extensions"]) # 'plpgsql' is a default extension that is always installed. assert any( - ext["extname"] == "plpgsql" and ext["versions"] == ["1.0"] for ext in res["extensions"] + ext["extname"] == "plpgsql" and ext["version"] == "1.0" for ext in res["extensions"] ), "The 'plpgsql' extension is missing" # check that the neon_test_utils extension is not installed @@ -63,7 +63,7 @@ def test_installed_extensions(neon_simple_env: NeonEnv): # and has the expected version assert any( ext["extname"] == "neon_test_utils" - and ext["versions"] == [neon_test_utils_version] + and ext["version"] == neon_test_utils_version and ext["n_databases"] == 1 for ext in res["extensions"] ) @@ -75,9 +75,8 @@ def test_installed_extensions(neon_simple_env: NeonEnv): # check that the neon extension is installed and has expected versions for ext in res["extensions"]: if ext["extname"] == "neon": - assert ext["n_databases"] == 2 - ext["versions"].sort() - assert ext["versions"] == ["1.1", "1.2"] + assert ext["version"] in ["1.1", "1.2"] + assert ext["n_databases"] == 1 with pg_conn.cursor() as cur: cur.execute("ALTER EXTENSION neon UPDATE TO '1.3'") @@ -90,9 +89,8 @@ def test_installed_extensions(neon_simple_env: NeonEnv): # check that the neon_test_utils extension is updated for ext in res["extensions"]: if ext["extname"] == "neon": - assert ext["n_databases"] == 2 - ext["versions"].sort() - assert ext["versions"] == ["1.2", "1.3"] + assert ext["version"] in ["1.2", "1.3"] + assert ext["n_databases"] == 1 # check that /metrics endpoint is available # ensure that we see the metric before and after restart @@ -100,13 +98,15 @@ def test_installed_extensions(neon_simple_env: NeonEnv): info("Metrics: %s", res) m = parse_metrics(res) neon_m = m.query_all( - "compute_installed_extensions", {"extension_name": "neon", "version": "1.2"} + "compute_installed_extensions", + {"extension_name": "neon", "version": "1.2", "owned_by_superuser": "1"}, ) assert len(neon_m) == 1 for sample in neon_m: - assert sample.value == 2 + assert sample.value == 1 neon_m = m.query_all( - "compute_installed_extensions", {"extension_name": "neon", "version": "1.3"} + "compute_installed_extensions", + {"extension_name": "neon", "version": "1.3", "owned_by_superuser": "1"}, ) assert len(neon_m) == 1 for sample in neon_m: @@ -138,14 +138,16 @@ def test_installed_extensions(neon_simple_env: NeonEnv): info("After restart metrics: %s", res) m = parse_metrics(res) neon_m = m.query_all( - "compute_installed_extensions", {"extension_name": "neon", "version": "1.2"} + "compute_installed_extensions", + {"extension_name": "neon", "version": "1.2", "owned_by_superuser": "1"}, ) assert len(neon_m) == 1 for sample in neon_m: assert sample.value == 1 neon_m = m.query_all( - "compute_installed_extensions", {"extension_name": "neon", "version": "1.3"} + "compute_installed_extensions", + {"extension_name": "neon", "version": "1.3", "owned_by_superuser": "1"}, ) assert len(neon_m) == 1 for sample in neon_m: From a3e80448e8af32f13bcfa3a605e7b9a22d74a01e Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Wed, 11 Dec 2024 19:16:33 +0000 Subject: [PATCH 24/44] pageserver/storcon: add patch endpoints for tenant config metrics (#10020) ## Problem Cplane and storage controller tenant config changes are not additive. Any change overrides all existing tenant configs. This would be fine if both did client side patching, but that's not the case. Once this merges, we must update cplane to use the PATCH endpoint. ## Summary of changes ### High Level Allow for patching of tenant configuration with a `PATCH /v1/tenant/config` endpoint. It takes the same data as it's PUT counterpart. For example the payload below will update `gc_period` and unset `compaction_period`. All other fields are left in their original state. ``` { "tenant_id": "1234", "gc_period": "10s", "compaction_period": null } ``` ### Low Level * PS and storcon gain `PATCH /v1/tenant/config` endpoints. PS endpoint is only used for cplane managed instances. * `storcon_cli` is updated to have separate commands for `set-tenant-config` and `patch-tenant-config` Related https://github.com/neondatabase/cloud/issues/21043 --- control_plane/src/pageserver.rs | 2 +- control_plane/storcon_cli/src/main.rs | 34 ++- libs/pageserver_api/src/models.rs | 261 +++++++++++++++++- libs/utils/src/lib.rs | 2 + libs/utils/src/try_rcu.rs | 77 ++++++ pageserver/client/src/mgmt_api.rs | 8 +- pageserver/pagebench/src/cmd/aux_files.rs | 2 +- pageserver/src/http/openapi_spec.yml | 22 +- pageserver/src/http/routes.rs | 46 ++- pageserver/src/tenant.rs | 28 +- pageserver/src/tenant/config.rs | 125 ++++++++- storage_controller/src/http.rs | 33 ++- storage_controller/src/service.rs | 80 +++++- test_runner/fixtures/pageserver/http.py | 29 +- .../regress/test_disk_usage_eviction.py | 4 +- .../regress/test_ingestion_layer_size.py | 2 +- .../regress/test_layers_from_future.py | 2 +- .../test_pageserver_crash_consistency.py | 2 +- .../regress/test_storage_controller.py | 4 +- test_runner/regress/test_tenant_conf.py | 83 +++++- .../regress/test_threshold_based_eviction.py | 2 +- .../regress/test_timeline_detach_ancestor.py | 2 +- 22 files changed, 800 insertions(+), 50 deletions(-) create mode 100644 libs/utils/src/try_rcu.rs diff --git a/control_plane/src/pageserver.rs b/control_plane/src/pageserver.rs index 1d1455b95b..9d3f018345 100644 --- a/control_plane/src/pageserver.rs +++ b/control_plane/src/pageserver.rs @@ -435,7 +435,7 @@ impl PageServerNode { ) -> anyhow::Result<()> { let config = Self::parse_config(settings)?; self.http_client - .tenant_config(&models::TenantConfigRequest { tenant_id, config }) + .set_tenant_config(&models::TenantConfigRequest { tenant_id, config }) .await?; Ok(()) diff --git a/control_plane/storcon_cli/src/main.rs b/control_plane/storcon_cli/src/main.rs index e879424532..df07216fde 100644 --- a/control_plane/storcon_cli/src/main.rs +++ b/control_plane/storcon_cli/src/main.rs @@ -9,8 +9,8 @@ use pageserver_api::{ }, models::{ EvictionPolicy, EvictionPolicyLayerAccessThreshold, LocationConfigSecondary, - ShardParameters, TenantConfig, TenantConfigRequest, TenantShardSplitRequest, - TenantShardSplitResponse, + ShardParameters, TenantConfig, TenantConfigPatchRequest, TenantConfigRequest, + TenantShardSplitRequest, TenantShardSplitResponse, }, shard::{ShardStripeSize, TenantShardId}, }; @@ -116,9 +116,19 @@ enum Command { #[arg(long)] tenant_shard_id: TenantShardId, }, - /// Modify the pageserver tenant configuration of a tenant: this is the configuration structure + /// Set the pageserver tenant configuration of a tenant: this is the configuration structure /// that is passed through to pageservers, and does not affect storage controller behavior. - TenantConfig { + /// Any previous tenant configs are overwritten. + SetTenantConfig { + #[arg(long)] + tenant_id: TenantId, + #[arg(long)] + config: String, + }, + /// Patch the pageserver tenant configuration of a tenant. Any fields with null values in the + /// provided JSON are unset from the tenant config and all fields with non-null values are set. + /// Unspecified fields are not changed. + PatchTenantConfig { #[arg(long)] tenant_id: TenantId, #[arg(long)] @@ -549,11 +559,21 @@ async fn main() -> anyhow::Result<()> { ) .await?; } - Command::TenantConfig { tenant_id, config } => { + Command::SetTenantConfig { tenant_id, config } => { let tenant_conf = serde_json::from_str(&config)?; vps_client - .tenant_config(&TenantConfigRequest { + .set_tenant_config(&TenantConfigRequest { + tenant_id, + config: tenant_conf, + }) + .await?; + } + Command::PatchTenantConfig { tenant_id, config } => { + let tenant_conf = serde_json::from_str(&config)?; + + vps_client + .patch_tenant_config(&TenantConfigPatchRequest { tenant_id, config: tenant_conf, }) @@ -736,7 +756,7 @@ async fn main() -> anyhow::Result<()> { threshold, } => { vps_client - .tenant_config(&TenantConfigRequest { + .set_tenant_config(&TenantConfigRequest { tenant_id, config: TenantConfig { eviction_policy: Some(EvictionPolicy::LayerAccessThreshold( diff --git a/libs/pageserver_api/src/models.rs b/libs/pageserver_api/src/models.rs index 5488f7b2c2..5690b643f0 100644 --- a/libs/pageserver_api/src/models.rs +++ b/libs/pageserver_api/src/models.rs @@ -17,7 +17,7 @@ use std::{ use byteorder::{BigEndian, ReadBytesExt}; use postgres_ffi::BLCKSZ; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_with::serde_as; use utils::{ completion, @@ -325,6 +325,115 @@ impl Default for ShardParameters { } } +#[derive(Debug, Default, Clone, Eq, PartialEq)] +pub enum FieldPatch { + Upsert(T), + Remove, + #[default] + Noop, +} + +impl FieldPatch { + fn is_noop(&self) -> bool { + matches!(self, FieldPatch::Noop) + } + + pub fn apply(self, target: &mut Option) { + match self { + Self::Upsert(v) => *target = Some(v), + Self::Remove => *target = None, + Self::Noop => {} + } + } + + pub fn map Result>(self, map: F) -> Result, E> { + match self { + Self::Upsert(v) => Ok(FieldPatch::::Upsert(map(v)?)), + Self::Remove => Ok(FieldPatch::::Remove), + Self::Noop => Ok(FieldPatch::::Noop), + } + } +} + +impl<'de, T: Deserialize<'de>> Deserialize<'de> for FieldPatch { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + Option::deserialize(deserializer).map(|opt| match opt { + None => FieldPatch::Remove, + Some(val) => FieldPatch::Upsert(val), + }) + } +} + +impl Serialize for FieldPatch { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + match self { + FieldPatch::Upsert(val) => serializer.serialize_some(val), + FieldPatch::Remove => serializer.serialize_none(), + FieldPatch::Noop => unreachable!(), + } + } +} + +#[derive(Serialize, Deserialize, Debug, Default, Clone, Eq, PartialEq)] +#[serde(default)] +pub struct TenantConfigPatch { + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub checkpoint_distance: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub checkpoint_timeout: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub compaction_target_size: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub compaction_period: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub compaction_threshold: FieldPatch, + // defer parsing compaction_algorithm, like eviction_policy + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub compaction_algorithm: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub gc_horizon: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub gc_period: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub image_creation_threshold: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub pitr_interval: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub walreceiver_connect_timeout: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub lagging_wal_timeout: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub max_lsn_wal_lag: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub eviction_policy: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub min_resident_size_override: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub evictions_low_residence_duration_metric_threshold: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub heatmap_period: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub lazy_slru_download: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub timeline_get_throttle: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub image_layer_creation_check_threshold: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub lsn_lease_length: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub lsn_lease_length_for_ts: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub timeline_offloading: FieldPatch, + #[serde(skip_serializing_if = "FieldPatch::is_noop")] + pub wal_receiver_protocol_override: FieldPatch, +} + /// An alternative representation of `pageserver::tenant::TenantConf` with /// simpler types. #[derive(Serialize, Deserialize, Debug, Default, Clone, Eq, PartialEq)] @@ -356,6 +465,107 @@ pub struct TenantConfig { pub wal_receiver_protocol_override: Option, } +impl TenantConfig { + pub fn apply_patch(self, patch: TenantConfigPatch) -> TenantConfig { + let Self { + mut checkpoint_distance, + mut checkpoint_timeout, + mut compaction_target_size, + mut compaction_period, + mut compaction_threshold, + mut compaction_algorithm, + mut gc_horizon, + mut gc_period, + mut image_creation_threshold, + mut pitr_interval, + mut walreceiver_connect_timeout, + mut lagging_wal_timeout, + mut max_lsn_wal_lag, + mut eviction_policy, + mut min_resident_size_override, + mut evictions_low_residence_duration_metric_threshold, + mut heatmap_period, + mut lazy_slru_download, + mut timeline_get_throttle, + mut image_layer_creation_check_threshold, + mut lsn_lease_length, + mut lsn_lease_length_for_ts, + mut timeline_offloading, + mut wal_receiver_protocol_override, + } = self; + + patch.checkpoint_distance.apply(&mut checkpoint_distance); + patch.checkpoint_timeout.apply(&mut checkpoint_timeout); + patch + .compaction_target_size + .apply(&mut compaction_target_size); + patch.compaction_period.apply(&mut compaction_period); + patch.compaction_threshold.apply(&mut compaction_threshold); + patch.compaction_algorithm.apply(&mut compaction_algorithm); + patch.gc_horizon.apply(&mut gc_horizon); + patch.gc_period.apply(&mut gc_period); + patch + .image_creation_threshold + .apply(&mut image_creation_threshold); + patch.pitr_interval.apply(&mut pitr_interval); + patch + .walreceiver_connect_timeout + .apply(&mut walreceiver_connect_timeout); + patch.lagging_wal_timeout.apply(&mut lagging_wal_timeout); + patch.max_lsn_wal_lag.apply(&mut max_lsn_wal_lag); + patch.eviction_policy.apply(&mut eviction_policy); + patch + .min_resident_size_override + .apply(&mut min_resident_size_override); + patch + .evictions_low_residence_duration_metric_threshold + .apply(&mut evictions_low_residence_duration_metric_threshold); + patch.heatmap_period.apply(&mut heatmap_period); + patch.lazy_slru_download.apply(&mut lazy_slru_download); + patch + .timeline_get_throttle + .apply(&mut timeline_get_throttle); + patch + .image_layer_creation_check_threshold + .apply(&mut image_layer_creation_check_threshold); + patch.lsn_lease_length.apply(&mut lsn_lease_length); + patch + .lsn_lease_length_for_ts + .apply(&mut lsn_lease_length_for_ts); + patch.timeline_offloading.apply(&mut timeline_offloading); + patch + .wal_receiver_protocol_override + .apply(&mut wal_receiver_protocol_override); + + Self { + checkpoint_distance, + checkpoint_timeout, + compaction_target_size, + compaction_period, + compaction_threshold, + compaction_algorithm, + gc_horizon, + gc_period, + image_creation_threshold, + pitr_interval, + walreceiver_connect_timeout, + lagging_wal_timeout, + max_lsn_wal_lag, + eviction_policy, + min_resident_size_override, + evictions_low_residence_duration_metric_threshold, + heatmap_period, + lazy_slru_download, + timeline_get_throttle, + image_layer_creation_check_threshold, + lsn_lease_length, + lsn_lease_length_for_ts, + timeline_offloading, + wal_receiver_protocol_override, + } + } +} + /// The policy for the aux file storage. /// /// It can be switched through `switch_aux_file_policy` tenant config. @@ -686,6 +896,14 @@ impl TenantConfigRequest { } } +#[derive(Serialize, Deserialize, Debug)] +#[serde(deny_unknown_fields)] +pub struct TenantConfigPatchRequest { + pub tenant_id: TenantId, + #[serde(flatten)] + pub config: TenantConfigPatch, // as we have a flattened field, we should reject all unknown fields in it +} + /// See [`TenantState::attachment_status`] and the OpenAPI docs for context. #[derive(Serialize, Deserialize, Clone)] #[serde(tag = "slug", content = "data", rename_all = "snake_case")] @@ -1699,4 +1917,45 @@ mod tests { ); } } + + #[test] + fn test_tenant_config_patch_request_serde() { + let patch_request = TenantConfigPatchRequest { + tenant_id: TenantId::from_str("17c6d121946a61e5ab0fe5a2fd4d8215").unwrap(), + config: TenantConfigPatch { + checkpoint_distance: FieldPatch::Upsert(42), + gc_horizon: FieldPatch::Remove, + compaction_threshold: FieldPatch::Noop, + ..TenantConfigPatch::default() + }, + }; + + let json = serde_json::to_string(&patch_request).unwrap(); + + let expected = r#"{"tenant_id":"17c6d121946a61e5ab0fe5a2fd4d8215","checkpoint_distance":42,"gc_horizon":null}"#; + assert_eq!(json, expected); + + let decoded: TenantConfigPatchRequest = serde_json::from_str(&json).unwrap(); + assert_eq!(decoded.tenant_id, patch_request.tenant_id); + assert_eq!(decoded.config, patch_request.config); + + // Now apply the patch to a config to demonstrate semantics + + let base = TenantConfig { + checkpoint_distance: Some(28), + gc_horizon: Some(100), + compaction_target_size: Some(1024), + ..Default::default() + }; + + let expected = TenantConfig { + checkpoint_distance: Some(42), + gc_horizon: None, + ..base.clone() + }; + + let patched = base.apply_patch(decoded.config); + + assert_eq!(patched, expected); + } } diff --git a/libs/utils/src/lib.rs b/libs/utils/src/lib.rs index d9b82b20da..bccd0e0488 100644 --- a/libs/utils/src/lib.rs +++ b/libs/utils/src/lib.rs @@ -94,6 +94,8 @@ pub mod toml_edit_ext; pub mod circuit_breaker; +pub mod try_rcu; + // Re-export used in macro. Avoids adding git-version as dep in target crates. #[doc(hidden)] pub use git_version; diff --git a/libs/utils/src/try_rcu.rs b/libs/utils/src/try_rcu.rs new file mode 100644 index 0000000000..6b53ab1316 --- /dev/null +++ b/libs/utils/src/try_rcu.rs @@ -0,0 +1,77 @@ +//! Try RCU extension lifted from + +pub trait ArcSwapExt { + /// [`ArcSwap::rcu`](arc_swap::ArcSwap::rcu), but with Result that short-circuits on error. + fn try_rcu(&self, f: F) -> Result + where + F: FnMut(&T) -> Result, + R: Into; +} + +impl ArcSwapExt for arc_swap::ArcSwapAny +where + T: arc_swap::RefCnt, + S: arc_swap::strategy::CaS, +{ + fn try_rcu(&self, mut f: F) -> Result + where + F: FnMut(&T) -> Result, + R: Into, + { + fn ptr_eq(a: A, b: B) -> bool + where + A: arc_swap::AsRaw, + B: arc_swap::AsRaw, + { + let a = a.as_raw(); + let b = b.as_raw(); + std::ptr::eq(a, b) + } + + let mut cur = self.load(); + loop { + let new = f(&cur)?.into(); + let prev = self.compare_and_swap(&*cur, new); + let swapped = ptr_eq(&*cur, &*prev); + if swapped { + return Ok(arc_swap::Guard::into_inner(prev)); + } else { + cur = prev; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use arc_swap::ArcSwap; + use std::sync::Arc; + + #[test] + fn test_try_rcu_success() { + let swap = ArcSwap::from(Arc::new(42)); + + let result = swap.try_rcu(|value| -> Result<_, String> { Ok(**value + 1) }); + + assert!(result.is_ok()); + assert_eq!(**swap.load(), 43); + } + + #[test] + fn test_try_rcu_error() { + let swap = ArcSwap::from(Arc::new(42)); + + let result = swap.try_rcu(|value| -> Result { + if **value == 42 { + Err("err") + } else { + Ok(**value + 1) + } + }); + + assert!(result.is_err()); + assert_eq!(result.unwrap_err(), "err"); + assert_eq!(**swap.load(), 42); + } +} diff --git a/pageserver/client/src/mgmt_api.rs b/pageserver/client/src/mgmt_api.rs index c3a1ef8140..4e9b11879d 100644 --- a/pageserver/client/src/mgmt_api.rs +++ b/pageserver/client/src/mgmt_api.rs @@ -270,12 +270,18 @@ impl Client { Ok(body) } - pub async fn tenant_config(&self, req: &TenantConfigRequest) -> Result<()> { + pub async fn set_tenant_config(&self, req: &TenantConfigRequest) -> Result<()> { let uri = format!("{}/v1/tenant/config", self.mgmt_api_endpoint); self.request(Method::PUT, &uri, req).await?; Ok(()) } + pub async fn patch_tenant_config(&self, req: &TenantConfigPatchRequest) -> Result<()> { + let uri = format!("{}/v1/tenant/config", self.mgmt_api_endpoint); + self.request(Method::PATCH, &uri, req).await?; + Ok(()) + } + pub async fn tenant_secondary_download( &self, tenant_id: TenantShardId, diff --git a/pageserver/pagebench/src/cmd/aux_files.rs b/pageserver/pagebench/src/cmd/aux_files.rs index 923a7f1f18..b869a0c6c7 100644 --- a/pageserver/pagebench/src/cmd/aux_files.rs +++ b/pageserver/pagebench/src/cmd/aux_files.rs @@ -64,7 +64,7 @@ async fn main_impl(args: Args) -> anyhow::Result<()> { println!("operating on timeline {}", timeline); mgmt_api_client - .tenant_config(&TenantConfigRequest { + .set_tenant_config(&TenantConfigRequest { tenant_id: timeline.tenant_id, config: TenantConfig::default(), }) diff --git a/pageserver/src/http/openapi_spec.yml b/pageserver/src/http/openapi_spec.yml index 7fb9247feb..ee43440534 100644 --- a/pageserver/src/http/openapi_spec.yml +++ b/pageserver/src/http/openapi_spec.yml @@ -767,7 +767,27 @@ paths: /v1/tenant/config: put: description: | - Update tenant's config. + Update tenant's config by setting it to the provided value + + Invalid fields in the tenant config will cause the request to be rejected with status 400. + requestBody: + content: + application/json: + schema: + $ref: "#/components/schemas/TenantConfigRequest" + responses: + "200": + description: OK + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/TenantInfo" + patch: + description: | + Update tenant's config additively by patching the updated fields provided. + Null values unset the field and non-null values upsert it. Invalid fields in the tenant config will cause the request to be rejected with status 400. requestBody: diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 75d25d0a6a..6e9ee976f4 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -28,6 +28,7 @@ use pageserver_api::models::LsnLease; use pageserver_api::models::LsnLeaseRequest; use pageserver_api::models::OffloadedTimelineInfo; use pageserver_api::models::ShardParameters; +use pageserver_api::models::TenantConfigPatchRequest; use pageserver_api::models::TenantDetails; use pageserver_api::models::TenantLocationConfigRequest; use pageserver_api::models::TenantLocationConfigResponse; @@ -1695,7 +1696,47 @@ async fn update_tenant_config_handler( crate::tenant::Tenant::persist_tenant_config(state.conf, &tenant_shard_id, &location_conf) .await .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?; - tenant.set_new_tenant_config(new_tenant_conf); + + let _ = tenant + .update_tenant_config(|_crnt| Ok(new_tenant_conf.clone())) + .expect("Closure returns Ok()"); + + json_response(StatusCode::OK, ()) +} + +async fn patch_tenant_config_handler( + mut request: Request, + _cancel: CancellationToken, +) -> Result, ApiError> { + let request_data: TenantConfigPatchRequest = json_request(&mut request).await?; + let tenant_id = request_data.tenant_id; + check_permission(&request, Some(tenant_id))?; + + let state = get_state(&request); + + let tenant_shard_id = TenantShardId::unsharded(tenant_id); + + let tenant = state + .tenant_manager + .get_attached_tenant_shard(tenant_shard_id)?; + tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?; + + let updated = tenant + .update_tenant_config(|crnt| crnt.apply_patch(request_data.config.clone())) + .map_err(ApiError::BadRequest)?; + + // This is a legacy API that only operates on attached tenants: the preferred + // API to use is the location_config/ endpoint, which lets the caller provide + // the full LocationConf. + let location_conf = LocationConf::attached_single( + updated, + tenant.get_generation(), + &ShardParameters::default(), + ); + + crate::tenant::Tenant::persist_tenant_config(state.conf, &tenant_shard_id, &location_conf) + .await + .map_err(|e| ApiError::InternalServerError(anyhow::anyhow!(e)))?; json_response(StatusCode::OK, ()) } @@ -3288,6 +3329,9 @@ pub fn make_router( .get("/v1/tenant/:tenant_shard_id/synthetic_size", |r| { api_handler(r, tenant_size_handler) }) + .patch("/v1/tenant/config", |r| { + api_handler(r, patch_tenant_config_handler) + }) .put("/v1/tenant/config", |r| { api_handler(r, update_tenant_config_handler) }) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 54fa95fc47..92078e4b08 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -68,6 +68,7 @@ use utils::sync::gate::Gate; use utils::sync::gate::GateGuard; use utils::timeout::timeout_cancellable; use utils::timeout::TimeoutCancellableError; +use utils::try_rcu::ArcSwapExt; use utils::zstd::create_zst_tarball; use utils::zstd::extract_zst_tarball; @@ -3921,25 +3922,28 @@ impl Tenant { } } - pub fn set_new_tenant_config(&self, new_tenant_conf: TenantConfOpt) { + pub fn update_tenant_config anyhow::Result>( + &self, + update: F, + ) -> anyhow::Result { // Use read-copy-update in order to avoid overwriting the location config // state if this races with [`Tenant::set_new_location_config`]. Note that // this race is not possible if both request types come from the storage // controller (as they should!) because an exclusive op lock is required // on the storage controller side. - self.tenant_conf.rcu(|inner| { - Arc::new(AttachedTenantConf { - tenant_conf: new_tenant_conf.clone(), - location: inner.location, - // Attached location is not changed, no need to update lsn lease deadline. - lsn_lease_deadline: inner.lsn_lease_deadline, - }) - }); + self.tenant_conf + .try_rcu(|attached_conf| -> Result<_, anyhow::Error> { + Ok(Arc::new(AttachedTenantConf { + tenant_conf: update(attached_conf.tenant_conf.clone())?, + location: attached_conf.location, + lsn_lease_deadline: attached_conf.lsn_lease_deadline, + })) + })?; - let updated = self.tenant_conf.load().clone(); + let updated = self.tenant_conf.load(); - self.tenant_conf_updated(&new_tenant_conf); + self.tenant_conf_updated(&updated.tenant_conf); // Don't hold self.timelines.lock() during the notifies. // There's no risk of deadlock right now, but there could be if we consolidate // mutexes in struct Timeline in the future. @@ -3947,6 +3951,8 @@ impl Tenant { for timeline in timelines { timeline.tenant_conf_updated(&updated); } + + Ok(updated.tenant_conf.clone()) } pub(crate) fn set_new_location_config(&self, new_conf: AttachedTenantConf) { diff --git a/pageserver/src/tenant/config.rs b/pageserver/src/tenant/config.rs index 5d3ac5a8e3..d54dded778 100644 --- a/pageserver/src/tenant/config.rs +++ b/pageserver/src/tenant/config.rs @@ -11,7 +11,7 @@ pub(crate) use pageserver_api::config::TenantConfigToml as TenantConf; use pageserver_api::models::CompactionAlgorithmSettings; use pageserver_api::models::EvictionPolicy; -use pageserver_api::models::{self, ThrottleConfig}; +use pageserver_api::models::{self, TenantConfigPatch, ThrottleConfig}; use pageserver_api::shard::{ShardCount, ShardIdentity, ShardNumber, ShardStripeSize}; use serde::de::IntoDeserializer; use serde::{Deserialize, Serialize}; @@ -427,6 +427,129 @@ impl TenantConfOpt { .or(global_conf.wal_receiver_protocol_override), } } + + pub fn apply_patch(self, patch: TenantConfigPatch) -> anyhow::Result { + let Self { + mut checkpoint_distance, + mut checkpoint_timeout, + mut compaction_target_size, + mut compaction_period, + mut compaction_threshold, + mut compaction_algorithm, + mut gc_horizon, + mut gc_period, + mut image_creation_threshold, + mut pitr_interval, + mut walreceiver_connect_timeout, + mut lagging_wal_timeout, + mut max_lsn_wal_lag, + mut eviction_policy, + mut min_resident_size_override, + mut evictions_low_residence_duration_metric_threshold, + mut heatmap_period, + mut lazy_slru_download, + mut timeline_get_throttle, + mut image_layer_creation_check_threshold, + mut lsn_lease_length, + mut lsn_lease_length_for_ts, + mut timeline_offloading, + mut wal_receiver_protocol_override, + } = self; + + patch.checkpoint_distance.apply(&mut checkpoint_distance); + patch + .checkpoint_timeout + .map(|v| humantime::parse_duration(&v))? + .apply(&mut checkpoint_timeout); + patch + .compaction_target_size + .apply(&mut compaction_target_size); + patch + .compaction_period + .map(|v| humantime::parse_duration(&v))? + .apply(&mut compaction_period); + patch.compaction_threshold.apply(&mut compaction_threshold); + patch.compaction_algorithm.apply(&mut compaction_algorithm); + patch.gc_horizon.apply(&mut gc_horizon); + patch + .gc_period + .map(|v| humantime::parse_duration(&v))? + .apply(&mut gc_period); + patch + .image_creation_threshold + .apply(&mut image_creation_threshold); + patch + .pitr_interval + .map(|v| humantime::parse_duration(&v))? + .apply(&mut pitr_interval); + patch + .walreceiver_connect_timeout + .map(|v| humantime::parse_duration(&v))? + .apply(&mut walreceiver_connect_timeout); + patch + .lagging_wal_timeout + .map(|v| humantime::parse_duration(&v))? + .apply(&mut lagging_wal_timeout); + patch.max_lsn_wal_lag.apply(&mut max_lsn_wal_lag); + patch.eviction_policy.apply(&mut eviction_policy); + patch + .min_resident_size_override + .apply(&mut min_resident_size_override); + patch + .evictions_low_residence_duration_metric_threshold + .map(|v| humantime::parse_duration(&v))? + .apply(&mut evictions_low_residence_duration_metric_threshold); + patch + .heatmap_period + .map(|v| humantime::parse_duration(&v))? + .apply(&mut heatmap_period); + patch.lazy_slru_download.apply(&mut lazy_slru_download); + patch + .timeline_get_throttle + .apply(&mut timeline_get_throttle); + patch + .image_layer_creation_check_threshold + .apply(&mut image_layer_creation_check_threshold); + patch + .lsn_lease_length + .map(|v| humantime::parse_duration(&v))? + .apply(&mut lsn_lease_length); + patch + .lsn_lease_length_for_ts + .map(|v| humantime::parse_duration(&v))? + .apply(&mut lsn_lease_length_for_ts); + patch.timeline_offloading.apply(&mut timeline_offloading); + patch + .wal_receiver_protocol_override + .apply(&mut wal_receiver_protocol_override); + + Ok(Self { + checkpoint_distance, + checkpoint_timeout, + compaction_target_size, + compaction_period, + compaction_threshold, + compaction_algorithm, + gc_horizon, + gc_period, + image_creation_threshold, + pitr_interval, + walreceiver_connect_timeout, + lagging_wal_timeout, + max_lsn_wal_lag, + eviction_policy, + min_resident_size_override, + evictions_low_residence_duration_metric_threshold, + heatmap_period, + lazy_slru_download, + timeline_get_throttle, + image_layer_creation_check_threshold, + lsn_lease_length, + lsn_lease_length_for_ts, + timeline_offloading, + wal_receiver_protocol_override, + }) + } } impl TryFrom<&'_ models::TenantConfig> for TenantConfOpt { diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index 39e078ba7c..dce5380aa0 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -18,8 +18,9 @@ use pageserver_api::controller_api::{ ShardsPreferredAzsRequest, TenantCreateRequest, }; use pageserver_api::models::{ - TenantConfigRequest, TenantLocationConfigRequest, TenantShardSplitRequest, - TenantTimeTravelRequest, TimelineArchivalConfigRequest, TimelineCreateRequest, + TenantConfigPatchRequest, TenantConfigRequest, TenantLocationConfigRequest, + TenantShardSplitRequest, TenantTimeTravelRequest, TimelineArchivalConfigRequest, + TimelineCreateRequest, }; use pageserver_api::shard::TenantShardId; use pageserver_client::{mgmt_api, BlockUnblock}; @@ -208,6 +209,27 @@ async fn handle_tenant_location_config( ) } +async fn handle_tenant_config_patch( + service: Arc, + req: Request, +) -> Result, ApiError> { + check_permissions(&req, Scope::PageServerApi)?; + + let mut req = match maybe_forward(req).await { + ForwardOutcome::Forwarded(res) => { + return res; + } + ForwardOutcome::NotForwarded(req) => req, + }; + + let config_req = json_request::(&mut req).await?; + + json_response( + StatusCode::OK, + service.tenant_config_patch(config_req).await?, + ) +} + async fn handle_tenant_config_set( service: Arc, req: Request, @@ -1863,6 +1885,13 @@ pub fn make_router( .delete("/v1/tenant/:tenant_id", |r| { tenant_service_handler(r, handle_tenant_delete, RequestName("v1_tenant")) }) + .patch("/v1/tenant/config", |r| { + tenant_service_handler( + r, + handle_tenant_config_patch, + RequestName("v1_tenant_config"), + ) + }) .put("/v1/tenant/config", |r| { tenant_service_handler(r, handle_tenant_config_set, RequestName("v1_tenant_config")) }) diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 7e4ee53b4c..e82e84fe89 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -52,8 +52,8 @@ use pageserver_api::{ TenantPolicyRequest, TenantShardMigrateRequest, TenantShardMigrateResponse, }, models::{ - SecondaryProgress, TenantConfigRequest, TimelineArchivalConfigRequest, - TopTenantShardsRequest, + SecondaryProgress, TenantConfigPatchRequest, TenantConfigRequest, + TimelineArchivalConfigRequest, TopTenantShardsRequest, }, }; use reqwest::StatusCode; @@ -139,6 +139,7 @@ enum TenantOperations { Create, LocationConfig, ConfigSet, + ConfigPatch, TimeTravelRemoteStorage, Delete, UpdatePolicy, @@ -2602,6 +2603,55 @@ impl Service { Ok(result) } + pub(crate) async fn tenant_config_patch( + &self, + req: TenantConfigPatchRequest, + ) -> Result<(), ApiError> { + let _tenant_lock = trace_exclusive_lock( + &self.tenant_op_locks, + req.tenant_id, + TenantOperations::ConfigPatch, + ) + .await; + + let tenant_id = req.tenant_id; + let patch = req.config; + + let base = { + let locked = self.inner.read().unwrap(); + let shards = locked + .tenants + .range(TenantShardId::tenant_range(req.tenant_id)); + + let mut configs = shards.map(|(_sid, shard)| &shard.config).peekable(); + + let first = match configs.peek() { + Some(first) => (*first).clone(), + None => { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant {} not found", req.tenant_id).into(), + )); + } + }; + + if !configs.all_equal() { + tracing::error!("Tenant configs for {} are mismatched. ", req.tenant_id); + // This can't happen because we atomically update the database records + // of all shards to the new value in [`Self::set_tenant_config_and_reconcile`]. + return Err(ApiError::InternalServerError(anyhow::anyhow!( + "Tenant configs for {} are mismatched", + req.tenant_id + ))); + } + + first + }; + + let updated_config = base.apply_patch(patch); + self.set_tenant_config_and_reconcile(tenant_id, updated_config) + .await + } + pub(crate) async fn tenant_config_set(&self, req: TenantConfigRequest) -> Result<(), ApiError> { // We require an exclusive lock, because we are updating persistent and in-memory state let _tenant_lock = trace_exclusive_lock( @@ -2611,12 +2661,32 @@ impl Service { ) .await; - let tenant_id = req.tenant_id; - let config = req.config; + let tenant_exists = { + let locked = self.inner.read().unwrap(); + let mut r = locked + .tenants + .range(TenantShardId::tenant_range(req.tenant_id)); + r.next().is_some() + }; + if !tenant_exists { + return Err(ApiError::NotFound( + anyhow::anyhow!("Tenant {} not found", req.tenant_id).into(), + )); + } + + self.set_tenant_config_and_reconcile(req.tenant_id, req.config) + .await + } + + async fn set_tenant_config_and_reconcile( + &self, + tenant_id: TenantId, + config: TenantConfig, + ) -> Result<(), ApiError> { self.persistence .update_tenant_shard( - TenantFilter::Tenant(req.tenant_id), + TenantFilter::Tenant(tenant_id), None, Some(config.clone()), None, diff --git a/test_runner/fixtures/pageserver/http.py b/test_runner/fixtures/pageserver/http.py index 0832eac22f..eabdeb1053 100644 --- a/test_runner/fixtures/pageserver/http.py +++ b/test_runner/fixtures/pageserver/http.py @@ -488,7 +488,20 @@ class PageserverHttpClient(requests.Session, MetricsGetter): ) self.verbose_error(res) - def patch_tenant_config_client_side( + def patch_tenant_config(self, tenant_id: TenantId | TenantShardId, updates: dict[str, Any]): + """ + Only use this via storage_controller.pageserver_api(). + + See `set_tenant_config` for more information. + """ + assert "tenant_id" not in updates.keys() + res = self.patch( + f"http://localhost:{self.port}/v1/tenant/config", + json={**updates, "tenant_id": str(tenant_id)}, + ) + self.verbose_error(res) + + def update_tenant_config( self, tenant_id: TenantId, inserts: dict[str, Any] | None = None, @@ -499,13 +512,13 @@ class PageserverHttpClient(requests.Session, MetricsGetter): See `set_tenant_config` for more information. """ - current = self.tenant_config(tenant_id).tenant_specific_overrides - if inserts is not None: - current.update(inserts) - if removes is not None: - for key in removes: - del current[key] - self.set_tenant_config(tenant_id, current) + if inserts is None: + inserts = {} + if removes is None: + removes = [] + + patch = inserts | {remove: None for remove in removes} + self.patch_tenant_config(tenant_id, patch) def tenant_size(self, tenant_id: TenantId | TenantShardId) -> int: return self.tenant_size_and_modelinputs(tenant_id)[0] diff --git a/test_runner/regress/test_disk_usage_eviction.py b/test_runner/regress/test_disk_usage_eviction.py index 954db914b9..7abcdb3838 100644 --- a/test_runner/regress/test_disk_usage_eviction.py +++ b/test_runner/regress/test_disk_usage_eviction.py @@ -460,10 +460,10 @@ def test_pageserver_respects_overridden_resident_size( assert ( du_by_timeline[large_tenant] > min_resident_size ), "ensure the larger tenant will get a haircut" - env.neon_env.storage_controller.pageserver_api().patch_tenant_config_client_side( + env.neon_env.storage_controller.pageserver_api().update_tenant_config( small_tenant[0], {"min_resident_size_override": min_resident_size} ) - env.neon_env.storage_controller.pageserver_api().patch_tenant_config_client_side( + env.neon_env.storage_controller.pageserver_api().update_tenant_config( large_tenant[0], {"min_resident_size_override": min_resident_size} ) diff --git a/test_runner/regress/test_ingestion_layer_size.py b/test_runner/regress/test_ingestion_layer_size.py index 9c9bc5b519..7e99d4b2f2 100644 --- a/test_runner/regress/test_ingestion_layer_size.py +++ b/test_runner/regress/test_ingestion_layer_size.py @@ -74,7 +74,7 @@ def test_ingesting_large_batches_of_images(neon_env_builder: NeonEnvBuilder): print_layer_size_histogram(post_ingest) # since all we have are L0s, we should be getting nice L1s and images out of them now - env.storage_controller.pageserver_api().patch_tenant_config_client_side( + env.storage_controller.pageserver_api().update_tenant_config( env.initial_tenant, { "compaction_threshold": 1, diff --git a/test_runner/regress/test_layers_from_future.py b/test_runner/regress/test_layers_from_future.py index 8818b40712..5e06a1d47f 100644 --- a/test_runner/regress/test_layers_from_future.py +++ b/test_runner/regress/test_layers_from_future.py @@ -132,7 +132,7 @@ def test_issue_5878(neon_env_builder: NeonEnvBuilder, attach_mode: str): ), "sanity check for what above loop is supposed to do" # create the image layer from the future - env.storage_controller.pageserver_api().patch_tenant_config_client_side( + env.storage_controller.pageserver_api().update_tenant_config( tenant_id, {"image_creation_threshold": image_creation_threshold}, None ) assert ps_http.tenant_config(tenant_id).effective_config["image_creation_threshold"] == 1 diff --git a/test_runner/regress/test_pageserver_crash_consistency.py b/test_runner/regress/test_pageserver_crash_consistency.py index fcae7983f4..e9eee2760e 100644 --- a/test_runner/regress/test_pageserver_crash_consistency.py +++ b/test_runner/regress/test_pageserver_crash_consistency.py @@ -46,7 +46,7 @@ def test_local_only_layers_after_crash(neon_env_builder: NeonEnvBuilder, pg_bin: for sk in env.safekeepers: sk.stop() - env.storage_controller.pageserver_api().patch_tenant_config_client_side( + env.storage_controller.pageserver_api().update_tenant_config( tenant_id, {"compaction_threshold": 3} ) # hit the exit failpoint diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 9f74dcccb9..4d1784d45a 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -1768,7 +1768,7 @@ def test_storcon_cli(neon_env_builder: NeonEnvBuilder): # Modify a tenant's config storcon_cli( [ - "tenant-config", + "patch-tenant-config", "--tenant-id", str(env.initial_tenant), "--config", @@ -2403,7 +2403,7 @@ def test_storage_controller_step_down(neon_env_builder: NeonEnvBuilder): # Make a change to the tenant config to trigger a slow reconcile virtual_ps_http = PageserverHttpClient(env.storage_controller_port, lambda: True) - virtual_ps_http.patch_tenant_config_client_side(tid, {"compaction_threshold": 5}, None) + virtual_ps_http.update_tenant_config(tid, {"compaction_threshold": 5}, None) env.storage_controller.allowed_errors.extend( [ ".*Accepted configuration update but reconciliation failed.*", diff --git a/test_runner/regress/test_tenant_conf.py b/test_runner/regress/test_tenant_conf.py index f8f240cfdc..0c2d535af4 100644 --- a/test_runner/regress/test_tenant_conf.py +++ b/test_runner/regress/test_tenant_conf.py @@ -3,13 +3,14 @@ from __future__ import annotations import json from typing import TYPE_CHECKING +import pytest from fixtures.common_types import Lsn from fixtures.neon_fixtures import ( NeonEnvBuilder, ) from fixtures.pageserver.utils import assert_tenant_state, wait_for_upload from fixtures.remote_storage import LocalFsStorage, RemoteStorageKind -from fixtures.utils import wait_until +from fixtures.utils import run_only_on_default_postgres, wait_until from fixtures.workload import Workload if TYPE_CHECKING: @@ -330,3 +331,83 @@ def test_live_reconfig_get_evictions_low_residence_duration_metric_threshold( metric = get_metric() assert int(metric.labels["low_threshold_secs"]) == 24 * 60 * 60, "label resets to default" assert int(metric.value) == 0, "value resets to default" + + +@run_only_on_default_postgres("Test does not start a compute") +@pytest.mark.parametrize("ps_managed_by", ["storcon", "cplane"]) +def test_tenant_config_patch(neon_env_builder: NeonEnvBuilder, ps_managed_by: str): + """ + Test tenant config patching (i.e. additive updates) + + The flow is different for storage controller and cplane managed pageserver. + 1. Storcon managed: /v1/tenant/config request lands on storcon, which generates + location_config calls containing the update to the pageserver + 2. Cplane managed: /v1/tenant/config is called directly on the pageserver + """ + + def assert_tenant_conf_semantically_equal(lhs, rhs): + """ + Storcon returns None for fields that are not set while the pageserver does not. + Compare two tenant's config overrides semantically, by dropping the None values. + """ + lhs = {k: v for k, v in lhs.items() if v is not None} + rhs = {k: v for k, v in rhs.items() if v is not None} + + assert lhs == rhs + + env = neon_env_builder.init_start() + + if ps_managed_by == "storcon": + api = env.storage_controller.pageserver_api() + elif ps_managed_by == "cplane": + # Disallow storcon from sending location_configs to the pageserver. + # These would overwrite the manually set tenant configs. + env.storage_controller.reconcile_until_idle() + env.storage_controller.tenant_policy_update(env.initial_tenant, {"scheduling": "Stop"}) + env.storage_controller.allowed_errors.append(".*Scheduling is disabled by policy Stop.*") + + api = env.pageserver.http_client() + else: + raise Exception(f"Unexpected value of ps_managed_by param: {ps_managed_by}") + + crnt_tenant_conf = api.tenant_config(env.initial_tenant).tenant_specific_overrides + + patch: dict[str, Any | None] = { + "gc_period": "3h", + "wal_receiver_protocol_override": { + "type": "interpreted", + "args": {"format": "bincode", "compression": {"zstd": {"level": 1}}}, + }, + } + api.patch_tenant_config(env.initial_tenant, patch) + tenant_conf_after_patch = api.tenant_config(env.initial_tenant).tenant_specific_overrides + if ps_managed_by == "storcon": + # Check that the config was propagated to the PS. + overrides_on_ps = ( + env.pageserver.http_client().tenant_config(env.initial_tenant).tenant_specific_overrides + ) + assert_tenant_conf_semantically_equal(overrides_on_ps, tenant_conf_after_patch) + assert_tenant_conf_semantically_equal(tenant_conf_after_patch, crnt_tenant_conf | patch) + crnt_tenant_conf = tenant_conf_after_patch + + patch = {"gc_period": "5h", "wal_receiver_protocol_override": None} + api.patch_tenant_config(env.initial_tenant, patch) + tenant_conf_after_patch = api.tenant_config(env.initial_tenant).tenant_specific_overrides + if ps_managed_by == "storcon": + overrides_on_ps = ( + env.pageserver.http_client().tenant_config(env.initial_tenant).tenant_specific_overrides + ) + assert_tenant_conf_semantically_equal(overrides_on_ps, tenant_conf_after_patch) + assert_tenant_conf_semantically_equal(tenant_conf_after_patch, crnt_tenant_conf | patch) + crnt_tenant_conf = tenant_conf_after_patch + + put = {"pitr_interval": "1m 1s"} + api.set_tenant_config(env.initial_tenant, put) + tenant_conf_after_put = api.tenant_config(env.initial_tenant).tenant_specific_overrides + if ps_managed_by == "storcon": + overrides_on_ps = ( + env.pageserver.http_client().tenant_config(env.initial_tenant).tenant_specific_overrides + ) + assert_tenant_conf_semantically_equal(overrides_on_ps, tenant_conf_after_put) + assert_tenant_conf_semantically_equal(tenant_conf_after_put, put) + crnt_tenant_conf = tenant_conf_after_put diff --git a/test_runner/regress/test_threshold_based_eviction.py b/test_runner/regress/test_threshold_based_eviction.py index 68e9385035..bedbd84aee 100644 --- a/test_runner/regress/test_threshold_based_eviction.py +++ b/test_runner/regress/test_threshold_based_eviction.py @@ -81,7 +81,7 @@ def test_threshold_based_eviction( # create a bunch of L1s, only the least of which will need to be resident compaction_threshold = 3 # create L1 layers quickly - vps_http.patch_tenant_config_client_side( + vps_http.update_tenant_config( tenant_id, inserts={ # Disable gc and compaction to avoid on-demand downloads from their side. diff --git a/test_runner/regress/test_timeline_detach_ancestor.py b/test_runner/regress/test_timeline_detach_ancestor.py index 2c3ee38bae..5234d8278f 100644 --- a/test_runner/regress/test_timeline_detach_ancestor.py +++ b/test_runner/regress/test_timeline_detach_ancestor.py @@ -514,7 +514,7 @@ def test_compaction_induced_by_detaches_in_history( assert len(delta_layers(branch_timeline_id)) == 5 - env.storage_controller.pageserver_api().patch_tenant_config_client_side( + env.storage_controller.pageserver_api().update_tenant_config( env.initial_tenant, {"compaction_threshold": 5}, None ) From e8395807a59478d007180a0e3943b1db937c4899 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Wed, 11 Dec 2024 19:43:40 +0000 Subject: [PATCH 25/44] storcon: allow for more concurrency in drain/fill operations (#10093) ## Problem We saw the drain/fill operations not drain fast enough in ap-southeast. ## Summary of changes These are some quick changes to speed it up: * double reconcile concurrency - this is now half of the available reconcile bandwidth * reduce the waiter polling timeout - this way we can spawn new reconciliations faster --- storage_controller/src/background_node_operations.rs | 2 +- storage_controller/src/service.rs | 6 ++++-- test_runner/regress/test_storage_controller.py | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/storage_controller/src/background_node_operations.rs b/storage_controller/src/background_node_operations.rs index 6f1355eb68..226d4942e7 100644 --- a/storage_controller/src/background_node_operations.rs +++ b/storage_controller/src/background_node_operations.rs @@ -3,7 +3,7 @@ use std::{borrow::Cow, fmt::Debug, fmt::Display}; use tokio_util::sync::CancellationToken; use utils::id::NodeId; -pub(crate) const MAX_RECONCILES_PER_OPERATION: usize = 32; +pub(crate) const MAX_RECONCILES_PER_OPERATION: usize = 64; #[derive(Copy, Clone)] pub(crate) struct Drain { diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index e82e84fe89..2600500a53 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -100,6 +100,8 @@ use crate::{ use context_iterator::TenantShardContextIterator; +const WAITER_FILL_DRAIN_POLL_TIMEOUT: Duration = Duration::from_millis(500); + // For operations that should be quick, like attaching a new tenant const SHORT_RECONCILE_TIMEOUT: Duration = Duration::from_secs(5); @@ -6798,7 +6800,7 @@ impl Service { } waiters = self - .await_waiters_remainder(waiters, SHORT_RECONCILE_TIMEOUT) + .await_waiters_remainder(waiters, WAITER_FILL_DRAIN_POLL_TIMEOUT) .await; failpoint_support::sleep_millis_async!("sleepy-drain-loop", &cancel); @@ -7051,7 +7053,7 @@ impl Service { } waiters = self - .await_waiters_remainder(waiters, SHORT_RECONCILE_TIMEOUT) + .await_waiters_remainder(waiters, WAITER_FILL_DRAIN_POLL_TIMEOUT) .await; } diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 4d1784d45a..02da389809 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -2136,7 +2136,7 @@ def test_background_operation_cancellation(neon_env_builder: NeonEnvBuilder): env.start() tenant_count = 10 - shard_count_per_tenant = 8 + shard_count_per_tenant = 16 tenant_ids = [] for _ in range(0, tenant_count): From 7fa986bc923b6af3c77e888be96123f7ba068313 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Wed, 11 Dec 2024 21:10:22 +0100 Subject: [PATCH 26/44] Do tenant manifest validation with index-part (#10007) This adds some validation of invariants that we want to uphold wrt the tenant manifest and `index_part.json`: * the data the manifest has about a timeline must match with the data in `index_part.json`. It might actually change, e.g. when we do reparenting during detach ancestor, but that requires the timeline to be unoffloaded, i.e. removed from the manifest. * any timeline mentioned in index part, must, if present, be archived. If we unarchive, we first update the tenant manifest to unoffload, and only then update index part. And one needs to archive before offloading. * it is legal for timelines to be mentioned in the manifest but have no `index_part`: this is a temporary state visible during deletion of the timeline. if the pageserver crashed, an attach of the tenant will clean the state up. * it is also legal for offloaded timelines to have an `ancestor_retain_lsn` of None while having an `ancestor_timeline_id`. This is for the to-be-added flattening functionality: the plan is to set former to None if we have flattened a timeline. follow-up of #9942 part of #8088 --- storage_scrubber/src/checks.rs | 45 +++-- .../src/pageserver_physical_gc.rs | 156 ++++++++++++++---- 2 files changed, 155 insertions(+), 46 deletions(-) diff --git a/storage_scrubber/src/checks.rs b/storage_scrubber/src/checks.rs index 1b4ff01a17..f759f54d19 100644 --- a/storage_scrubber/src/checks.rs +++ b/storage_scrubber/src/checks.rs @@ -533,8 +533,9 @@ async fn list_timeline_blobs_impl( } pub(crate) struct RemoteTenantManifestInfo { - pub(crate) latest_generation: Option, - pub(crate) manifests: Vec<(Generation, ListingObject)>, + pub(crate) generation: Generation, + pub(crate) manifest: TenantManifest, + pub(crate) listing_object: ListingObject, } pub(crate) enum ListTenantManifestResult { @@ -543,7 +544,10 @@ pub(crate) enum ListTenantManifestResult { #[allow(dead_code)] unknown_keys: Vec, }, - NoErrors(RemoteTenantManifestInfo), + NoErrors { + latest_generation: Option, + manifests: Vec<(Generation, ListingObject)>, + }, } /// Lists the tenant manifests in remote storage and parses the latest one, returning a [`ListTenantManifestResult`] object. @@ -592,14 +596,6 @@ pub(crate) async fn list_tenant_manifests( unknown_keys.push(obj); } - if manifests.is_empty() { - tracing::debug!("No manifest for timeline."); - - return Ok(ListTenantManifestResult::WithErrors { - errors, - unknown_keys, - }); - } if !unknown_keys.is_empty() { errors.push(((*prefix_str).to_owned(), "unknown keys listed".to_string())); @@ -609,6 +605,15 @@ pub(crate) async fn list_tenant_manifests( }); } + if manifests.is_empty() { + tracing::debug!("No manifest for timeline."); + + return Ok(ListTenantManifestResult::NoErrors { + latest_generation: None, + manifests, + }); + } + // Find the manifest with the highest generation let (latest_generation, latest_listing_object) = manifests .iter() @@ -616,6 +621,8 @@ pub(crate) async fn list_tenant_manifests( .map(|(g, obj)| (*g, obj.clone())) .unwrap(); + manifests.retain(|(gen, _obj)| gen != &latest_generation); + let manifest_bytes = match download_object_with_retries(remote_client, &latest_listing_object.key).await { Ok(bytes) => bytes, @@ -634,13 +641,15 @@ pub(crate) async fn list_tenant_manifests( }; match TenantManifest::from_json_bytes(&manifest_bytes) { - Ok(_manifest) => { - return Ok(ListTenantManifestResult::NoErrors( - RemoteTenantManifestInfo { - latest_generation: Some(latest_generation), - manifests, - }, - )); + Ok(manifest) => { + return Ok(ListTenantManifestResult::NoErrors { + latest_generation: Some(RemoteTenantManifestInfo { + generation: latest_generation, + manifest, + listing_object: latest_listing_object, + }), + manifests, + }); } Err(parse_error) => errors.push(( latest_listing_object.key.get_path().as_str().to_owned(), diff --git a/storage_scrubber/src/pageserver_physical_gc.rs b/storage_scrubber/src/pageserver_physical_gc.rs index 20cb9c3633..d19b8a5f91 100644 --- a/storage_scrubber/src/pageserver_physical_gc.rs +++ b/storage_scrubber/src/pageserver_physical_gc.rs @@ -4,11 +4,13 @@ use std::time::Duration; use crate::checks::{ list_tenant_manifests, list_timeline_blobs, BlobDataParseResult, ListTenantManifestResult, + RemoteTenantManifestInfo, }; use crate::metadata_stream::{stream_tenant_timelines, stream_tenants}; use crate::{init_remote, BucketConfig, NodeKind, RootTarget, TenantShardTimelineId, MAX_RETRIES}; use futures_util::{StreamExt, TryStreamExt}; use pageserver::tenant::remote_timeline_client::index::LayerFileMetadata; +use pageserver::tenant::remote_timeline_client::manifest::OffloadedTimelineManifest; use pageserver::tenant::remote_timeline_client::{ parse_remote_index_path, parse_remote_tenant_manifest_path, remote_layer_path, }; @@ -527,7 +529,7 @@ async fn gc_tenant_manifests( target: &RootTarget, mode: GcMode, tenant_shard_id: TenantShardId, -) -> anyhow::Result { +) -> anyhow::Result<(GcSummary, Option)> { let mut gc_summary = GcSummary::default(); match list_tenant_manifests(remote_client, tenant_shard_id, target).await? { ListTenantManifestResult::WithErrors { @@ -537,33 +539,35 @@ async fn gc_tenant_manifests( for (_key, error) in errors { tracing::warn!(%tenant_shard_id, "list_tenant_manifests: {error}"); } + Ok((gc_summary, None)) } - ListTenantManifestResult::NoErrors(mut manifest_info) => { - let Some(latest_gen) = manifest_info.latest_generation else { - return Ok(gc_summary); + ListTenantManifestResult::NoErrors { + latest_generation, + mut manifests, + } => { + let Some(latest_generation) = latest_generation else { + return Ok((gc_summary, None)); }; - manifest_info - .manifests - .sort_by_key(|(generation, _obj)| *generation); + manifests.sort_by_key(|(generation, _obj)| *generation); // skip the two latest generations (they don't neccessarily have to be 1 apart from each other) - let candidates = manifest_info.manifests.iter().rev().skip(2); + let candidates = manifests.iter().rev().skip(2); for (_generation, key) in candidates { maybe_delete_tenant_manifest( remote_client, &min_age, - latest_gen, + latest_generation.generation, key, mode, &mut gc_summary, ) .instrument( - info_span!("maybe_delete_tenant_manifest", %tenant_shard_id, ?latest_gen, %key.key), + info_span!("maybe_delete_tenant_manifest", %tenant_shard_id, ?latest_generation.generation, %key.key), ) .await; } + Ok((gc_summary, Some(latest_generation))) } } - Ok(gc_summary) } async fn gc_timeline( @@ -573,6 +577,7 @@ async fn gc_timeline( mode: GcMode, ttid: TenantShardTimelineId, accumulator: &Arc>, + tenant_manifest_info: Arc>, ) -> anyhow::Result { let mut summary = GcSummary::default(); let data = list_timeline_blobs(remote_client, ttid, target).await?; @@ -597,6 +602,60 @@ async fn gc_timeline( } }; + if let Some(tenant_manifest_info) = &*tenant_manifest_info { + // TODO: this is O(n^2) in the number of offloaded timelines. Do a hashmap lookup instead. + let maybe_offloaded = tenant_manifest_info + .manifest + .offloaded_timelines + .iter() + .find(|offloaded_timeline| offloaded_timeline.timeline_id == ttid.timeline_id); + if let Some(offloaded) = maybe_offloaded { + let warnings = validate_index_part_with_offloaded(index_part, offloaded); + let warn = if warnings.is_empty() { + false + } else { + // Verify that the manifest hasn't changed. If it has, a potential racing change could have been cause for our troubles. + match list_tenant_manifests(remote_client, ttid.tenant_shard_id, target).await? { + ListTenantManifestResult::WithErrors { + errors, + unknown_keys: _, + } => { + for (_key, error) in errors { + tracing::warn!(%ttid, "list_tenant_manifests in gc_timeline: {error}"); + } + true + } + ListTenantManifestResult::NoErrors { + latest_generation, + manifests: _, + } => { + if let Some(new_latest_gen) = latest_generation { + let manifest_changed = ( + new_latest_gen.generation, + new_latest_gen.listing_object.last_modified, + ) == ( + tenant_manifest_info.generation, + tenant_manifest_info.listing_object.last_modified, + ); + if manifest_changed { + tracing::debug!(%ttid, "tenant manifest changed since it was loaded, suppressing {} warnings", warnings.len()); + } + manifest_changed + } else { + // The latest generation is gone. This timeline is in the progress of being deleted? + false + } + } + } + }; + if warn { + for warning in warnings { + tracing::warn!(%ttid, "{}", warning); + } + } + } + } + accumulator.lock().unwrap().update(ttid, index_part); for key in candidates { @@ -608,6 +667,35 @@ async fn gc_timeline( Ok(summary) } +fn validate_index_part_with_offloaded( + index_part: &IndexPart, + offloaded: &OffloadedTimelineManifest, +) -> Vec { + let mut warnings = Vec::new(); + if let Some(archived_at_index_part) = index_part.archived_at { + if archived_at_index_part + .signed_duration_since(offloaded.archived_at) + .num_seconds() + != 0 + { + warnings.push(format!( + "index-part archived_at={} differs from manifest archived_at={}", + archived_at_index_part, offloaded.archived_at + )); + } + } else { + warnings.push("Timeline offloaded in manifest but not archived in index-part".to_string()); + } + if index_part.metadata.ancestor_timeline() != offloaded.ancestor_timeline_id { + warnings.push(format!( + "index-part anestor={:?} differs from manifest ancestor={:?}", + index_part.metadata.ancestor_timeline(), + offloaded.ancestor_timeline_id + )); + } + warnings +} + /// Physical garbage collection: removing unused S3 objects. /// /// This is distinct from the garbage collection done inside the pageserver, which operates at a higher level @@ -650,29 +738,38 @@ pub async fn pageserver_physical_gc( let target_ref = ⌖ let remote_client_ref = &remote_client; async move { - let summaries_from_manifests = match gc_tenant_manifests( + let gc_manifest_result = gc_tenant_manifests( remote_client_ref, min_age, target_ref, mode, tenant_shard_id, ) - .await - { - Ok(gc_summary) => vec![Ok(GcSummaryOrContent::::GcSummary( - gc_summary, - ))], + .await; + let (summary_from_manifest, tenant_manifest_opt) = match gc_manifest_result { + Ok((gc_summary, tenant_manifest)) => (gc_summary, tenant_manifest), Err(e) => { tracing::warn!(%tenant_shard_id, "Error in gc_tenant_manifests: {e}"); - Vec::new() + (GcSummary::default(), None) } }; + let tenant_manifest_arc = Arc::new(tenant_manifest_opt); + let summary_from_manifest = Ok(GcSummaryOrContent::<(_, _)>::GcSummary( + summary_from_manifest, + )); stream_tenant_timelines(remote_client_ref, target_ref, tenant_shard_id) .await .map(|stream| { stream - .map_ok(GcSummaryOrContent::Content) - .chain(futures::stream::iter(summaries_from_manifests.into_iter())) + .zip(futures::stream::iter(std::iter::repeat( + tenant_manifest_arc, + ))) + .map(|(ttid_res, tenant_manifest_arc)| { + ttid_res.map(move |ttid| { + GcSummaryOrContent::Content((ttid, tenant_manifest_arc)) + }) + }) + .chain(futures::stream::iter([summary_from_manifest].into_iter())) }) } }); @@ -684,14 +781,17 @@ pub async fn pageserver_physical_gc( // Drain futures for per-shard GC, populating accumulator as a side effect { let timelines = timelines.map_ok(|summary_or_ttid| match summary_or_ttid { - GcSummaryOrContent::Content(ttid) => futures::future::Either::Left(gc_timeline( - &remote_client, - &min_age, - &target, - mode, - ttid, - &accumulator, - )), + GcSummaryOrContent::Content((ttid, tenant_manifest_arc)) => { + futures::future::Either::Left(gc_timeline( + &remote_client, + &min_age, + &target, + mode, + ttid, + &accumulator, + tenant_manifest_arc, + )) + } GcSummaryOrContent::GcSummary(gc_summary) => { futures::future::Either::Right(futures::future::ok(gc_summary)) } From 5126ebbfed85404d89b068288f02765e7141aaf7 Mon Sep 17 00:00:00 2001 From: Erik Grinaker Date: Wed, 11 Dec 2024 22:37:25 +0100 Subject: [PATCH 27/44] test_runner: bump test_check_visibility_map timeout (#10091) ## Problem `test_check_visibility_map` has been seen to time out in debug tests. ## Summary of changes Bump the timeout to 10 minutes (test reports indicate 7 minutes is sufficient). We don't want to disable the test entirely in debug builds, to exercise this with debug assertions enabled. Resolves #10069. --- test_runner/regress/test_vm_bits.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test_runner/regress/test_vm_bits.py b/test_runner/regress/test_vm_bits.py index 46e90852a6..d9e59c71f4 100644 --- a/test_runner/regress/test_vm_bits.py +++ b/test_runner/regress/test_vm_bits.py @@ -3,6 +3,7 @@ from __future__ import annotations import time from contextlib import closing +import pytest from fixtures.log_helper import log from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, PgBin, fork_at_current_lsn from fixtures.utils import query_scalar @@ -294,6 +295,7 @@ def test_vm_bit_clear_on_heap_lock_blackbox(neon_env_builder: NeonEnvBuilder): cur.execute("commit transaction") +@pytest.mark.timeout(600) # slow in debug builds def test_check_visibility_map(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin): """ Runs pgbench across a few databases on a sharded tenant, then performs a visibility map From b391b29bdc2af059261dc00ecfe13a98cf9f2e54 Mon Sep 17 00:00:00 2001 From: Tristan Partin Date: Wed, 11 Dec 2024 16:21:42 -0600 Subject: [PATCH 28/44] Improve typing in test_runner/fixtures/httpserver.py (#10103) Signed-off-by: Tristan Partin --- test_runner/fixtures/httpserver.py | 15 ++++++++------- test_runner/regress/test_ddl_forwarding.py | 4 +++- test_runner/regress/test_download_extensions.py | 6 ++++-- .../regress/test_pageserver_metric_collection.py | 6 ++++-- .../regress/test_proxy_metric_collection.py | 6 +++++- test_runner/regress/test_sharding.py | 7 +++++-- test_runner/regress/test_storage_controller.py | 8 +++++--- .../regress/test_threshold_based_eviction.py | 6 +++++- 8 files changed, 39 insertions(+), 19 deletions(-) diff --git a/test_runner/fixtures/httpserver.py b/test_runner/fixtures/httpserver.py index f653fd804c..1f46bb22b2 100644 --- a/test_runner/fixtures/httpserver.py +++ b/test_runner/fixtures/httpserver.py @@ -7,24 +7,25 @@ from pytest_httpserver import HTTPServer if TYPE_CHECKING: from collections.abc import Iterator + from ssl import SSLContext from fixtures.port_distributor import PortDistributor -# TODO: mypy fails with: -# Module "fixtures.neon_fixtures" does not explicitly export attribute "PortDistributor" [attr-defined] -# from fixtures.neon_fixtures import PortDistributor + ListenAddress = tuple[str, int] # compared to the fixtures from pytest_httpserver with same names, these are # always function scoped, so you can check and stop the server in tests. @pytest.fixture(scope="function") -def httpserver_ssl_context(): - return None +def httpserver_ssl_context() -> Iterator[SSLContext | None]: + yield None @pytest.fixture(scope="function") -def make_httpserver(httpserver_listen_address, httpserver_ssl_context) -> Iterator[HTTPServer]: +def make_httpserver( + httpserver_listen_address: ListenAddress, httpserver_ssl_context: SSLContext | None +) -> Iterator[HTTPServer]: host, port = httpserver_listen_address if not host: host = HTTPServer.DEFAULT_LISTEN_HOST @@ -47,6 +48,6 @@ def httpserver(make_httpserver: HTTPServer) -> Iterator[HTTPServer]: @pytest.fixture(scope="function") -def httpserver_listen_address(port_distributor: PortDistributor) -> tuple[str, int]: +def httpserver_listen_address(port_distributor: PortDistributor) -> ListenAddress: port = port_distributor.get_port() return ("localhost", port) diff --git a/test_runner/regress/test_ddl_forwarding.py b/test_runner/regress/test_ddl_forwarding.py index 1c5554c379..de44bbcbc8 100644 --- a/test_runner/regress/test_ddl_forwarding.py +++ b/test_runner/regress/test_ddl_forwarding.py @@ -15,6 +15,8 @@ from werkzeug.wrappers.response import Response if TYPE_CHECKING: from typing import Any, Self + from fixtures.httpserver import ListenAddress + def handle_db(dbs, roles, operation): if operation["op"] == "set": @@ -120,7 +122,7 @@ class DdlForwardingContext: @pytest.fixture(scope="function") def ddl( - httpserver: HTTPServer, vanilla_pg: VanillaPostgres, httpserver_listen_address: tuple[str, int] + httpserver: HTTPServer, vanilla_pg: VanillaPostgres, httpserver_listen_address: ListenAddress ): (host, port) = httpserver_listen_address with DdlForwardingContext(httpserver, vanilla_pg, host, port) as ddl: diff --git a/test_runner/regress/test_download_extensions.py b/test_runner/regress/test_download_extensions.py index b2e19ad713..f18f4e78bd 100644 --- a/test_runner/regress/test_download_extensions.py +++ b/test_runner/regress/test_download_extensions.py @@ -20,6 +20,8 @@ from werkzeug.wrappers.response import Response if TYPE_CHECKING: from typing import Any + from fixtures.httpserver import ListenAddress + # use neon_env_builder_local fixture to override the default neon_env_builder fixture # and use a test-specific pg_install instead of shared one @@ -47,8 +49,8 @@ def neon_env_builder_local( def test_remote_extensions( httpserver: HTTPServer, neon_env_builder_local: NeonEnvBuilder, - httpserver_listen_address, - pg_version, + httpserver_listen_address: ListenAddress, + pg_version: PgVersion, ): # setup mock http server # that expects request for anon.tar.zst diff --git a/test_runner/regress/test_pageserver_metric_collection.py b/test_runner/regress/test_pageserver_metric_collection.py index 5ec8357597..aedfdbd210 100644 --- a/test_runner/regress/test_pageserver_metric_collection.py +++ b/test_runner/regress/test_pageserver_metric_collection.py @@ -27,6 +27,8 @@ from werkzeug.wrappers.response import Response if TYPE_CHECKING: from typing import Any + from fixtures.httpserver import ListenAddress + # TODO: collect all of the env setup *AFTER* removal of RemoteStorageKind.NOOP @@ -34,7 +36,7 @@ if TYPE_CHECKING: def test_metric_collection( httpserver: HTTPServer, neon_env_builder: NeonEnvBuilder, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, ): (host, port) = httpserver_listen_address metric_collection_endpoint = f"http://{host}:{port}/billing/api/v1/usage_events" @@ -195,7 +197,7 @@ def test_metric_collection( def test_metric_collection_cleans_up_tempfile( httpserver: HTTPServer, neon_env_builder: NeonEnvBuilder, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, ): (host, port) = httpserver_listen_address metric_collection_endpoint = f"http://{host}:{port}/billing/api/v1/usage_events" diff --git a/test_runner/regress/test_proxy_metric_collection.py b/test_runner/regress/test_proxy_metric_collection.py index dd63256388..5ff4a99c51 100644 --- a/test_runner/regress/test_proxy_metric_collection.py +++ b/test_runner/regress/test_proxy_metric_collection.py @@ -2,6 +2,7 @@ from __future__ import annotations from collections.abc import Iterator from pathlib import Path +from typing import TYPE_CHECKING import pytest from fixtures.log_helper import log @@ -15,6 +16,9 @@ from pytest_httpserver import HTTPServer from werkzeug.wrappers.request import Request from werkzeug.wrappers.response import Response +if TYPE_CHECKING: + from fixtures.httpserver import ListenAddress + def proxy_metrics_handler(request: Request) -> Response: if request.json is None: @@ -38,7 +42,7 @@ def proxy_metrics_handler(request: Request) -> Response: def proxy_with_metric_collector( port_distributor: PortDistributor, neon_binpath: Path, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, test_output_dir: Path, ) -> Iterator[NeonProxy]: """Neon proxy that routes through link auth and has metric collection enabled.""" diff --git a/test_runner/regress/test_sharding.py b/test_runner/regress/test_sharding.py index 30abf91d3a..743ab0088b 100644 --- a/test_runner/regress/test_sharding.py +++ b/test_runner/regress/test_sharding.py @@ -3,7 +3,7 @@ from __future__ import annotations import os import time from collections import defaultdict -from typing import Any +from typing import TYPE_CHECKING, Any import pytest import requests @@ -27,6 +27,9 @@ from typing_extensions import override from werkzeug.wrappers.request import Request from werkzeug.wrappers.response import Response +if TYPE_CHECKING: + from fixtures.httpserver import ListenAddress + def test_sharding_smoke( neon_env_builder: NeonEnvBuilder, @@ -759,7 +762,7 @@ def test_sharding_split_smoke( def test_sharding_split_stripe_size( neon_env_builder: NeonEnvBuilder, httpserver: HTTPServer, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, initial_stripe_size: int, ): """ diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 02da389809..5f3a7b39d3 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -58,6 +58,8 @@ from werkzeug.wrappers.response import Response if TYPE_CHECKING: from typing import Any + from fixtures.httpserver import ListenAddress + def get_node_shard_counts(env: NeonEnv, tenant_ids): counts: defaultdict[int, int] = defaultdict(int) @@ -563,7 +565,7 @@ def test_storage_controller_onboard_detached(neon_env_builder: NeonEnvBuilder): def test_storage_controller_compute_hook( httpserver: HTTPServer, neon_env_builder: NeonEnvBuilder, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, ): """ Test that the sharding service calls out to the configured HTTP endpoint on attachment changes @@ -681,7 +683,7 @@ NOTIFY_FAILURE_LOGS = [ def test_storage_controller_stuck_compute_hook( httpserver: HTTPServer, neon_env_builder: NeonEnvBuilder, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, ): """ Test the migration process's behavior when the compute hook does not enable it to proceed @@ -818,7 +820,7 @@ def test_storage_controller_stuck_compute_hook( def test_storage_controller_compute_hook_revert( httpserver: HTTPServer, neon_env_builder: NeonEnvBuilder, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, ): """ 'revert' in the sense of a migration which gets reversed shortly after, as may happen during diff --git a/test_runner/regress/test_threshold_based_eviction.py b/test_runner/regress/test_threshold_based_eviction.py index bedbd84aee..c87b520366 100644 --- a/test_runner/regress/test_threshold_based_eviction.py +++ b/test_runner/regress/test_threshold_based_eviction.py @@ -2,6 +2,7 @@ from __future__ import annotations import time from dataclasses import dataclass +from typing import TYPE_CHECKING from fixtures.log_helper import log from fixtures.neon_fixtures import ( @@ -13,12 +14,15 @@ from fixtures.pageserver.http import LayerMapInfo from fixtures.remote_storage import RemoteStorageKind from pytest_httpserver import HTTPServer +if TYPE_CHECKING: + from fixtures.httpserver import ListenAddress + # NB: basic config change tests are in test_tenant_conf.py def test_threshold_based_eviction( httpserver: HTTPServer, - httpserver_listen_address, + httpserver_listen_address: ListenAddress, pg_bin: PgBin, neon_env_builder: NeonEnvBuilder, ): From 342cbea255aa601079ed01f91f902c8f2da8c3e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arpad=20M=C3=BCller?= Date: Thu, 12 Dec 2024 02:09:24 +0100 Subject: [PATCH 29/44] storcon: add safekeeper list API (#10089) This adds an API to the storage controller to list safekeepers registered to it. This PR does a `diesel print-schema > storage_controller/src/schema.rs` because of an inconsistency between up.sql and schema.rs, introduced by [this](https://github.com/neondatabase/neon/pull/8879/commits/2c142f14f7ba9bd9a178ae96723d196330eb34ba) commit, so there is some updates of `schema.rs` due to that. As a followup to this, we should maybe think about running `diesel print-schema` in CI. Part of #9981 --- storage_controller/src/http.rs | 41 +++++++++++++++---- storage_controller/src/persistence.rs | 17 ++++++++ storage_controller/src/schema.rs | 35 +++++++++------- storage_controller/src/service.rs | 6 +++ test_runner/fixtures/neon_fixtures.py | 10 +++++ .../regress/test_storage_controller.py | 5 +++ 6 files changed, 90 insertions(+), 24 deletions(-) diff --git a/storage_controller/src/http.rs b/storage_controller/src/http.rs index dce5380aa0..24fd4c341a 100644 --- a/storage_controller/src/http.rs +++ b/storage_controller/src/http.rs @@ -879,6 +879,21 @@ async fn handle_cancel_node_fill(req: Request) -> Result, A json_response(StatusCode::ACCEPTED, ()) } +async fn handle_safekeeper_list(req: Request) -> Result, ApiError> { + check_permissions(&req, Scope::Infra)?; + + let req = match maybe_forward(req).await { + ForwardOutcome::Forwarded(res) => { + return res; + } + ForwardOutcome::NotForwarded(req) => req, + }; + + let state = get_state(&req); + let safekeepers = state.service.safekeepers_list().await?; + json_response(StatusCode::OK, safekeepers) +} + async fn handle_metadata_health_update(req: Request) -> Result, ApiError> { check_permissions(&req, Scope::Scrubber)?; @@ -1203,7 +1218,7 @@ impl From for ApiError { /// /// Not used by anything except manual testing. async fn handle_get_safekeeper(req: Request) -> Result, ApiError> { - check_permissions(&req, Scope::Admin)?; + check_permissions(&req, Scope::Infra)?; let id = parse_request_param::(&req, "id")?; @@ -1221,7 +1236,7 @@ async fn handle_get_safekeeper(req: Request) -> Result, Api match res { Ok(b) => json_response(StatusCode::OK, b), Err(crate::persistence::DatabaseError::Query(diesel::result::Error::NotFound)) => { - Err(ApiError::NotFound("unknown instance_id".into())) + Err(ApiError::NotFound("unknown instance id".into())) } Err(other) => Err(other.into()), } @@ -1817,6 +1832,21 @@ pub fn make_router( RequestName("control_v1_metadata_health_list_outdated"), ) }) + // Safekeepers + .get("/control/v1/safekeeper", |r| { + named_request_span( + r, + handle_safekeeper_list, + RequestName("control_v1_safekeeper_list"), + ) + }) + .get("/control/v1/safekeeper/:id", |r| { + named_request_span(r, handle_get_safekeeper, RequestName("v1_safekeeper")) + }) + .post("/control/v1/safekeeper/:id", |r| { + // id is in the body + named_request_span(r, handle_upsert_safekeeper, RequestName("v1_safekeeper")) + }) // Tenant Shard operations .put("/control/v1/tenant/:tenant_shard_id/migrate", |r| { tenant_service_handler( @@ -1869,13 +1899,6 @@ pub fn make_router( .put("/control/v1/step_down", |r| { named_request_span(r, handle_step_down, RequestName("control_v1_step_down")) }) - .get("/control/v1/safekeeper/:id", |r| { - named_request_span(r, handle_get_safekeeper, RequestName("v1_safekeeper")) - }) - .post("/control/v1/safekeeper/:id", |r| { - // id is in the body - named_request_span(r, handle_upsert_safekeeper, RequestName("v1_safekeeper")) - }) // Tenant operations // The ^/v1/ endpoints act as a "Virtual Pageserver", enabling shard-naive clients to call into // this service to manage tenants that actually consist of many tenant shards, as if they are a single entity. diff --git a/storage_controller/src/persistence.rs b/storage_controller/src/persistence.rs index 7ca80c7dfe..e17fe78d25 100644 --- a/storage_controller/src/persistence.rs +++ b/storage_controller/src/persistence.rs @@ -104,6 +104,7 @@ pub(crate) enum DatabaseOperation { ListMetadataHealth, ListMetadataHealthUnhealthy, ListMetadataHealthOutdated, + ListSafekeepers, GetLeader, UpdateLeader, SetPreferredAzs, @@ -1011,6 +1012,22 @@ impl Persistence { Ok(()) } + /// At startup, populate the list of nodes which our shards may be placed on + pub(crate) async fn list_safekeepers(&self) -> DatabaseResult> { + let safekeepers: Vec = self + .with_measured_conn( + DatabaseOperation::ListNodes, + move |conn| -> DatabaseResult<_> { + Ok(crate::schema::safekeepers::table.load::(conn)?) + }, + ) + .await?; + + tracing::info!("list_safekeepers: loaded {} nodes", safekeepers.len()); + + Ok(safekeepers) + } + pub(crate) async fn safekeeper_get( &self, id: i64, diff --git a/storage_controller/src/schema.rs b/storage_controller/src/schema.rs index 1717a9369d..9e005ab932 100644 --- a/storage_controller/src/schema.rs +++ b/storage_controller/src/schema.rs @@ -29,6 +29,19 @@ diesel::table! { } } +diesel::table! { + safekeepers (id) { + id -> Int8, + region_id -> Text, + version -> Int8, + host -> Text, + port -> Int4, + active -> Bool, + http_port -> Int4, + availability_zone_id -> Text, + } +} + diesel::table! { tenant_shards (tenant_id, shard_number, shard_count) { tenant_id -> Varchar, @@ -45,18 +58,10 @@ diesel::table! { } } -diesel::allow_tables_to_appear_in_same_query!(controllers, metadata_health, nodes, tenant_shards,); - -diesel::table! { - safekeepers { - id -> Int8, - region_id -> Text, - version -> Int8, - instance_id -> Text, - host -> Text, - port -> Int4, - active -> Bool, - http_port -> Int4, - availability_zone_id -> Text, - } -} +diesel::allow_tables_to_appear_in_same_query!( + controllers, + metadata_health, + nodes, + safekeepers, + tenant_shards, +); diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 2600500a53..894b67fdc6 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -7185,6 +7185,12 @@ impl Service { global_observed } + pub(crate) async fn safekeepers_list( + &self, + ) -> Result, DatabaseError> { + self.persistence.list_safekeepers().await + } + pub(crate) async fn get_safekeeper( &self, id: i64, diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 8354432c0c..0ecc324030 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -2329,6 +2329,16 @@ class NeonStorageController(MetricsGetter, LogUtils): return None raise e + def get_safekeepers(self) -> list[dict[str, Any]]: + response = self.request( + "GET", + f"{self.api}/control/v1/safekeeper", + headers=self.headers(TokenScope.ADMIN), + ) + json = response.json() + assert isinstance(json, list) + return json + def set_preferred_azs(self, preferred_azs: dict[TenantShardId, str]) -> list[TenantShardId]: response = self.request( "PUT", diff --git a/test_runner/regress/test_storage_controller.py b/test_runner/regress/test_storage_controller.py index 5f3a7b39d3..ae9b596a1b 100644 --- a/test_runner/regress/test_storage_controller.py +++ b/test_runner/regress/test_storage_controller.py @@ -2955,6 +2955,8 @@ def test_safekeeper_deployment_time_update(neon_env_builder: NeonEnvBuilder): assert target.get_safekeeper(fake_id) is None + assert len(target.get_safekeepers()) == 0 + body = { "active": True, "id": fake_id, @@ -2972,6 +2974,7 @@ def test_safekeeper_deployment_time_update(neon_env_builder: NeonEnvBuilder): inserted = target.get_safekeeper(fake_id) assert inserted is not None + assert target.get_safekeepers() == [inserted] assert eq_safekeeper_records(body, inserted) # error out if pk is changed (unexpected) @@ -2983,6 +2986,7 @@ def test_safekeeper_deployment_time_update(neon_env_builder: NeonEnvBuilder): assert exc.value.status_code == 400 inserted_again = target.get_safekeeper(fake_id) + assert target.get_safekeepers() == [inserted_again] assert inserted_again is not None assert eq_safekeeper_records(inserted, inserted_again) @@ -2991,6 +2995,7 @@ def test_safekeeper_deployment_time_update(neon_env_builder: NeonEnvBuilder): body["version"] += 1 target.on_safekeeper_deploy(fake_id, body) inserted_now = target.get_safekeeper(fake_id) + assert target.get_safekeepers() == [inserted_now] assert inserted_now is not None assert eq_safekeeper_records(body, inserted_now) From 739f627b96b3b0e00479f90621559375b3382582 Mon Sep 17 00:00:00 2001 From: Misha Sakhnov Date: Thu, 12 Dec 2024 10:45:52 +0200 Subject: [PATCH 30/44] Bump vm-builder v0.35.0 -> v0.37.1 (#10015) Bump version to pick up changes introduced in the neonvm-daemon to support sys fs based CPU scaling (https://github.com/neondatabase/autoscaling/issues/1082). Previous update: https://github.com/neondatabase/neon/pull/9208 --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index ee22f2ff54..67d59c7da1 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -797,7 +797,7 @@ jobs: - pg: v17 debian: bookworm env: - VM_BUILDER_VERSION: v0.35.0 + VM_BUILDER_VERSION: v0.37.1 steps: - uses: actions/checkout@v4 From 0bd8eca9ca98ffbcaa403a47b2e6dfc2961fafa1 Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 12 Dec 2024 09:18:50 +0000 Subject: [PATCH 31/44] Storage: create release PRs On Fridays (#10017) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem To give Storage more time on preprod — create a release branch on Friday ## Summary of changes - Automatically create Storage release PR on Friday instead of Monday --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0273b977f..3c1af1d9c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: Create Release Branch on: schedule: # It should be kept in sync with if-condition in jobs - - cron: '0 6 * * MON' # Storage release + - cron: '0 6 * * FRI' # Storage release - cron: '0 6 * * THU' # Proxy release workflow_dispatch: inputs: @@ -29,7 +29,7 @@ defaults: jobs: create-storage-release-branch: - if: ${{ github.event.schedule == '0 6 * * MON' || inputs.create-storage-release-branch }} + if: ${{ github.event.schedule == '0 6 * * FRI' || inputs.create-storage-release-branch }} permissions: contents: write From ec0ce06c164dcbd301acf1bc96822b6aa542d173 Mon Sep 17 00:00:00 2001 From: Vlad Lazar Date: Thu, 12 Dec 2024 10:53:10 +0000 Subject: [PATCH 32/44] tests: default interpreted proto in tests (#10079) ## Problem We aren't using the sharded interpreted wal receiver protocol in all tests. ## Summary of changes Default to the interpreted protocol. --- test_runner/fixtures/neon_fixtures.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 0ecc324030..0b7cdec50d 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -435,7 +435,10 @@ class NeonEnvBuilder: self.pageserver_virtual_file_io_mode = pageserver_virtual_file_io_mode - self.pageserver_wal_receiver_protocol = pageserver_wal_receiver_protocol + if pageserver_wal_receiver_protocol is not None: + self.pageserver_wal_receiver_protocol = pageserver_wal_receiver_protocol + else: + self.pageserver_wal_receiver_protocol = PageserverWalReceiverProtocol.INTERPRETED assert test_name.startswith( "test_" From c9a773af37207979955ba08e40edcf2eb927745d Mon Sep 17 00:00:00 2001 From: Arseny Sher Date: Thu, 12 Dec 2024 14:57:00 +0300 Subject: [PATCH 33/44] Fix test_subscriber_synchronous_commit flakiness. (#10057) 6f7aeaa configured LFC for USE_LFC case, but omitted setting shared_buffers for non USE_LFC, causing flakiness. ref https://github.com/neondatabase/neon/issues/9989 --- .../regress/test_logical_replication.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/test_runner/regress/test_logical_replication.py b/test_runner/regress/test_logical_replication.py index db18e1758c..8908763109 100644 --- a/test_runner/regress/test_logical_replication.py +++ b/test_runner/regress/test_logical_replication.py @@ -573,17 +573,18 @@ def test_subscriber_synchronous_commit(neon_simple_env: NeonEnv, vanilla_pg: Van vanilla_pg.safe_psql("create extension neon;") env.create_branch("subscriber") - # We want all data to fit into shared_buffers because later we stop - # safekeeper and insert more; this shouldn't cause page requests as they - # will be stuck. + # We want all data to fit into shared_buffers or LFC cache because later we + # stop safekeeper and insert more; this shouldn't cause page requests as + # they will be stuck. + if USE_LFC: + config_lines = ["neon.max_file_cache_size = 32MB", "neon.file_cache_size_limit = 32MB"] + else: + config_lines = [ + "shared_buffers = 32MB", + ] sub = env.endpoints.create( "subscriber", - config_lines=[ - "neon.max_file_cache_size = 32MB", - "neon.file_cache_size_limit = 32MB", - ] - if USE_LFC - else [], + config_lines=config_lines, ) sub.start() From e502e880b542e54b13f29a45ec74d36c93361520 Mon Sep 17 00:00:00 2001 From: Conrad Ludgate Date: Thu, 12 Dec 2024 13:42:50 +0000 Subject: [PATCH 34/44] chore(proxy): remove code for old API (#10109) ## Problem Now that https://github.com/neondatabase/cloud/issues/15245 is done, we can remove the old code. ## Summary of changes Removes support for the ManagementV2 API, in favour of the ProxyV1 API. --- proxy/src/auth/backend/mod.rs | 4 - proxy/src/bin/proxy.rs | 97 +---- proxy/src/control_plane/client/mod.rs | 9 +- proxy/src/control_plane/client/neon.rs | 511 ------------------------- proxy/src/control_plane/messages.rs | 22 +- test_runner/fixtures/neon_fixtures.py | 14 +- 6 files changed, 12 insertions(+), 645 deletions(-) delete mode 100644 proxy/src/control_plane/client/neon.rs diff --git a/proxy/src/auth/backend/mod.rs b/proxy/src/auth/backend/mod.rs index 1bad7b3086..f38ecf715f 100644 --- a/proxy/src/auth/backend/mod.rs +++ b/proxy/src/auth/backend/mod.rs @@ -74,10 +74,6 @@ impl std::fmt::Display for Backend<'_, ()> { .debug_tuple("ControlPlane::ProxyV1") .field(&endpoint.url()) .finish(), - ControlPlaneClient::Neon(endpoint) => fmt - .debug_tuple("ControlPlane::Neon") - .field(&endpoint.url()) - .finish(), #[cfg(any(test, feature = "testing"))] ControlPlaneClient::PostgresMock(endpoint) => fmt .debug_tuple("ControlPlane::PostgresMock") diff --git a/proxy/src/bin/proxy.rs b/proxy/src/bin/proxy.rs index 99144acef0..97c4037009 100644 --- a/proxy/src/bin/proxy.rs +++ b/proxy/src/bin/proxy.rs @@ -43,9 +43,6 @@ static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; #[derive(Clone, Debug, ValueEnum)] enum AuthBackendType { - #[value(name("console"), alias("cplane"))] - ControlPlane, - #[value(name("cplane-v1"), alias("control-plane"))] ControlPlaneV1, @@ -488,40 +485,7 @@ async fn main() -> anyhow::Result<()> { } if let Either::Left(auth::Backend::ControlPlane(api, _)) = &auth_backend { - if let proxy::control_plane::client::ControlPlaneClient::Neon(api) = &**api { - match (redis_notifications_client, regional_redis_client.clone()) { - (None, None) => {} - (client1, client2) => { - let cache = api.caches.project_info.clone(); - if let Some(client) = client1 { - maintenance_tasks.spawn(notifications::task_main( - client, - cache.clone(), - cancel_map.clone(), - args.region.clone(), - )); - } - if let Some(client) = client2 { - maintenance_tasks.spawn(notifications::task_main( - client, - cache.clone(), - cancel_map.clone(), - args.region.clone(), - )); - } - maintenance_tasks.spawn(async move { cache.clone().gc_worker().await }); - } - } - if let Some(regional_redis_client) = regional_redis_client { - let cache = api.caches.endpoints_cache.clone(); - let con = regional_redis_client; - let span = tracing::info_span!("endpoints_cache"); - maintenance_tasks.spawn( - async move { cache.do_read(con, cancellation_token.clone()).await } - .instrument(span), - ); - } - } else if let proxy::control_plane::client::ControlPlaneClient::ProxyV1(api) = &**api { + if let proxy::control_plane::client::ControlPlaneClient::ProxyV1(api) = &**api { match (redis_notifications_client, regional_redis_client.clone()) { (None, None) => {} (client1, client2) => { @@ -757,65 +721,6 @@ fn build_auth_backend( Ok(Either::Left(config)) } - AuthBackendType::ControlPlane => { - let wake_compute_cache_config: CacheOptions = args.wake_compute_cache.parse()?; - let project_info_cache_config: ProjectInfoCacheOptions = - args.project_info_cache.parse()?; - let endpoint_cache_config: config::EndpointCacheConfig = - args.endpoint_cache_config.parse()?; - - info!("Using NodeInfoCache (wake_compute) with options={wake_compute_cache_config:?}"); - info!( - "Using AllowedIpsCache (wake_compute) with options={project_info_cache_config:?}" - ); - info!("Using EndpointCacheConfig with options={endpoint_cache_config:?}"); - let caches = Box::leak(Box::new(control_plane::caches::ApiCaches::new( - wake_compute_cache_config, - project_info_cache_config, - endpoint_cache_config, - ))); - - let config::ConcurrencyLockOptions { - shards, - limiter, - epoch, - timeout, - } = args.wake_compute_lock.parse()?; - info!(?limiter, shards, ?epoch, "Using NodeLocks (wake_compute)"); - let locks = Box::leak(Box::new(control_plane::locks::ApiLocks::new( - "wake_compute_lock", - limiter, - shards, - timeout, - epoch, - &Metrics::get().wake_compute_lock, - )?)); - tokio::spawn(locks.garbage_collect_worker()); - - let url: proxy::url::ApiUrl = args.auth_endpoint.parse()?; - - let endpoint = http::Endpoint::new(url, http::new_client()); - - let mut wake_compute_rps_limit = args.wake_compute_limit.clone(); - RateBucketInfo::validate(&mut wake_compute_rps_limit)?; - let wake_compute_endpoint_rate_limiter = - Arc::new(WakeComputeRateLimiter::new(wake_compute_rps_limit)); - - let api = control_plane::client::neon::NeonControlPlaneClient::new( - endpoint, - args.control_plane_token.clone(), - caches, - locks, - wake_compute_endpoint_rate_limiter, - ); - let api = control_plane::client::ControlPlaneClient::Neon(api); - let auth_backend = auth::Backend::ControlPlane(MaybeOwned::Owned(api), ()); - - let config = Box::leak(Box::new(auth_backend)); - - Ok(Either::Left(config)) - } - #[cfg(feature = "testing")] AuthBackendType::Postgres => { let url = args.auth_endpoint.parse()?; diff --git a/proxy/src/control_plane/client/mod.rs b/proxy/src/control_plane/client/mod.rs index 7ef5a9c9fd..d559d96bbc 100644 --- a/proxy/src/control_plane/client/mod.rs +++ b/proxy/src/control_plane/client/mod.rs @@ -1,7 +1,6 @@ pub mod cplane_proxy_v1; #[cfg(any(test, feature = "testing"))] pub mod mock; -pub mod neon; use std::hash::Hash; use std::sync::Arc; @@ -28,10 +27,8 @@ use crate::types::EndpointId; #[non_exhaustive] #[derive(Clone)] pub enum ControlPlaneClient { - /// New Proxy V1 control plane API + /// Proxy V1 control plane API ProxyV1(cplane_proxy_v1::NeonControlPlaneClient), - /// Current Management API (V2). - Neon(neon::NeonControlPlaneClient), /// Local mock control plane. #[cfg(any(test, feature = "testing"))] PostgresMock(mock::MockControlPlane), @@ -49,7 +46,6 @@ impl ControlPlaneApi for ControlPlaneClient { ) -> Result { match self { Self::ProxyV1(api) => api.get_role_secret(ctx, user_info).await, - Self::Neon(api) => api.get_role_secret(ctx, user_info).await, #[cfg(any(test, feature = "testing"))] Self::PostgresMock(api) => api.get_role_secret(ctx, user_info).await, #[cfg(test)] @@ -66,7 +62,6 @@ impl ControlPlaneApi for ControlPlaneClient { ) -> Result<(CachedAllowedIps, Option), errors::GetAuthInfoError> { match self { Self::ProxyV1(api) => api.get_allowed_ips_and_secret(ctx, user_info).await, - Self::Neon(api) => api.get_allowed_ips_and_secret(ctx, user_info).await, #[cfg(any(test, feature = "testing"))] Self::PostgresMock(api) => api.get_allowed_ips_and_secret(ctx, user_info).await, #[cfg(test)] @@ -81,7 +76,6 @@ impl ControlPlaneApi for ControlPlaneClient { ) -> Result, errors::GetEndpointJwksError> { match self { Self::ProxyV1(api) => api.get_endpoint_jwks(ctx, endpoint).await, - Self::Neon(api) => api.get_endpoint_jwks(ctx, endpoint).await, #[cfg(any(test, feature = "testing"))] Self::PostgresMock(api) => api.get_endpoint_jwks(ctx, endpoint).await, #[cfg(test)] @@ -96,7 +90,6 @@ impl ControlPlaneApi for ControlPlaneClient { ) -> Result { match self { Self::ProxyV1(api) => api.wake_compute(ctx, user_info).await, - Self::Neon(api) => api.wake_compute(ctx, user_info).await, #[cfg(any(test, feature = "testing"))] Self::PostgresMock(api) => api.wake_compute(ctx, user_info).await, #[cfg(test)] diff --git a/proxy/src/control_plane/client/neon.rs b/proxy/src/control_plane/client/neon.rs deleted file mode 100644 index bf62c0d6ab..0000000000 --- a/proxy/src/control_plane/client/neon.rs +++ /dev/null @@ -1,511 +0,0 @@ -//! Stale console backend, remove after migrating to Proxy V1 API (#15245). - -use std::sync::Arc; -use std::time::Duration; - -use ::http::header::AUTHORIZATION; -use ::http::HeaderName; -use futures::TryFutureExt; -use postgres_client::config::SslMode; -use tokio::time::Instant; -use tracing::{debug, info, info_span, warn, Instrument}; - -use super::super::messages::{ControlPlaneErrorMessage, GetRoleSecret, WakeCompute}; -use crate::auth::backend::jwt::AuthRule; -use crate::auth::backend::ComputeUserInfo; -use crate::cache::Cached; -use crate::context::RequestContext; -use crate::control_plane::caches::ApiCaches; -use crate::control_plane::errors::{ - ControlPlaneError, GetAuthInfoError, GetEndpointJwksError, WakeComputeError, -}; -use crate::control_plane::locks::ApiLocks; -use crate::control_plane::messages::{ColdStartInfo, EndpointJwksResponse, Reason}; -use crate::control_plane::{ - AuthInfo, AuthSecret, CachedAllowedIps, CachedNodeInfo, CachedRoleSecret, NodeInfo, -}; -use crate::metrics::{CacheOutcome, Metrics}; -use crate::rate_limiter::WakeComputeRateLimiter; -use crate::types::{EndpointCacheKey, EndpointId}; -use crate::{compute, http, scram}; - -const X_REQUEST_ID: HeaderName = HeaderName::from_static("x-request-id"); - -#[derive(Clone)] -pub struct NeonControlPlaneClient { - endpoint: http::Endpoint, - pub caches: &'static ApiCaches, - pub(crate) locks: &'static ApiLocks, - pub(crate) wake_compute_endpoint_rate_limiter: Arc, - // put in a shared ref so we don't copy secrets all over in memory - jwt: Arc, -} - -impl NeonControlPlaneClient { - /// Construct an API object containing the auth parameters. - pub fn new( - endpoint: http::Endpoint, - jwt: Arc, - caches: &'static ApiCaches, - locks: &'static ApiLocks, - wake_compute_endpoint_rate_limiter: Arc, - ) -> Self { - Self { - endpoint, - caches, - locks, - wake_compute_endpoint_rate_limiter, - jwt, - } - } - - pub(crate) fn url(&self) -> &str { - self.endpoint.url().as_str() - } - - async fn do_get_auth_info( - &self, - ctx: &RequestContext, - user_info: &ComputeUserInfo, - ) -> Result { - if !self - .caches - .endpoints_cache - .is_valid(ctx, &user_info.endpoint.normalize()) - { - // TODO: refactor this because it's weird - // this is a failure to authenticate but we return Ok. - info!("endpoint is not valid, skipping the request"); - return Ok(AuthInfo::default()); - } - let request_id = ctx.session_id().to_string(); - let application_name = ctx.console_application_name(); - async { - let request = self - .endpoint - .get_path("proxy_get_role_secret") - .header(X_REQUEST_ID, &request_id) - .header(AUTHORIZATION, format!("Bearer {}", &self.jwt)) - .query(&[("session_id", ctx.session_id())]) - .query(&[ - ("application_name", application_name.as_str()), - ("project", user_info.endpoint.as_str()), - ("role", user_info.user.as_str()), - ]) - .build()?; - - debug!(url = request.url().as_str(), "sending http request"); - let start = Instant::now(); - let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Cplane); - let response = self.endpoint.execute(request).await?; - drop(pause); - info!(duration = ?start.elapsed(), "received http response"); - let body = match parse_body::(response).await { - Ok(body) => body, - // Error 404 is special: it's ok not to have a secret. - // TODO(anna): retry - Err(e) => { - return if e.get_reason().is_not_found() { - // TODO: refactor this because it's weird - // this is a failure to authenticate but we return Ok. - Ok(AuthInfo::default()) - } else { - Err(e.into()) - }; - } - }; - - let secret = if body.role_secret.is_empty() { - None - } else { - let secret = scram::ServerSecret::parse(&body.role_secret) - .map(AuthSecret::Scram) - .ok_or(GetAuthInfoError::BadSecret)?; - Some(secret) - }; - let allowed_ips = body.allowed_ips.unwrap_or_default(); - Metrics::get() - .proxy - .allowed_ips_number - .observe(allowed_ips.len() as f64); - Ok(AuthInfo { - secret, - allowed_ips, - project_id: body.project_id, - }) - } - .inspect_err(|e| tracing::debug!(error = ?e)) - .instrument(info_span!("do_get_auth_info")) - .await - } - - async fn do_get_endpoint_jwks( - &self, - ctx: &RequestContext, - endpoint: EndpointId, - ) -> Result, GetEndpointJwksError> { - if !self - .caches - .endpoints_cache - .is_valid(ctx, &endpoint.normalize()) - { - return Err(GetEndpointJwksError::EndpointNotFound); - } - let request_id = ctx.session_id().to_string(); - async { - let request = self - .endpoint - .get_with_url(|url| { - url.path_segments_mut() - .push("endpoints") - .push(endpoint.as_str()) - .push("jwks"); - }) - .header(X_REQUEST_ID, &request_id) - .header(AUTHORIZATION, format!("Bearer {}", &self.jwt)) - .query(&[("session_id", ctx.session_id())]) - .build() - .map_err(GetEndpointJwksError::RequestBuild)?; - - debug!(url = request.url().as_str(), "sending http request"); - let start = Instant::now(); - let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Cplane); - let response = self - .endpoint - .execute(request) - .await - .map_err(GetEndpointJwksError::RequestExecute)?; - drop(pause); - info!(duration = ?start.elapsed(), "received http response"); - - let body = parse_body::(response).await?; - - let rules = body - .jwks - .into_iter() - .map(|jwks| AuthRule { - id: jwks.id, - jwks_url: jwks.jwks_url, - audience: jwks.jwt_audience, - role_names: jwks.role_names, - }) - .collect(); - - Ok(rules) - } - .inspect_err(|e| tracing::debug!(error = ?e)) - .instrument(info_span!("do_get_endpoint_jwks")) - .await - } - - async fn do_wake_compute( - &self, - ctx: &RequestContext, - user_info: &ComputeUserInfo, - ) -> Result { - let request_id = ctx.session_id().to_string(); - let application_name = ctx.console_application_name(); - async { - let mut request_builder = self - .endpoint - .get_path("proxy_wake_compute") - .header("X-Request-ID", &request_id) - .header("Authorization", format!("Bearer {}", &self.jwt)) - .query(&[("session_id", ctx.session_id())]) - .query(&[ - ("application_name", application_name.as_str()), - ("project", user_info.endpoint.as_str()), - ]); - - let options = user_info.options.to_deep_object(); - if !options.is_empty() { - request_builder = request_builder.query(&options); - } - - let request = request_builder.build()?; - - debug!(url = request.url().as_str(), "sending http request"); - let start = Instant::now(); - let pause = ctx.latency_timer_pause(crate::metrics::Waiting::Cplane); - let response = self.endpoint.execute(request).await?; - drop(pause); - info!(duration = ?start.elapsed(), "received http response"); - let body = parse_body::(response).await?; - - // Unfortunately, ownership won't let us use `Option::ok_or` here. - let (host, port) = match parse_host_port(&body.address) { - None => return Err(WakeComputeError::BadComputeAddress(body.address)), - Some(x) => x, - }; - - // Don't set anything but host and port! This config will be cached. - // We'll set username and such later using the startup message. - // TODO: add more type safety (in progress). - let mut config = compute::ConnCfg::new(host.to_owned(), port); - config.ssl_mode(SslMode::Disable); // TLS is not configured on compute nodes. - - let node = NodeInfo { - config, - aux: body.aux, - allow_self_signed_compute: false, - }; - - Ok(node) - } - .inspect_err(|e| tracing::debug!(error = ?e)) - .instrument(info_span!("do_wake_compute")) - .await - } -} - -impl super::ControlPlaneApi for NeonControlPlaneClient { - #[tracing::instrument(skip_all)] - async fn get_role_secret( - &self, - ctx: &RequestContext, - user_info: &ComputeUserInfo, - ) -> Result { - let normalized_ep = &user_info.endpoint.normalize(); - let user = &user_info.user; - if let Some(role_secret) = self - .caches - .project_info - .get_role_secret(normalized_ep, user) - { - return Ok(role_secret); - } - let auth_info = self.do_get_auth_info(ctx, user_info).await?; - if let Some(project_id) = auth_info.project_id { - let normalized_ep_int = normalized_ep.into(); - self.caches.project_info.insert_role_secret( - project_id, - normalized_ep_int, - user.into(), - auth_info.secret.clone(), - ); - self.caches.project_info.insert_allowed_ips( - project_id, - normalized_ep_int, - Arc::new(auth_info.allowed_ips), - ); - ctx.set_project_id(project_id); - } - // When we just got a secret, we don't need to invalidate it. - Ok(Cached::new_uncached(auth_info.secret)) - } - - async fn get_allowed_ips_and_secret( - &self, - ctx: &RequestContext, - user_info: &ComputeUserInfo, - ) -> Result<(CachedAllowedIps, Option), GetAuthInfoError> { - let normalized_ep = &user_info.endpoint.normalize(); - if let Some(allowed_ips) = self.caches.project_info.get_allowed_ips(normalized_ep) { - Metrics::get() - .proxy - .allowed_ips_cache_misses - .inc(CacheOutcome::Hit); - return Ok((allowed_ips, None)); - } - Metrics::get() - .proxy - .allowed_ips_cache_misses - .inc(CacheOutcome::Miss); - let auth_info = self.do_get_auth_info(ctx, user_info).await?; - let allowed_ips = Arc::new(auth_info.allowed_ips); - let user = &user_info.user; - if let Some(project_id) = auth_info.project_id { - let normalized_ep_int = normalized_ep.into(); - self.caches.project_info.insert_role_secret( - project_id, - normalized_ep_int, - user.into(), - auth_info.secret.clone(), - ); - self.caches.project_info.insert_allowed_ips( - project_id, - normalized_ep_int, - allowed_ips.clone(), - ); - ctx.set_project_id(project_id); - } - Ok(( - Cached::new_uncached(allowed_ips), - Some(Cached::new_uncached(auth_info.secret)), - )) - } - - #[tracing::instrument(skip_all)] - async fn get_endpoint_jwks( - &self, - ctx: &RequestContext, - endpoint: EndpointId, - ) -> Result, GetEndpointJwksError> { - self.do_get_endpoint_jwks(ctx, endpoint).await - } - - #[tracing::instrument(skip_all)] - async fn wake_compute( - &self, - ctx: &RequestContext, - user_info: &ComputeUserInfo, - ) -> Result { - let key = user_info.endpoint_cache_key(); - - macro_rules! check_cache { - () => { - if let Some(cached) = self.caches.node_info.get(&key) { - let (cached, info) = cached.take_value(); - let info = info.map_err(|c| { - info!(key = &*key, "found cached wake_compute error"); - WakeComputeError::ControlPlane(ControlPlaneError::Message(Box::new(*c))) - })?; - - debug!(key = &*key, "found cached compute node info"); - ctx.set_project(info.aux.clone()); - return Ok(cached.map(|()| info)); - } - }; - } - - // Every time we do a wakeup http request, the compute node will stay up - // for some time (highly depends on the console's scale-to-zero policy); - // The connection info remains the same during that period of time, - // which means that we might cache it to reduce the load and latency. - check_cache!(); - - let permit = self.locks.get_permit(&key).await?; - - // after getting back a permit - it's possible the cache was filled - // double check - if permit.should_check_cache() { - // TODO: if there is something in the cache, mark the permit as success. - check_cache!(); - } - - // check rate limit - if !self - .wake_compute_endpoint_rate_limiter - .check(user_info.endpoint.normalize_intern(), 1) - { - return Err(WakeComputeError::TooManyConnections); - } - - let node = permit.release_result(self.do_wake_compute(ctx, user_info).await); - match node { - Ok(node) => { - ctx.set_project(node.aux.clone()); - debug!(key = &*key, "created a cache entry for woken compute node"); - - let mut stored_node = node.clone(); - // store the cached node as 'warm_cached' - stored_node.aux.cold_start_info = ColdStartInfo::WarmCached; - - let (_, cached) = self.caches.node_info.insert_unit(key, Ok(stored_node)); - - Ok(cached.map(|()| node)) - } - Err(err) => match err { - WakeComputeError::ControlPlane(ControlPlaneError::Message(err)) => { - let Some(status) = &err.status else { - return Err(WakeComputeError::ControlPlane(ControlPlaneError::Message( - err, - ))); - }; - - let reason = status - .details - .error_info - .map_or(Reason::Unknown, |x| x.reason); - - // if we can retry this error, do not cache it. - if reason.can_retry() { - return Err(WakeComputeError::ControlPlane(ControlPlaneError::Message( - err, - ))); - } - - // at this point, we should only have quota errors. - debug!( - key = &*key, - "created a cache entry for the wake compute error" - ); - - self.caches.node_info.insert_ttl( - key, - Err(err.clone()), - Duration::from_secs(30), - ); - - Err(WakeComputeError::ControlPlane(ControlPlaneError::Message( - err, - ))) - } - err => return Err(err), - }, - } - } -} - -/// Parse http response body, taking status code into account. -async fn parse_body serde::Deserialize<'a>>( - response: http::Response, -) -> Result { - let status = response.status(); - if status.is_success() { - // We shouldn't log raw body because it may contain secrets. - info!("request succeeded, processing the body"); - return Ok(response.json().await?); - } - let s = response.bytes().await?; - // Log plaintext to be able to detect, whether there are some cases not covered by the error struct. - info!("response_error plaintext: {:?}", s); - - // Don't throw an error here because it's not as important - // as the fact that the request itself has failed. - let mut body = serde_json::from_slice(&s).unwrap_or_else(|e| { - warn!("failed to parse error body: {e}"); - ControlPlaneErrorMessage { - error: "reason unclear (malformed error message)".into(), - http_status_code: status, - status: None, - } - }); - body.http_status_code = status; - - warn!("console responded with an error ({status}): {body:?}"); - Err(ControlPlaneError::Message(Box::new(body))) -} - -fn parse_host_port(input: &str) -> Option<(&str, u16)> { - let (host, port) = input.rsplit_once(':')?; - let ipv6_brackets: &[_] = &['[', ']']; - Some((host.trim_matches(ipv6_brackets), port.parse().ok()?)) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_host_port_v4() { - let (host, port) = parse_host_port("127.0.0.1:5432").expect("failed to parse"); - assert_eq!(host, "127.0.0.1"); - assert_eq!(port, 5432); - } - - #[test] - fn test_parse_host_port_v6() { - let (host, port) = parse_host_port("[2001:db8::1]:5432").expect("failed to parse"); - assert_eq!(host, "2001:db8::1"); - assert_eq!(port, 5432); - } - - #[test] - fn test_parse_host_port_url() { - let (host, port) = parse_host_port("compute-foo-bar-1234.default.svc.cluster.local:5432") - .expect("failed to parse"); - assert_eq!(host, "compute-foo-bar-1234.default.svc.cluster.local"); - assert_eq!(port, 5432); - } -} diff --git a/proxy/src/control_plane/messages.rs b/proxy/src/control_plane/messages.rs index 2662ab85f9..d068614b24 100644 --- a/proxy/src/control_plane/messages.rs +++ b/proxy/src/control_plane/messages.rs @@ -221,15 +221,6 @@ pub(crate) struct UserFacingMessage { pub(crate) message: Box, } -/// Response which holds client's auth secret, e.g. [`crate::scram::ServerSecret`]. -/// Returned by the `/proxy_get_role_secret` API method. -#[derive(Deserialize)] -pub(crate) struct GetRoleSecret { - pub(crate) role_secret: Box, - pub(crate) allowed_ips: Option>, - pub(crate) project_id: Option, -} - /// Response which holds client's auth secret, e.g. [`crate::scram::ServerSecret`]. /// Returned by the `/get_endpoint_access_control` API method. #[derive(Deserialize)] @@ -240,13 +231,6 @@ pub(crate) struct GetEndpointAccessControl { pub(crate) allowed_vpc_endpoint_ids: Option>, } -// Manually implement debug to omit sensitive info. -impl fmt::Debug for GetRoleSecret { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("GetRoleSecret").finish_non_exhaustive() - } -} - /// Response which holds compute node's `host:port` pair. /// Returned by the `/proxy_wake_compute` API method. #[derive(Debug, Deserialize)] @@ -477,18 +461,18 @@ mod tests { let json = json!({ "role_secret": "secret", }); - serde_json::from_str::(&json.to_string())?; + serde_json::from_str::(&json.to_string())?; let json = json!({ "role_secret": "secret", "allowed_ips": ["8.8.8.8"], }); - serde_json::from_str::(&json.to_string())?; + serde_json::from_str::(&json.to_string())?; let json = json!({ "role_secret": "secret", "allowed_ips": ["8.8.8.8"], "project_id": "project", }); - serde_json::from_str::(&json.to_string())?; + serde_json::from_str::(&json.to_string())?; Ok(()) } diff --git a/test_runner/fixtures/neon_fixtures.py b/test_runner/fixtures/neon_fixtures.py index 0b7cdec50d..13ada1361e 100644 --- a/test_runner/fixtures/neon_fixtures.py +++ b/test_runner/fixtures/neon_fixtures.py @@ -3222,7 +3222,7 @@ class NeonProxy(PgProtocol): *["--allow-self-signed-compute", "true"], ] - class Console(AuthBackend): + class ProxyV1(AuthBackend): def __init__(self, endpoint: str, fixed_rate_limit: int | None = None): self.endpoint = endpoint self.fixed_rate_limit = fixed_rate_limit @@ -3230,7 +3230,7 @@ class NeonProxy(PgProtocol): def extra_args(self) -> list[str]: args = [ # Console auth backend params - *["--auth-backend", "console"], + *["--auth-backend", "cplane-v1"], *["--auth-endpoint", self.endpoint], *["--sql-over-http-pool-opt-in", "false"], ] @@ -3478,13 +3478,13 @@ class NeonProxy(PgProtocol): class NeonAuthBroker: - class ControlPlane: + class ProxyV1: def __init__(self, endpoint: str): self.endpoint = endpoint def extra_args(self) -> list[str]: args = [ - *["--auth-backend", "console"], + *["--auth-backend", "cplane-v1"], *["--auth-endpoint", self.endpoint], ] return args @@ -3496,7 +3496,7 @@ class NeonAuthBroker: http_port: int, mgmt_port: int, external_http_port: int, - auth_backend: NeonAuthBroker.ControlPlane, + auth_backend: NeonAuthBroker.ProxyV1, ): self.domain = "apiauth.localtest.me" # resolves to 127.0.0.1 self.host = "127.0.0.1" @@ -3682,7 +3682,7 @@ def static_auth_broker( local_proxy_addr = f"{http2_echoserver.host}:{http2_echoserver.port}" # return local_proxy addr on ProxyWakeCompute. - httpserver.expect_request("/cplane/proxy_wake_compute").respond_with_json( + httpserver.expect_request("/cplane/wake_compute").respond_with_json( { "address": local_proxy_addr, "aux": { @@ -3722,7 +3722,7 @@ def static_auth_broker( http_port=http_port, mgmt_port=mgmt_port, external_http_port=external_http_port, - auth_backend=NeonAuthBroker.ControlPlane(httpserver.url_for("/cplane")), + auth_backend=NeonAuthBroker.ProxyV1(httpserver.url_for("/cplane")), ) as proxy: proxy.start() yield proxy From 58d45c6e86b88b9693e8ab4dd1e8d402b51f1a4d Mon Sep 17 00:00:00 2001 From: Rahul Patil Date: Thu, 12 Dec 2024 16:13:08 +0100 Subject: [PATCH 35/44] ci(fix): Use OIDC auth to login on ECR (#10055) ## Problem CI currently uses static credentials in some places. These are less secure and hard to maintain, so we are going to deprecate them and use OIDC auth. ## Summary of changes - ci(fix): Use OIDC auth to upload artifact on s3 - ci(fix): Use OIDC auth to login on ECR --- .github/actions/download/action.yml | 11 +++ .../actions/run-python-test-set/action.yml | 5 ++ .github/actions/save-coverage-data/action.yml | 2 + .github/actions/upload/action.yml | 11 +++ .../workflows/_benchmarking_preparation.yml | 1 + .github/workflows/_build-and-test-locally.yml | 18 +++- .github/workflows/benchmarking.yml | 6 ++ .github/workflows/build_and_test.yml | 87 ++++++++++++++----- .github/workflows/cloud-regress.yml | 8 +- .github/workflows/ingest_benchmark.yml | 1 + .github/workflows/neon_extra_builds.yml | 13 ++- .github/workflows/periodic_pagebench.yml | 8 +- .github/workflows/pg-clients.yml | 10 ++- .github/workflows/pin-build-tools-image.yml | 14 +-- .github/workflows/pre-merge-checks.yml | 1 + control_plane/src/background_process.rs | 1 + test_runner/fixtures/remote_storage.py | 23 ++++- 17 files changed, 180 insertions(+), 40 deletions(-) diff --git a/.github/actions/download/action.yml b/.github/actions/download/action.yml index 01c216b1ac..d6b1fac9f7 100644 --- a/.github/actions/download/action.yml +++ b/.github/actions/download/action.yml @@ -15,10 +15,21 @@ inputs: prefix: description: "S3 prefix. Default is '${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}'" required: false + aws_oicd_role_arn: + description: "the OIDC role arn for aws auth" + required: false + default: "" runs: using: "composite" steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ inputs.aws_oicd_role_arn }} + role-duration-seconds: 3600 + - name: Download artifact id: download-artifact shell: bash -euxo pipefail {0} diff --git a/.github/actions/run-python-test-set/action.yml b/.github/actions/run-python-test-set/action.yml index 1159627302..dd5c890f5b 100644 --- a/.github/actions/run-python-test-set/action.yml +++ b/.github/actions/run-python-test-set/action.yml @@ -62,6 +62,7 @@ runs: with: name: neon-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build_type }}-artifact path: /tmp/neon + aws_oicd_role_arn: ${{ inputs.aws_oicd_role_arn }} - name: Download Neon binaries for the previous release if: inputs.build_type != 'remote' @@ -70,6 +71,7 @@ runs: name: neon-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build_type }}-artifact path: /tmp/neon-previous prefix: latest + aws_oicd_role_arn: ${{ inputs.aws_oicd_role_arn }} - name: Download compatibility snapshot if: inputs.build_type != 'remote' @@ -81,6 +83,7 @@ runs: # The lack of compatibility snapshot (for example, for the new Postgres version) # shouldn't fail the whole job. Only relevant test should fail. skip-if-does-not-exist: true + aws_oicd_role_arn: ${{ inputs.aws_oicd_role_arn }} - name: Checkout if: inputs.needs_postgres_source == 'true' @@ -218,6 +221,7 @@ runs: # The lack of compatibility snapshot shouldn't fail the job # (for example if we didn't run the test for non build-and-test workflow) skip-if-does-not-exist: true + aws_oicd_role_arn: ${{ inputs.aws_oicd_role_arn }} - name: (Re-)configure AWS credentials # necessary to upload reports to S3 after a long-running test if: ${{ !cancelled() && (inputs.aws_oicd_role_arn != '') }} @@ -232,3 +236,4 @@ runs: with: report-dir: /tmp/test_output/allure/results unique-key: ${{ inputs.build_type }}-${{ inputs.pg_version }} + aws_oicd_role_arn: ${{ inputs.aws_oicd_role_arn }} diff --git a/.github/actions/save-coverage-data/action.yml b/.github/actions/save-coverage-data/action.yml index 6fbe19a96e..9e3a7cba24 100644 --- a/.github/actions/save-coverage-data/action.yml +++ b/.github/actions/save-coverage-data/action.yml @@ -14,9 +14,11 @@ runs: name: coverage-data-artifact path: /tmp/coverage skip-if-does-not-exist: true # skip if there's no previous coverage to download + aws_oicd_role_arn: ${{ inputs.aws_oicd_role_arn }} - name: Upload coverage data uses: ./.github/actions/upload with: name: coverage-data-artifact path: /tmp/coverage + aws_oicd_role_arn: ${{ inputs.aws_oicd_role_arn }} diff --git a/.github/actions/upload/action.yml b/.github/actions/upload/action.yml index 8a4cfe2eff..6616d08899 100644 --- a/.github/actions/upload/action.yml +++ b/.github/actions/upload/action.yml @@ -14,6 +14,10 @@ inputs: prefix: description: "S3 prefix. Default is '${GITHUB_SHA}/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}'" required: false + aws_oicd_role_arn: + description: "the OIDC role arn for aws auth" + required: false + default: "" runs: using: "composite" @@ -53,6 +57,13 @@ runs: echo 'SKIPPED=false' >> $GITHUB_OUTPUT + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ inputs.aws_oicd_role_arn }} + role-duration-seconds: 3600 + - name: Upload artifact if: ${{ steps.prepare-artifact.outputs.SKIPPED == 'false' }} shell: bash -euxo pipefail {0} diff --git a/.github/workflows/_benchmarking_preparation.yml b/.github/workflows/_benchmarking_preparation.yml index 5cdc16f248..371d815fc8 100644 --- a/.github/workflows/_benchmarking_preparation.yml +++ b/.github/workflows/_benchmarking_preparation.yml @@ -70,6 +70,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} # we create a table that has one row for each database that we want to restore with the status whether the restore is done - name: Create benchmark_restore_status table if it does not exist diff --git a/.github/workflows/_build-and-test-locally.yml b/.github/workflows/_build-and-test-locally.yml index 7d47f78d6b..456399f3c3 100644 --- a/.github/workflows/_build-and-test-locally.yml +++ b/.github/workflows/_build-and-test-locally.yml @@ -31,12 +31,13 @@ defaults: env: RUST_BACKTRACE: 1 COPT: '-Werror' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} jobs: build-neon: runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} + permissions: + id-token: write # aws-actions/configure-aws-credentials + contents: read container: image: ${{ inputs.build-tools-image }} credentials: @@ -205,6 +206,13 @@ jobs: done fi + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 18000 # 5 hours + - name: Run rust tests env: NEXTEST_RETRIES: 3 @@ -256,6 +264,7 @@ jobs: with: name: neon-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-artifact path: /tmp/neon + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} # XXX: keep this after the binaries.list is formed, so the coverage can properly work later - name: Merge and upload coverage data @@ -265,6 +274,10 @@ jobs: regress-tests: # Don't run regression tests on debug arm64 builds if: inputs.build-type != 'debug' || inputs.arch != 'arm64' + permissions: + id-token: write # aws-actions/configure-aws-credentials + contents: read + statuses: write needs: [ build-neon ] runs-on: ${{ fromJson(format('["self-hosted", "{0}"]', inputs.arch == 'arm64' && 'large-arm64' || 'large')) }} container: @@ -295,6 +308,7 @@ jobs: real_s3_region: eu-central-1 rerun_failed: true pg_version: ${{ matrix.pg_version }} + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} env: TEST_RESULT_CONNSTR: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} CHECK_ONDISK_DATA_COMPATIBILITY: nonempty diff --git a/.github/workflows/benchmarking.yml b/.github/workflows/benchmarking.yml index 7621d72f64..2d37be8837 100644 --- a/.github/workflows/benchmarking.yml +++ b/.github/workflows/benchmarking.yml @@ -105,6 +105,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Create Neon Project id: create-neon-project @@ -204,6 +205,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Run Logical Replication benchmarks uses: ./.github/actions/run-python-test-set @@ -405,6 +407,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Create Neon Project if: contains(fromJson('["neonvm-captest-new", "neonvm-captest-freetier", "neonvm-azure-captest-freetier", "neonvm-azure-captest-new"]'), matrix.platform) @@ -708,6 +711,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Set up Connection String id: set-up-connstr @@ -818,6 +822,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Get Connstring Secret Name run: | @@ -926,6 +931,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Set up Connection String id: set-up-connstr diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 67d59c7da1..62f190a0c2 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -21,8 +21,6 @@ concurrency: env: RUST_BACKTRACE: 1 COPT: '-Werror' - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} # A concurrency group that we use for e2e-tests runs, matches `concurrency.group` above with `github.repository` as a prefix E2E_CONCURRENCY_GROUP: ${{ github.repository }}-e2e-tests-${{ github.ref_name }}-${{ github.ref_name == 'main' && github.sha || 'anysha' }} @@ -362,6 +360,10 @@ jobs: create-test-report: needs: [ check-permissions, build-and-test-locally, coverage-report, build-build-tools-image, benchmarks ] if: ${{ !cancelled() && contains(fromJSON('["skipped", "success"]'), needs.check-permissions.result) }} + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: write outputs: report-url: ${{ steps.create-allure-report.outputs.report-url }} @@ -382,6 +384,7 @@ jobs: uses: ./.github/actions/allure-report-generate with: store-test-results-into-db: true + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} env: REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} @@ -413,6 +416,10 @@ jobs: coverage-report: if: ${{ !startsWith(github.ref_name, 'release') }} needs: [ check-permissions, build-build-tools-image, build-and-test-locally ] + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: write runs-on: [ self-hosted, small ] container: image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm @@ -439,12 +446,14 @@ jobs: with: name: neon-${{ runner.os }}-${{ runner.arch }}-${{ matrix.build_type }}-artifact path: /tmp/neon + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Get coverage artifact uses: ./.github/actions/download with: name: coverage-data-artifact path: /tmp/coverage + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Merge coverage data run: scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage merge @@ -575,6 +584,10 @@ jobs: neon-image: needs: [ neon-image-arch, tag ] runs-on: ubuntu-22.04 + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: read steps: - uses: docker/login-action@v3 @@ -589,11 +602,15 @@ jobs: neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-x64 \ neondatabase/neon:${{ needs.tag.outputs.build-tag }}-bookworm-arm64 - - uses: docker/login-action@v3 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_DEV }} - password: ${{ secrets.AWS_SECRET_KEY_DEV }} + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 3600 + + - name: Login to Amazon Dev ECR + uses: aws-actions/amazon-ecr-login@v2 - name: Push multi-arch image to ECR run: | @@ -602,6 +619,10 @@ jobs: compute-node-image-arch: needs: [ check-permissions, build-build-tools-image, tag ] + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: read strategy: fail-fast: false matrix: @@ -642,11 +663,15 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - uses: docker/login-action@v3 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_DEV }} - password: ${{ secrets.AWS_SECRET_KEY_DEV }} + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 3600 + + - name: Login to Amazon Dev ECR + uses: aws-actions/amazon-ecr-login@v2 - uses: docker/login-action@v3 with: @@ -719,6 +744,10 @@ jobs: compute-node-image: needs: [ compute-node-image-arch, tag ] + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: read runs-on: ubuntu-22.04 strategy: @@ -763,11 +792,15 @@ jobs: neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-x64 \ neondatabase/compute-tools:${{ needs.tag.outputs.build-tag }}-${{ matrix.version.debian }}-arm64 - - uses: docker/login-action@v3 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_DEV }} - password: ${{ secrets.AWS_SECRET_KEY_DEV }} + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 3600 + + - name: Login to Amazon Dev ECR + uses: aws-actions/amazon-ecr-login@v2 - name: Push multi-arch compute-node-${{ matrix.version.pg }} image to ECR run: | @@ -892,7 +925,9 @@ jobs: runs-on: ubuntu-22.04 permissions: - id-token: write # for `aws-actions/configure-aws-credentials` + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: read env: VERSIONS: v14 v15 v16 v17 @@ -903,12 +938,15 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - name: Login to dev ECR - uses: docker/login-action@v3 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_DEV }} - password: ${{ secrets.AWS_SECRET_KEY_DEV }} + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 3600 + + - name: Login to Amazon Dev ECR + uses: aws-actions/amazon-ecr-login@v2 - name: Copy vm-compute-node images to ECR run: | @@ -1062,7 +1100,10 @@ jobs: needs: [ check-permissions, promote-images, tag, build-and-test-locally, trigger-custom-extensions-build-and-wait, push-to-acr-dev, push-to-acr-prod ] # `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod` if: (github.ref_name == 'main' || github.ref_name == 'release' || github.ref_name == 'release-proxy' || github.ref_name == 'release-compute') && !failure() && !cancelled() - + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: write runs-on: [ self-hosted, small ] container: 369495373322.dkr.ecr.eu-central-1.amazonaws.com/ansible:latest steps: @@ -1184,6 +1225,10 @@ jobs: # The job runs on `release` branch and copies compatibility data and Neon artifact from the last *release PR* to the latest directory promote-compatibility-data: needs: [ deploy ] + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: read # `!failure() && !cancelled()` is required because the workflow transitively depends on the job that can be skipped: `push-to-acr-dev` and `push-to-acr-prod` if: github.ref_name == 'release' && !failure() && !cancelled() diff --git a/.github/workflows/cloud-regress.yml b/.github/workflows/cloud-regress.yml index 57194090cf..457634ddad 100644 --- a/.github/workflows/cloud-regress.yml +++ b/.github/workflows/cloud-regress.yml @@ -19,14 +19,15 @@ concurrency: group: ${{ github.workflow }} cancel-in-progress: true +permissions: + id-token: write # aws-actions/configure-aws-credentials + jobs: regress: env: POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install TEST_OUTPUT: /tmp/test_output BUILD_TYPE: remote - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} strategy: fail-fast: false matrix: @@ -78,6 +79,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Create a new branch id: create-branch @@ -107,6 +109,8 @@ jobs: id: create-allure-report if: ${{ !cancelled() }} uses: ./.github/actions/allure-report-generate + with: + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Post to a Slack channel if: ${{ github.event.schedule && failure() }} diff --git a/.github/workflows/ingest_benchmark.yml b/.github/workflows/ingest_benchmark.yml index a5810e91a4..6773032263 100644 --- a/.github/workflows/ingest_benchmark.yml +++ b/.github/workflows/ingest_benchmark.yml @@ -64,6 +64,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Create Neon Project if: ${{ matrix.target_project == 'new_empty_project' }} diff --git a/.github/workflows/neon_extra_builds.yml b/.github/workflows/neon_extra_builds.yml index 092831adb9..1f85c2e102 100644 --- a/.github/workflows/neon_extra_builds.yml +++ b/.github/workflows/neon_extra_builds.yml @@ -143,6 +143,10 @@ jobs: gather-rust-build-stats: needs: [ check-permissions, build-build-tools-image ] + permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write + contents: write if: | contains(github.event.pull_request.labels.*.name, 'run-extra-build-stats') || contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') || @@ -177,13 +181,18 @@ jobs: - name: Produce the build stats run: PQ_LIB_DIR=$(pwd)/pg_install/v17/lib cargo build --all --release --timings -j$(nproc) + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 3600 + - name: Upload the build stats id: upload-stats env: BUCKET: neon-github-public-dev SHA: ${{ github.event.pull_request.head.sha || github.sha }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} run: | REPORT_URL=https://${BUCKET}.s3.amazonaws.com/build-stats/${SHA}/${GITHUB_RUN_ID}/cargo-timing.html aws s3 cp --only-show-errors ./target/cargo-timings/cargo-timing.html "s3://${BUCKET}/build-stats/${SHA}/${GITHUB_RUN_ID}/" diff --git a/.github/workflows/periodic_pagebench.yml b/.github/workflows/periodic_pagebench.yml index 6b98bc873f..a04ceb4a24 100644 --- a/.github/workflows/periodic_pagebench.yml +++ b/.github/workflows/periodic_pagebench.yml @@ -21,6 +21,9 @@ defaults: run: shell: bash -euo pipefail {0} +permissions: + id-token: write # aws-actions/configure-aws-credentials + concurrency: group: ${{ github.workflow }} cancel-in-progress: false @@ -124,11 +127,10 @@ jobs: cat "test_log_${GITHUB_RUN_ID}" - name: Create Allure report - env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} if: ${{ !cancelled() }} uses: ./.github/actions/allure-report-generate + with: + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Post to a Slack channel if: ${{ github.event.schedule && failure() }} diff --git a/.github/workflows/pg-clients.yml b/.github/workflows/pg-clients.yml index 4f5495cbe2..5c999d3810 100644 --- a/.github/workflows/pg-clients.yml +++ b/.github/workflows/pg-clients.yml @@ -25,11 +25,13 @@ defaults: run: shell: bash -euxo pipefail {0} +permissions: + id-token: write # aws-actions/configure-aws-credentials + statuses: write # require for posting a status update + env: DEFAULT_PG_VERSION: 16 PLATFORM: neon-captest-new - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_DEV }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_KEY_DEV }} AWS_DEFAULT_REGION: eu-central-1 jobs: @@ -94,6 +96,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Create Neon Project id: create-neon-project @@ -126,6 +129,7 @@ jobs: uses: ./.github/actions/allure-report-generate with: store-test-results-into-db: true + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} env: REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} @@ -159,6 +163,7 @@ jobs: name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact path: /tmp/neon/ prefix: latest + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} - name: Create Neon Project id: create-neon-project @@ -191,6 +196,7 @@ jobs: uses: ./.github/actions/allure-report-generate with: store-test-results-into-db: true + aws_oicd_role_arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} env: REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }} diff --git a/.github/workflows/pin-build-tools-image.yml b/.github/workflows/pin-build-tools-image.yml index 5b43d97de6..626de2b0e0 100644 --- a/.github/workflows/pin-build-tools-image.yml +++ b/.github/workflows/pin-build-tools-image.yml @@ -67,7 +67,7 @@ jobs: runs-on: ubuntu-22.04 permissions: - id-token: write # for `azure/login` + id-token: write # for `azure/login` and aws auth steps: - uses: docker/login-action@v3 @@ -75,11 +75,15 @@ jobs: username: ${{ secrets.NEON_DOCKERHUB_USERNAME }} password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }} - - uses: docker/login-action@v3 + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 with: - registry: 369495373322.dkr.ecr.eu-central-1.amazonaws.com - username: ${{ secrets.AWS_ACCESS_KEY_DEV }} - password: ${{ secrets.AWS_SECRET_KEY_DEV }} + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }} + role-duration-seconds: 3600 + + - name: Login to Amazon Dev ECR + uses: aws-actions/amazon-ecr-login@v2 - name: Azure login uses: azure/login@6c251865b4e6290e7b78be643ea2d005bc51f69a # @v2.1.1 diff --git a/.github/workflows/pre-merge-checks.yml b/.github/workflows/pre-merge-checks.yml index d2f9d8a666..b2e00d94f7 100644 --- a/.github/workflows/pre-merge-checks.yml +++ b/.github/workflows/pre-merge-checks.yml @@ -63,6 +63,7 @@ jobs: if: always() permissions: statuses: write # for `github.repos.createCommitStatus(...)` + contents: write needs: - get-changed-files - check-codestyle-python diff --git a/control_plane/src/background_process.rs b/control_plane/src/background_process.rs index 94a072e394..af312d73a7 100644 --- a/control_plane/src/background_process.rs +++ b/control_plane/src/background_process.rs @@ -274,6 +274,7 @@ fn fill_remote_storage_secrets_vars(mut cmd: &mut Command) -> &mut Command { for env_key in [ "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", + "AWS_SESSION_TOKEN", "AWS_PROFILE", // HOME is needed in combination with `AWS_PROFILE` to pick up the SSO sessions. "HOME", diff --git a/test_runner/fixtures/remote_storage.py b/test_runner/fixtures/remote_storage.py index 4e1e8a884f..d969971a35 100644 --- a/test_runner/fixtures/remote_storage.py +++ b/test_runner/fixtures/remote_storage.py @@ -70,6 +70,9 @@ class MockS3Server: def secret_key(self) -> str: return "test" + def session_token(self) -> str: + return "test" + def kill(self): self.server.stop() @@ -161,6 +164,7 @@ class S3Storage: bucket_region: str access_key: str | None secret_key: str | None + session_token: str | None aws_profile: str | None prefix_in_bucket: str client: S3Client @@ -181,13 +185,18 @@ class S3Storage: if home is not None: env["HOME"] = home return env - if self.access_key is not None and self.secret_key is not None: + if ( + self.access_key is not None + and self.secret_key is not None + and self.session_token is not None + ): return { "AWS_ACCESS_KEY_ID": self.access_key, "AWS_SECRET_ACCESS_KEY": self.secret_key, + "AWS_SESSION_TOKEN": self.session_token, } raise RuntimeError( - "Either AWS_PROFILE or (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY) have to be set for S3Storage" + "Either AWS_PROFILE or (AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN) have to be set for S3Storage" ) def to_string(self) -> str: @@ -352,6 +361,7 @@ class RemoteStorageKind(StrEnum): mock_region = mock_s3_server.region() access_key, secret_key = mock_s3_server.access_key(), mock_s3_server.secret_key() + session_token = mock_s3_server.session_token() client = boto3.client( "s3", @@ -359,6 +369,7 @@ class RemoteStorageKind(StrEnum): region_name=mock_region, aws_access_key_id=access_key, aws_secret_access_key=secret_key, + aws_session_token=session_token, ) bucket_name = to_bucket_name(user, test_name) @@ -372,6 +383,7 @@ class RemoteStorageKind(StrEnum): bucket_region=mock_region, access_key=access_key, secret_key=secret_key, + session_token=session_token, aws_profile=None, prefix_in_bucket="", client=client, @@ -383,9 +395,10 @@ class RemoteStorageKind(StrEnum): env_access_key = os.getenv("AWS_ACCESS_KEY_ID") env_secret_key = os.getenv("AWS_SECRET_ACCESS_KEY") + env_access_token = os.getenv("AWS_SESSION_TOKEN") env_profile = os.getenv("AWS_PROFILE") assert ( - env_access_key and env_secret_key + env_access_key and env_secret_key and env_access_token ) or env_profile, "need to specify either access key and secret access key or profile" bucket_name = bucket_name or os.getenv("REMOTE_STORAGE_S3_BUCKET") @@ -398,6 +411,9 @@ class RemoteStorageKind(StrEnum): client = boto3.client( "s3", region_name=bucket_region, + aws_access_key_id=env_access_key, + aws_secret_access_key=env_secret_key, + aws_session_token=env_access_token, ) return S3Storage( @@ -405,6 +421,7 @@ class RemoteStorageKind(StrEnum): bucket_region=bucket_region, access_key=env_access_key, secret_key=env_secret_key, + session_token=env_access_token, aws_profile=env_profile, prefix_in_bucket=prefix_in_bucket, client=client, From 2f3433876fdce12093ee0c706caf2e04b4de2f51 Mon Sep 17 00:00:00 2001 From: a-masterov <72613290+a-masterov@users.noreply.github.com> Date: Thu, 12 Dec 2024 17:34:07 +0100 Subject: [PATCH 36/44] Change the channel for notification. (#10112) ## Problem Now notifications about failures in `pg_regress` tests run on the staging cloud instance, reach the channel `on-call-staging-stream`, while they should reach `on-call-qa-staging-stream` ## Summary of changes The channel changed. --- .github/actionlint.yml | 1 + .github/workflows/cloud-regress.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 27c8fb3c23..9d8389faa5 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -23,3 +23,4 @@ config-variables: - BENCHMARK_INGEST_TARGET_PROJECTID - PGREGRESS_PG16_PROJECT_ID - PGREGRESS_PG17_PROJECT_ID + - SLACK_ON_CALL_QA_STAGING_STREAM diff --git a/.github/workflows/cloud-regress.yml b/.github/workflows/cloud-regress.yml index 457634ddad..2fc26baa21 100644 --- a/.github/workflows/cloud-regress.yml +++ b/.github/workflows/cloud-regress.yml @@ -116,7 +116,7 @@ jobs: if: ${{ github.event.schedule && failure() }} uses: slackapi/slack-github-action@v1 with: - channel-id: "C033QLM5P7D" # on-call-staging-stream + channel-id: ${{ vars.SLACK_ON_CALL_QA_STAGING_STREAM }} slack-message: | Periodic pg_regress on staging: ${{ job.status }} <${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|GitHub Run> From 53721266f1cf6f835541a8b64bbd0b46c60761e2 Mon Sep 17 00:00:00 2001 From: Heikki Linnakangas Date: Thu, 12 Dec 2024 19:05:58 +0200 Subject: [PATCH 37/44] Disable connection logging in pgbouncer by default (#10118) It can produce a lot of logs, making pgbouncer itself consume all CPU in extreme cases. We saw that happen in stress testing. --- compute/etc/pgbouncer.ini | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/compute/etc/pgbouncer.ini b/compute/etc/pgbouncer.ini index abcd165636..604b4e41ea 100644 --- a/compute/etc/pgbouncer.ini +++ b/compute/etc/pgbouncer.ini @@ -19,3 +19,10 @@ max_prepared_statements=0 admin_users=postgres unix_socket_dir=/tmp/ unix_socket_mode=0777 + +;; Disable connection logging. It produces a lot of logs that no one looks at, +;; and we can get similar log entries from the proxy too. We had incidents in +;; the past where the logging significantly stressed the log device or pgbouncer +;; itself. +log_connections=0 +log_disconnections=0 From 6d5687521b6eede3166884c0247405ebd59c6f8a Mon Sep 17 00:00:00 2001 From: Rahul Patil Date: Thu, 12 Dec 2024 19:53:35 +0100 Subject: [PATCH 38/44] fix(ci): Allow github-script to post test reports (#10120) Allow github-script to post test reports --- .github/workflows/build_and_test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 62f190a0c2..4bfb5077c6 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -364,6 +364,7 @@ jobs: id-token: write # aws-actions/configure-aws-credentials statuses: write contents: write + pull-requests: write outputs: report-url: ${{ steps.create-allure-report.outputs.report-url }} From a93e3d31cc5152c279480ded76ea886867fbfea3 Mon Sep 17 00:00:00 2001 From: John Spray Date: Thu, 12 Dec 2024 19:35:38 +0000 Subject: [PATCH 39/44] storcon: refine logic for choosing AZ on tenant creation (#10054) ## Problem When we update our scheduler/optimization code to respect AZs properly (https://github.com/neondatabase/neon/pull/9916), the choice of AZ becomes a much higher-stakes decision. We will pretty much always run a tenant in its preferred AZ, and that AZ is fixed for the lifetime of the tenant (unless a human intervenes) Eventually, when we do auto-balancing based on utilization, I anticipate that part of that will be to automatically change the AZ of tenants if our original scheduling decisions have caused imbalance, but as an interim measure, we can at least avoid making this scheduling decision based purely on which AZ contains the emptiest node. This is a precursor to https://github.com/neondatabase/neon/pull/9947 ## Summary of changes - When creating a tenant, instead of scheduling a shard and then reading its preferred AZ back, make the AZ decision first. - Instead of choosing AZ based on which node is emptiest, use the median utilization of nodes in each AZ to pick the AZ to use. This avoids bad AZ decisions during periods when some node has very low utilization (such as after replacing a dead node) I considered also making the selection a weighted pseudo-random choice based on utilization, but wanted to avoid destabilising tests with that for now. --- libs/pageserver_api/src/controller_api.rs | 2 +- storage_controller/src/scheduler.rs | 93 +++++++++++++++++++++++ storage_controller/src/service.rs | 62 +++++---------- storage_controller/src/tenant_shard.rs | 15 ++-- 4 files changed, 118 insertions(+), 54 deletions(-) diff --git a/libs/pageserver_api/src/controller_api.rs b/libs/pageserver_api/src/controller_api.rs index 6839ef69f5..ec7b81423a 100644 --- a/libs/pageserver_api/src/controller_api.rs +++ b/libs/pageserver_api/src/controller_api.rs @@ -75,7 +75,7 @@ pub struct TenantPolicyRequest { pub scheduling: Option, } -#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Hash, Debug)] +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq, Hash, Debug, PartialOrd, Ord)] pub struct AvailabilityZone(pub String); impl Display for AvailabilityZone { diff --git a/storage_controller/src/scheduler.rs b/storage_controller/src/scheduler.rs index ecc6b11e47..51a4cf35be 100644 --- a/storage_controller/src/scheduler.rs +++ b/storage_controller/src/scheduler.rs @@ -742,6 +742,50 @@ impl Scheduler { self.schedule_shard::(&[], &None, &ScheduleContext::default()) } + /// For choosing which AZ to schedule a new shard into, use this. It will return the + /// AZ with the lowest median utilization. + /// + /// We use an AZ-wide measure rather than simply selecting the AZ of the least-loaded + /// node, because while tenants start out single sharded, when they grow and undergo + /// shard-split, they will occupy space on many nodes within an AZ. + /// + /// We use median rather than total free space or mean utilization, because + /// we wish to avoid preferring AZs that have low-load nodes resulting from + /// recent replacements. + /// + /// The practical result is that we will pick an AZ based on its median node, and + /// then actually _schedule_ the new shard onto the lowest-loaded node in that AZ. + pub(crate) fn get_az_for_new_tenant(&self) -> Option { + if self.nodes.is_empty() { + return None; + } + + let mut scores_by_az = HashMap::new(); + for (node_id, node) in &self.nodes { + let az_scores = scores_by_az.entry(&node.az).or_insert_with(Vec::new); + let score = match &node.may_schedule { + MaySchedule::Yes(utilization) => utilization.score(), + MaySchedule::No => PageserverUtilization::full().score(), + }; + az_scores.push((node_id, node, score)); + } + + // Sort by utilization. Also include the node ID to break ties. + for scores in scores_by_az.values_mut() { + scores.sort_by_key(|i| (i.2, i.0)); + } + + let mut median_by_az = scores_by_az + .iter() + .map(|(az, nodes)| (*az, nodes.get(nodes.len() / 2).unwrap().2)) + .collect::>(); + // Sort by utilization. Also include the AZ to break ties. + median_by_az.sort_by_key(|i| (i.1, i.0)); + + // Return the AZ with the lowest median utilization + Some(median_by_az.first().unwrap().0.clone()) + } + /// Unit test access to internal state #[cfg(test)] pub(crate) fn get_node_shard_count(&self, node_id: NodeId) -> usize { @@ -1087,4 +1131,53 @@ mod tests { intent.clear(&mut scheduler); } } + + #[test] + fn az_scheduling_for_new_tenant() { + let az_a_tag = AvailabilityZone("az-a".to_string()); + let az_b_tag = AvailabilityZone("az-b".to_string()); + let nodes = test_utils::make_test_nodes( + 6, + &[ + az_a_tag.clone(), + az_a_tag.clone(), + az_a_tag.clone(), + az_b_tag.clone(), + az_b_tag.clone(), + az_b_tag.clone(), + ], + ); + + let mut scheduler = Scheduler::new(nodes.values()); + + /// Force the utilization of a node in Scheduler's state to a particular + /// number of bytes used. + fn set_utilization(scheduler: &mut Scheduler, node_id: NodeId, shard_count: u32) { + let mut node = Node::new( + node_id, + "".to_string(), + 0, + "".to_string(), + 0, + scheduler.nodes.get(&node_id).unwrap().az.clone(), + ); + node.set_availability(NodeAvailability::Active(test_utilization::simple( + shard_count, + 0, + ))); + scheduler.node_upsert(&node); + } + + // Initial empty state. Scores are tied, scheduler prefers lower AZ ID. + assert_eq!(scheduler.get_az_for_new_tenant(), Some(az_a_tag.clone())); + + // Put some utilization on one node in AZ A: this should change nothing, as the median hasn't changed + set_utilization(&mut scheduler, NodeId(1), 1000000); + assert_eq!(scheduler.get_az_for_new_tenant(), Some(az_a_tag.clone())); + + // Put some utilization on a second node in AZ A: now the median has changed, so the scheduler + // should prefer the other AZ. + set_utilization(&mut scheduler, NodeId(2), 1000000); + assert_eq!(scheduler.get_az_for_new_tenant(), Some(az_b_tag.clone())); + } } diff --git a/storage_controller/src/service.rs b/storage_controller/src/service.rs index 894b67fdc6..746177c089 100644 --- a/storage_controller/src/service.rs +++ b/storage_controller/src/service.rs @@ -1582,6 +1582,7 @@ impl Service { attach_req.tenant_shard_id, ShardIdentity::unsharded(), PlacementPolicy::Attached(0), + None, ), ); tracing::info!("Inserted shard {} in memory", attach_req.tenant_shard_id); @@ -2109,6 +2110,16 @@ impl Service { ) }; + let preferred_az_id = { + let locked = self.inner.read().unwrap(); + // Idempotency: take the existing value if the tenant already exists + if let Some(shard) = locked.tenants.get(create_ids.first().unwrap()) { + shard.preferred_az().cloned() + } else { + locked.scheduler.get_az_for_new_tenant() + } + }; + // Ordering: we persist tenant shards before creating them on the pageserver. This enables a caller // to clean up after themselves by issuing a tenant deletion if something goes wrong and we restart // during the creation, rather than risking leaving orphan objects in S3. @@ -2128,7 +2139,7 @@ impl Service { splitting: SplitState::default(), scheduling_policy: serde_json::to_string(&ShardSchedulingPolicy::default()) .unwrap(), - preferred_az_id: None, + preferred_az_id: preferred_az_id.as_ref().map(|az| az.to_string()), }) .collect(); @@ -2164,6 +2175,7 @@ impl Service { &create_req.shard_parameters, create_req.config.clone(), placement_policy.clone(), + preferred_az_id.as_ref(), &mut schedule_context, ) .await; @@ -2177,44 +2189,6 @@ impl Service { } } - let preferred_azs = { - let locked = self.inner.read().unwrap(); - response_shards - .iter() - .filter_map(|resp| { - let az_id = locked - .nodes - .get(&resp.node_id) - .map(|n| n.get_availability_zone_id().clone())?; - - Some((resp.shard_id, az_id)) - }) - .collect::>() - }; - - // Note that we persist the preferred AZ for the new shards separately. - // In theory, we could "peek" the scheduler to determine where the shard will - // land, but the subsequent "real" call into the scheduler might select a different - // node. Hence, we do this awkward update to keep things consistent. - let updated = self - .persistence - .set_tenant_shard_preferred_azs(preferred_azs) - .await - .map_err(|err| { - ApiError::InternalServerError(anyhow::anyhow!( - "Failed to persist preferred az ids: {err}" - )) - })?; - - { - let mut locked = self.inner.write().unwrap(); - for (tid, az_id) in updated { - if let Some(shard) = locked.tenants.get_mut(&tid) { - shard.set_preferred_az(az_id); - } - } - } - // If we failed to schedule shards, then they are still created in the controller, // but we return an error to the requester to avoid a silent failure when someone // tries to e.g. create a tenant whose placement policy requires more nodes than @@ -2245,6 +2219,7 @@ impl Service { /// Helper for tenant creation that does the scheduling for an individual shard. Covers both the /// case of a new tenant and a pre-existing one. + #[allow(clippy::too_many_arguments)] async fn do_initial_shard_scheduling( &self, tenant_shard_id: TenantShardId, @@ -2252,6 +2227,7 @@ impl Service { shard_params: &ShardParameters, config: TenantConfig, placement_policy: PlacementPolicy, + preferred_az_id: Option<&AvailabilityZone>, schedule_context: &mut ScheduleContext, ) -> InitialShardScheduleOutcome { let mut locked = self.inner.write().unwrap(); @@ -2262,10 +2238,6 @@ impl Service { Entry::Occupied(mut entry) => { tracing::info!("Tenant shard {tenant_shard_id} already exists while creating"); - // TODO: schedule() should take an anti-affinity expression that pushes - // attached and secondary locations (independently) away frorm those - // pageservers also holding a shard for this tenant. - if let Err(err) = entry.get_mut().schedule(scheduler, schedule_context) { return InitialShardScheduleOutcome::ShardScheduleError(err); } @@ -2289,6 +2261,7 @@ impl Service { tenant_shard_id, ShardIdentity::from_params(tenant_shard_id.shard_number, shard_params), placement_policy, + preferred_az_id.cloned(), )); state.generation = initial_generation; @@ -4256,7 +4229,8 @@ impl Service { }, ); - let mut child_state = TenantShard::new(child, child_shard, policy.clone()); + let mut child_state = + TenantShard::new(child, child_shard, policy.clone(), preferred_az.clone()); child_state.intent = IntentState::single(scheduler, Some(pageserver)); child_state.observed = ObservedState { locations: child_observed, diff --git a/storage_controller/src/tenant_shard.rs b/storage_controller/src/tenant_shard.rs index 2eb98ee825..f1b921646f 100644 --- a/storage_controller/src/tenant_shard.rs +++ b/storage_controller/src/tenant_shard.rs @@ -472,6 +472,7 @@ impl TenantShard { tenant_shard_id: TenantShardId, shard: ShardIdentity, policy: PlacementPolicy, + preferred_az_id: Option, ) -> Self { metrics::METRICS_REGISTRY .metrics_group @@ -495,7 +496,7 @@ impl TenantShard { last_error: Arc::default(), pending_compute_notification: false, scheduling_policy: ShardSchedulingPolicy::default(), - preferred_az_id: None, + preferred_az_id, } } @@ -1571,6 +1572,7 @@ pub(crate) mod tests { ) .unwrap(), policy, + None, ) } @@ -1597,7 +1599,7 @@ pub(crate) mod tests { shard_number, shard_count, }; - let mut ts = TenantShard::new( + TenantShard::new( tenant_shard_id, ShardIdentity::new( shard_number, @@ -1606,13 +1608,8 @@ pub(crate) mod tests { ) .unwrap(), policy.clone(), - ); - - if let Some(az) = &preferred_az { - ts.set_preferred_az(az.clone()); - } - - ts + preferred_az.clone(), + ) }) .collect() } From 5ff4b991c74141505927a2498310b20c617a8786 Mon Sep 17 00:00:00 2001 From: "Alex Chi Z." <4198311+skyzh@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:23:24 -0500 Subject: [PATCH 40/44] feat(pageserver): gc-compaction split over LSN (#9900) ## Problem part of https://github.com/neondatabase/neon/issues/9114, stacked PR over https://github.com/neondatabase/neon/pull/9897, partially refactored to help with https://github.com/neondatabase/neon/issues/10031 ## Summary of changes * gc-compaction takes `above_lsn` parameter. We only compact the layers above this LSN, and all data below the LSN are treated as if they are on the ancestor branch. * refactored gc-compaction to take `GcCompactJob` that describes the rectangular range to be compacted. * Added unit test for this case. --------- Signed-off-by: Alex Chi Z Co-authored-by: Christian Schwarz --- pageserver/src/http/routes.rs | 13 +- pageserver/src/tenant.rs | 661 ++++++++++++++++++- pageserver/src/tenant/timeline.rs | 71 +- pageserver/src/tenant/timeline/compaction.rs | 233 ++++--- test_runner/regress/test_compaction.py | 4 +- 5 files changed, 879 insertions(+), 103 deletions(-) diff --git a/pageserver/src/http/routes.rs b/pageserver/src/http/routes.rs index 6e9ee976f4..db7d293856 100644 --- a/pageserver/src/http/routes.rs +++ b/pageserver/src/http/routes.rs @@ -2081,13 +2081,20 @@ async fn timeline_compact_handler( .as_ref() .map(|r| r.sub_compaction) .unwrap_or(false); + let sub_compaction_max_job_size_mb = compact_request + .as_ref() + .and_then(|r| r.sub_compaction_max_job_size_mb); + let options = CompactOptions { - compact_range: compact_request + compact_key_range: compact_request .as_ref() - .and_then(|r| r.compact_range.clone()), - compact_below_lsn: compact_request.as_ref().and_then(|r| r.compact_below_lsn), + .and_then(|r| r.compact_key_range.clone()), + compact_lsn_range: compact_request + .as_ref() + .and_then(|r| r.compact_lsn_range.clone()), flags, sub_compaction, + sub_compaction_max_job_size_mb, }; let scheduled = compact_request diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 92078e4b08..99289d5f15 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -44,6 +44,7 @@ use std::sync::atomic::AtomicBool; use std::sync::Weak; use std::time::SystemTime; use storage_broker::BrokerClientChannel; +use timeline::compaction::GcCompactJob; use timeline::compaction::ScheduledCompactionTask; use timeline::import_pgdata; use timeline::offload::offload_timeline; @@ -3017,8 +3018,15 @@ impl Tenant { warn!("ignoring scheduled compaction task: scheduled task must be gc compaction: {:?}", next_scheduled_compaction_task.options); } else if next_scheduled_compaction_task.options.sub_compaction { info!("running scheduled enhanced gc bottom-most compaction with sub-compaction, splitting compaction jobs"); - let jobs = timeline - .gc_compaction_split_jobs(next_scheduled_compaction_task.options) + let jobs: Vec = timeline + .gc_compaction_split_jobs( + GcCompactJob::from_compact_options( + next_scheduled_compaction_task.options.clone(), + ), + next_scheduled_compaction_task + .options + .sub_compaction_max_job_size_mb, + ) .await .map_err(CompactionError::Other)?; if jobs.is_empty() { @@ -3029,9 +3037,23 @@ impl Tenant { let mut guard = self.scheduled_compaction_tasks.lock().unwrap(); let tline_pending_tasks = guard.entry(*timeline_id).or_default(); for (idx, job) in jobs.into_iter().enumerate() { + // Unfortunately we need to convert the `GcCompactJob` back to `CompactionOptions` + // until we do further refactors to allow directly call `compact_with_gc`. + let mut flags: EnumSet = EnumSet::default(); + flags |= CompactFlags::EnhancedGcBottomMostCompaction; + if job.dry_run { + flags |= CompactFlags::DryRun; + } + let options = CompactOptions { + flags, + sub_compaction: false, + compact_key_range: Some(job.compact_key_range.into()), + compact_lsn_range: Some(job.compact_lsn_range.into()), + sub_compaction_max_job_size_mb: None, + }; tline_pending_tasks.push_back(if idx == jobs_len - 1 { ScheduledCompactionTask { - options: job, + options, // The last job in the queue sends the signal and releases the gc guard result_tx: next_scheduled_compaction_task .result_tx @@ -3042,7 +3064,7 @@ impl Tenant { } } else { ScheduledCompactionTask { - options: job, + options, result_tx: None, gc_block: None, } @@ -5742,6 +5764,8 @@ mod tests { #[cfg(feature = "testing")] use timeline::compaction::{KeyHistoryRetention, KeyLogAtLsn}; #[cfg(feature = "testing")] + use timeline::CompactLsnRange; + #[cfg(feature = "testing")] use timeline::GcInfo; static TEST_KEY: Lazy = @@ -9333,7 +9357,6 @@ mod tests { &cancel, CompactOptions { flags: dryrun_flags, - compact_range: None, ..Default::default() }, &ctx, @@ -9582,7 +9605,6 @@ mod tests { &cancel, CompactOptions { flags: dryrun_flags, - compact_range: None, ..Default::default() }, &ctx, @@ -9612,6 +9634,8 @@ mod tests { #[cfg(feature = "testing")] #[tokio::test] async fn test_simple_bottom_most_compaction_on_branch() -> anyhow::Result<()> { + use timeline::CompactLsnRange; + let harness = TenantHarness::create("test_simple_bottom_most_compaction_on_branch").await?; let (tenant, ctx) = harness.load().await; @@ -9804,6 +9828,22 @@ mod tests { verify_result().await; + // Piggyback a compaction with above_lsn. Ensure it works correctly when the specified LSN intersects with the layer files. + // Now we already have a single large delta layer, so the compaction min_layer_lsn should be the same as ancestor LSN (0x18). + branch_tline + .compact_with_gc( + &cancel, + CompactOptions { + compact_lsn_range: Some(CompactLsnRange::above(Lsn(0x40))), + ..Default::default() + }, + &ctx, + ) + .await + .unwrap(); + + verify_result().await; + Ok(()) } @@ -10092,7 +10132,7 @@ mod tests { &cancel, CompactOptions { flags: EnumSet::new(), - compact_range: Some((get_key(0)..get_key(2)).into()), + compact_key_range: Some((get_key(0)..get_key(2)).into()), ..Default::default() }, &ctx, @@ -10139,7 +10179,7 @@ mod tests { &cancel, CompactOptions { flags: EnumSet::new(), - compact_range: Some((get_key(2)..get_key(4)).into()), + compact_key_range: Some((get_key(2)..get_key(4)).into()), ..Default::default() }, &ctx, @@ -10191,7 +10231,7 @@ mod tests { &cancel, CompactOptions { flags: EnumSet::new(), - compact_range: Some((get_key(4)..get_key(9)).into()), + compact_key_range: Some((get_key(4)..get_key(9)).into()), ..Default::default() }, &ctx, @@ -10242,7 +10282,7 @@ mod tests { &cancel, CompactOptions { flags: EnumSet::new(), - compact_range: Some((get_key(9)..get_key(10)).into()), + compact_key_range: Some((get_key(9)..get_key(10)).into()), ..Default::default() }, &ctx, @@ -10298,7 +10338,7 @@ mod tests { &cancel, CompactOptions { flags: EnumSet::new(), - compact_range: Some((get_key(0)..get_key(10)).into()), + compact_key_range: Some((get_key(0)..get_key(10)).into()), ..Default::default() }, &ctx, @@ -10327,7 +10367,6 @@ mod tests { }, ], ); - Ok(()) } @@ -10380,4 +10419,602 @@ mod tests { Ok(()) } + + #[cfg(feature = "testing")] + #[tokio::test] + async fn test_simple_bottom_most_compaction_above_lsn() -> anyhow::Result<()> { + let harness = TenantHarness::create("test_simple_bottom_most_compaction_above_lsn").await?; + let (tenant, ctx) = harness.load().await; + + fn get_key(id: u32) -> Key { + // using aux key here b/c they are guaranteed to be inside `collect_keyspace`. + let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + + let img_layer = (0..10) + .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10")))) + .collect_vec(); + + let delta1 = vec![( + get_key(1), + Lsn(0x20), + Value::WalRecord(NeonWalRecord::wal_append("@0x20")), + )]; + let delta4 = vec![( + get_key(1), + Lsn(0x28), + Value::WalRecord(NeonWalRecord::wal_append("@0x28")), + )]; + let delta2 = vec![ + ( + get_key(1), + Lsn(0x30), + Value::WalRecord(NeonWalRecord::wal_append("@0x30")), + ), + ( + get_key(1), + Lsn(0x38), + Value::WalRecord(NeonWalRecord::wal_append("@0x38")), + ), + ]; + let delta3 = vec![ + ( + get_key(8), + Lsn(0x48), + Value::WalRecord(NeonWalRecord::wal_append("@0x48")), + ), + ( + get_key(9), + Lsn(0x48), + Value::WalRecord(NeonWalRecord::wal_append("@0x48")), + ), + ]; + + let tline = tenant + .create_test_timeline_with_layers( + TIMELINE_ID, + Lsn(0x10), + DEFAULT_PG_VERSION, + &ctx, + vec![ + // delta1/2/4 only contain a single key but multiple updates + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x28), delta1), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta2), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x28)..Lsn(0x30), delta4), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta3), + ], // delta layers + vec![(Lsn(0x10), img_layer)], // image layers + Lsn(0x50), + ) + .await?; + { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x30)) + .wait() + .await; + // Update GC info + let mut guard = tline.gc_info.write().unwrap(); + *guard = GcInfo { + retain_lsns: vec![ + (Lsn(0x10), tline.timeline_id, MaybeOffloaded::No), + (Lsn(0x20), tline.timeline_id, MaybeOffloaded::No), + ], + cutoffs: GcCutoffs { + time: Lsn(0x30), + space: Lsn(0x30), + }, + leases: Default::default(), + within_ancestor_pitr: false, + }; + } + + let expected_result = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30@0x38"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10@0x48"), + Bytes::from_static(b"value 9@0x10@0x48"), + ]; + + let expected_result_at_gc_horizon = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; + + let expected_result_at_lsn_20 = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; + + let expected_result_at_lsn_10 = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; + + let verify_result = || async { + let gc_horizon = { + let gc_info = tline.gc_info.read().unwrap(); + gc_info.cutoffs.time + }; + for idx in 0..10 { + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x50), &ctx) + .await + .unwrap(), + &expected_result[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), gc_horizon, &ctx) + .await + .unwrap(), + &expected_result_at_gc_horizon[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x20), &ctx) + .await + .unwrap(), + &expected_result_at_lsn_20[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x10), &ctx) + .await + .unwrap(), + &expected_result_at_lsn_10[idx] + ); + } + }; + + verify_result().await; + + let cancel = CancellationToken::new(); + tline + .compact_with_gc( + &cancel, + CompactOptions { + compact_lsn_range: Some(CompactLsnRange::above(Lsn(0x28))), + ..Default::default() + }, + &ctx, + ) + .await + .unwrap(); + verify_result().await; + + let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await; + check_layer_map_key_eq( + all_layers, + vec![ + // The original image layer, not compacted + PersistentLayerKey { + key_range: get_key(0)..get_key(10), + lsn_range: Lsn(0x10)..Lsn(0x11), + is_delta: false, + }, + // Delta layer below the specified above_lsn not compacted + PersistentLayerKey { + key_range: get_key(1)..get_key(2), + lsn_range: Lsn(0x20)..Lsn(0x28), + is_delta: true, + }, + // Delta layer compacted above the LSN + PersistentLayerKey { + key_range: get_key(1)..get_key(10), + lsn_range: Lsn(0x28)..Lsn(0x50), + is_delta: true, + }, + ], + ); + + // compact again + tline + .compact_with_gc(&cancel, CompactOptions::default(), &ctx) + .await + .unwrap(); + verify_result().await; + + let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await; + check_layer_map_key_eq( + all_layers, + vec![ + // The compacted image layer (full key range) + PersistentLayerKey { + key_range: Key::MIN..Key::MAX, + lsn_range: Lsn(0x10)..Lsn(0x11), + is_delta: false, + }, + // All other data in the delta layer + PersistentLayerKey { + key_range: get_key(1)..get_key(10), + lsn_range: Lsn(0x10)..Lsn(0x50), + is_delta: true, + }, + ], + ); + + Ok(()) + } + + #[cfg(feature = "testing")] + #[tokio::test] + async fn test_simple_bottom_most_compaction_rectangle() -> anyhow::Result<()> { + let harness = TenantHarness::create("test_simple_bottom_most_compaction_rectangle").await?; + let (tenant, ctx) = harness.load().await; + + fn get_key(id: u32) -> Key { + // using aux key here b/c they are guaranteed to be inside `collect_keyspace`. + let mut key = Key::from_hex("620000000033333333444444445500000000").unwrap(); + key.field6 = id; + key + } + + let img_layer = (0..10) + .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10")))) + .collect_vec(); + + let delta1 = vec![( + get_key(1), + Lsn(0x20), + Value::WalRecord(NeonWalRecord::wal_append("@0x20")), + )]; + let delta4 = vec![( + get_key(1), + Lsn(0x28), + Value::WalRecord(NeonWalRecord::wal_append("@0x28")), + )]; + let delta2 = vec![ + ( + get_key(1), + Lsn(0x30), + Value::WalRecord(NeonWalRecord::wal_append("@0x30")), + ), + ( + get_key(1), + Lsn(0x38), + Value::WalRecord(NeonWalRecord::wal_append("@0x38")), + ), + ]; + let delta3 = vec![ + ( + get_key(8), + Lsn(0x48), + Value::WalRecord(NeonWalRecord::wal_append("@0x48")), + ), + ( + get_key(9), + Lsn(0x48), + Value::WalRecord(NeonWalRecord::wal_append("@0x48")), + ), + ]; + + let tline = tenant + .create_test_timeline_with_layers( + TIMELINE_ID, + Lsn(0x10), + DEFAULT_PG_VERSION, + &ctx, + vec![ + // delta1/2/4 only contain a single key but multiple updates + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x20)..Lsn(0x28), delta1), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta2), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x28)..Lsn(0x30), delta4), + DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x30)..Lsn(0x50), delta3), + ], // delta layers + vec![(Lsn(0x10), img_layer)], // image layers + Lsn(0x50), + ) + .await?; + { + tline + .latest_gc_cutoff_lsn + .lock_for_write() + .store_and_unlock(Lsn(0x30)) + .wait() + .await; + // Update GC info + let mut guard = tline.gc_info.write().unwrap(); + *guard = GcInfo { + retain_lsns: vec![ + (Lsn(0x10), tline.timeline_id, MaybeOffloaded::No), + (Lsn(0x20), tline.timeline_id, MaybeOffloaded::No), + ], + cutoffs: GcCutoffs { + time: Lsn(0x30), + space: Lsn(0x30), + }, + leases: Default::default(), + within_ancestor_pitr: false, + }; + } + + let expected_result = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30@0x38"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10@0x48"), + Bytes::from_static(b"value 9@0x10@0x48"), + ]; + + let expected_result_at_gc_horizon = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20@0x28@0x30"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; + + let expected_result_at_lsn_20 = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10@0x20"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; + + let expected_result_at_lsn_10 = [ + Bytes::from_static(b"value 0@0x10"), + Bytes::from_static(b"value 1@0x10"), + Bytes::from_static(b"value 2@0x10"), + Bytes::from_static(b"value 3@0x10"), + Bytes::from_static(b"value 4@0x10"), + Bytes::from_static(b"value 5@0x10"), + Bytes::from_static(b"value 6@0x10"), + Bytes::from_static(b"value 7@0x10"), + Bytes::from_static(b"value 8@0x10"), + Bytes::from_static(b"value 9@0x10"), + ]; + + let verify_result = || async { + let gc_horizon = { + let gc_info = tline.gc_info.read().unwrap(); + gc_info.cutoffs.time + }; + for idx in 0..10 { + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x50), &ctx) + .await + .unwrap(), + &expected_result[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), gc_horizon, &ctx) + .await + .unwrap(), + &expected_result_at_gc_horizon[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x20), &ctx) + .await + .unwrap(), + &expected_result_at_lsn_20[idx] + ); + assert_eq!( + tline + .get(get_key(idx as u32), Lsn(0x10), &ctx) + .await + .unwrap(), + &expected_result_at_lsn_10[idx] + ); + } + }; + + verify_result().await; + + let cancel = CancellationToken::new(); + + tline + .compact_with_gc( + &cancel, + CompactOptions { + compact_key_range: Some((get_key(0)..get_key(2)).into()), + compact_lsn_range: Some((Lsn(0x20)..Lsn(0x28)).into()), + ..Default::default() + }, + &ctx, + ) + .await + .unwrap(); + verify_result().await; + + let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await; + check_layer_map_key_eq( + all_layers, + vec![ + // The original image layer, not compacted + PersistentLayerKey { + key_range: get_key(0)..get_key(10), + lsn_range: Lsn(0x10)..Lsn(0x11), + is_delta: false, + }, + // According the selection logic, we select all layers with start key <= 0x28, so we would merge the layer 0x20-0x28 and + // the layer 0x28-0x30 into one. + PersistentLayerKey { + key_range: get_key(1)..get_key(2), + lsn_range: Lsn(0x20)..Lsn(0x30), + is_delta: true, + }, + // Above the upper bound and untouched + PersistentLayerKey { + key_range: get_key(1)..get_key(2), + lsn_range: Lsn(0x30)..Lsn(0x50), + is_delta: true, + }, + // This layer is untouched + PersistentLayerKey { + key_range: get_key(8)..get_key(10), + lsn_range: Lsn(0x30)..Lsn(0x50), + is_delta: true, + }, + ], + ); + + tline + .compact_with_gc( + &cancel, + CompactOptions { + compact_key_range: Some((get_key(3)..get_key(8)).into()), + compact_lsn_range: Some((Lsn(0x28)..Lsn(0x40)).into()), + ..Default::default() + }, + &ctx, + ) + .await + .unwrap(); + verify_result().await; + + let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await; + check_layer_map_key_eq( + all_layers, + vec![ + // The original image layer, not compacted + PersistentLayerKey { + key_range: get_key(0)..get_key(10), + lsn_range: Lsn(0x10)..Lsn(0x11), + is_delta: false, + }, + // Not in the compaction key range, uncompacted + PersistentLayerKey { + key_range: get_key(1)..get_key(2), + lsn_range: Lsn(0x20)..Lsn(0x30), + is_delta: true, + }, + // Not in the compaction key range, uncompacted but need rewrite because the delta layer overlaps with the range + PersistentLayerKey { + key_range: get_key(1)..get_key(2), + lsn_range: Lsn(0x30)..Lsn(0x50), + is_delta: true, + }, + // Note that when we specify the LSN upper bound to be 0x40, the compaction algorithm will not try to cut the layer + // horizontally in half. Instead, it will include all LSNs that overlap with 0x40. So the real max_lsn of the compaction + // becomes 0x50. + PersistentLayerKey { + key_range: get_key(8)..get_key(10), + lsn_range: Lsn(0x30)..Lsn(0x50), + is_delta: true, + }, + ], + ); + + // compact again + tline + .compact_with_gc( + &cancel, + CompactOptions { + compact_key_range: Some((get_key(0)..get_key(5)).into()), + compact_lsn_range: Some((Lsn(0x20)..Lsn(0x50)).into()), + ..Default::default() + }, + &ctx, + ) + .await + .unwrap(); + verify_result().await; + + let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await; + check_layer_map_key_eq( + all_layers, + vec![ + // The original image layer, not compacted + PersistentLayerKey { + key_range: get_key(0)..get_key(10), + lsn_range: Lsn(0x10)..Lsn(0x11), + is_delta: false, + }, + // The range gets compacted + PersistentLayerKey { + key_range: get_key(1)..get_key(2), + lsn_range: Lsn(0x20)..Lsn(0x50), + is_delta: true, + }, + // Not touched during this iteration of compaction + PersistentLayerKey { + key_range: get_key(8)..get_key(10), + lsn_range: Lsn(0x30)..Lsn(0x50), + is_delta: true, + }, + ], + ); + + // final full compaction + tline + .compact_with_gc(&cancel, CompactOptions::default(), &ctx) + .await + .unwrap(); + verify_result().await; + + let all_layers = inspect_and_sort(&tline, Some(get_key(0)..get_key(10))).await; + check_layer_map_key_eq( + all_layers, + vec![ + // The compacted image layer (full key range) + PersistentLayerKey { + key_range: Key::MIN..Key::MAX, + lsn_range: Lsn(0x10)..Lsn(0x11), + is_delta: false, + }, + // All other data in the delta layer + PersistentLayerKey { + key_range: get_key(1)..get_key(10), + lsn_range: Lsn(0x10)..Lsn(0x50), + is_delta: true, + }, + ], + ); + + Ok(()) + } } diff --git a/pageserver/src/tenant/timeline.rs b/pageserver/src/tenant/timeline.rs index 8f1d5f6577..b5c7079226 100644 --- a/pageserver/src/tenant/timeline.rs +++ b/pageserver/src/tenant/timeline.rs @@ -780,46 +780,90 @@ pub(crate) enum CompactFlags { #[serde_with::serde_as] #[derive(Debug, Clone, serde::Deserialize)] pub(crate) struct CompactRequest { - pub compact_range: Option, - pub compact_below_lsn: Option, + pub compact_key_range: Option, + pub compact_lsn_range: Option, /// Whether the compaction job should be scheduled. #[serde(default)] pub scheduled: bool, /// Whether the compaction job should be split across key ranges. #[serde(default)] pub sub_compaction: bool, + /// Max job size for each subcompaction job. + pub sub_compaction_max_job_size_mb: Option, } #[serde_with::serde_as] #[derive(Debug, Clone, serde::Deserialize)] -pub(crate) struct CompactRange { +pub(crate) struct CompactLsnRange { + pub start: Lsn, + pub end: Lsn, +} + +#[serde_with::serde_as] +#[derive(Debug, Clone, serde::Deserialize)] +pub(crate) struct CompactKeyRange { #[serde_as(as = "serde_with::DisplayFromStr")] pub start: Key, #[serde_as(as = "serde_with::DisplayFromStr")] pub end: Key, } -impl From> for CompactRange { - fn from(range: Range) -> Self { - CompactRange { +impl From> for CompactLsnRange { + fn from(range: Range) -> Self { + Self { start: range.start, end: range.end, } } } +impl From> for CompactKeyRange { + fn from(range: Range) -> Self { + Self { + start: range.start, + end: range.end, + } + } +} + +impl From for Range { + fn from(range: CompactLsnRange) -> Self { + range.start..range.end + } +} + +impl From for Range { + fn from(range: CompactKeyRange) -> Self { + range.start..range.end + } +} + +impl CompactLsnRange { + #[cfg(test)] + #[cfg(feature = "testing")] + pub fn above(lsn: Lsn) -> Self { + Self { + start: lsn, + end: Lsn::MAX, + } + } +} + #[derive(Debug, Clone, Default)] pub(crate) struct CompactOptions { pub flags: EnumSet, /// If set, the compaction will only compact the key range specified by this option. - /// This option is only used by GC compaction. - pub compact_range: Option, - /// If set, the compaction will only compact the LSN below this value. - /// This option is only used by GC compaction. - pub compact_below_lsn: Option, + /// This option is only used by GC compaction. For the full explanation, see [`compaction::GcCompactJob`]. + pub compact_key_range: Option, + /// If set, the compaction will only compact the LSN within this value. + /// This option is only used by GC compaction. For the full explanation, see [`compaction::GcCompactJob`]. + pub compact_lsn_range: Option, /// Enable sub-compaction (split compaction job across key ranges). /// This option is only used by GC compaction. pub sub_compaction: bool, + /// Set job size for the GC compaction. + /// This option is only used by GC compaction. + pub sub_compaction_max_job_size_mb: Option, } impl std::fmt::Debug for Timeline { @@ -1641,9 +1685,10 @@ impl Timeline { cancel, CompactOptions { flags, - compact_range: None, - compact_below_lsn: None, + compact_key_range: None, + compact_lsn_range: None, sub_compaction: false, + sub_compaction_max_job_size_mb: None, }, ctx, ) diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index fa924d23b0..701247194b 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -10,8 +10,8 @@ use std::sync::Arc; use super::layer_manager::LayerManager; use super::{ - CompactFlags, CompactOptions, CompactRange, CreateImageLayersError, DurationRecorder, - ImageLayerCreationMode, RecordedDuration, Timeline, + CompactFlags, CompactOptions, CreateImageLayersError, DurationRecorder, ImageLayerCreationMode, + RecordedDuration, Timeline, }; use anyhow::{anyhow, bail, Context}; @@ -64,6 +64,9 @@ const COMPACTION_DELTA_THRESHOLD: usize = 5; /// A scheduled compaction task. pub(crate) struct ScheduledCompactionTask { + /// It's unfortunate that we need to store a compact options struct here because the only outer + /// API we can call here is `compact_with_options` which does a few setup calls before starting the + /// actual compaction job... We should refactor this to store `GcCompactionJob` in the future. pub options: CompactOptions, /// The channel to send the compaction result. If this is a subcompaction, the last compaction job holds the sender. pub result_tx: Option>, @@ -71,16 +74,57 @@ pub(crate) struct ScheduledCompactionTask { pub gc_block: Option, } +/// A job description for the gc-compaction job. This structure describes the rectangle range that the job will +/// process. The exact layers that need to be compacted/rewritten will be generated when `compact_with_gc` gets +/// called. +#[derive(Debug, Clone)] +pub(crate) struct GcCompactJob { + pub dry_run: bool, + /// The key range to be compacted. The compaction algorithm will only regenerate key-value pairs within this range + /// [left inclusive, right exclusive), and other pairs will be rewritten into new files if necessary. + pub compact_key_range: Range, + /// The LSN range to be compacted. The compaction algorithm will use this range to determine the layers to be + /// selected for the compaction, and it does not guarantee the generated layers will have exactly the same LSN range + /// as specified here. The true range being compacted is `min_lsn/max_lsn` in [`GcCompactionJobDescription`]. + /// min_lsn will always <= the lower bound specified here, and max_lsn will always >= the upper bound specified here. + pub compact_lsn_range: Range, +} + +impl GcCompactJob { + pub fn from_compact_options(options: CompactOptions) -> Self { + GcCompactJob { + dry_run: options.flags.contains(CompactFlags::DryRun), + compact_key_range: options + .compact_key_range + .map(|x| x.into()) + .unwrap_or(Key::MIN..Key::MAX), + compact_lsn_range: options + .compact_lsn_range + .map(|x| x.into()) + .unwrap_or(Lsn::INVALID..Lsn::MAX), + } + } +} + +/// A job description for the gc-compaction job. This structure is generated when `compact_with_gc` is called +/// and contains the exact layers we want to compact. pub struct GcCompactionJobDescription { /// All layers to read in the compaction job selected_layers: Vec, - /// GC cutoff of the job + /// GC cutoff of the job. This is the lowest LSN that will be accessed by the read/GC path and we need to + /// keep all deltas <= this LSN or generate an image == this LSN. gc_cutoff: Lsn, - /// LSNs to retain for the job + /// LSNs to retain for the job. Read path will use this LSN so we need to keep deltas <= this LSN or + /// generate an image == this LSN. retain_lsns_below_horizon: Vec, - /// Maximum layer LSN processed in this compaction + /// Maximum layer LSN processed in this compaction, that is max(end_lsn of layers). Exclusive. All data + /// \>= this LSN will be kept and will not be rewritten. max_layer_lsn: Lsn, - /// Only compact layers overlapping with this range + /// Minimum layer LSN processed in this compaction, that is min(start_lsn of layers). Inclusive. + /// All access below (strict lower than `<`) this LSN will be routed through the normal read path instead of + /// k-merge within gc-compaction. + min_layer_lsn: Lsn, + /// Only compact layers overlapping with this range. compaction_key_range: Range, /// When partial compaction is enabled, these layers need to be rewritten to ensure no overlap. /// This field is here solely for debugging. The field will not be read once the compaction @@ -299,7 +343,7 @@ impl Timeline { ))); } - if options.compact_range.is_some() { + if options.compact_key_range.is_some() || options.compact_lsn_range.is_some() { // maybe useful in the future? could implement this at some point return Err(CompactionError::Other(anyhow!( "compaction range is not supported for legacy compaction for now" @@ -1754,25 +1798,26 @@ impl Timeline { Ok(()) } - /// Split a gc-compaction job into multiple compaction jobs. Optimally, this function should return a vector of - /// `GcCompactionJobDesc`. But we want to keep it simple on the tenant scheduling side without exposing too much - /// ad-hoc information about gc compaction itself. + /// Split a gc-compaction job into multiple compaction jobs. The split is based on the key range and the estimated size of the compaction job. + /// The function returns a list of compaction jobs that can be executed separately. If the upper bound of the compact LSN + /// range is not specified, we will use the latest gc_cutoff as the upper bound, so that all jobs in the jobset acts + /// like a full compaction of the specified keyspace. pub(crate) async fn gc_compaction_split_jobs( self: &Arc, - options: CompactOptions, - ) -> anyhow::Result> { - if !options.sub_compaction { - return Ok(vec![options]); - } - let compact_range = options.compact_range.clone().unwrap_or(CompactRange { - start: Key::MIN, - end: Key::MAX, - }); - let compact_below_lsn = if let Some(compact_below_lsn) = options.compact_below_lsn { - compact_below_lsn + job: GcCompactJob, + sub_compaction_max_job_size_mb: Option, + ) -> anyhow::Result> { + let compact_below_lsn = if job.compact_lsn_range.end != Lsn::MAX { + job.compact_lsn_range.end } else { *self.get_latest_gc_cutoff_lsn() // use the real gc cutoff }; + + // Split compaction job to about 4GB each + const GC_COMPACT_MAX_SIZE_MB: u64 = 4 * 1024; + let sub_compaction_max_job_size_mb = + sub_compaction_max_job_size_mb.unwrap_or(GC_COMPACT_MAX_SIZE_MB); + let mut compact_jobs = Vec::new(); // For now, we simply use the key partitioning information; we should do a more fine-grained partitioning // by estimating the amount of files read for a compaction job. We should also partition on LSN. @@ -1808,8 +1853,8 @@ impl Timeline { let Some((start, end)) = truncate_to( &range.start, &range.end, - &compact_range.start, - &compact_range.end, + &job.compact_key_range.start, + &job.compact_key_range.end, ) else { continue; }; @@ -1819,8 +1864,6 @@ impl Timeline { let guard = self.layers.read().await; let layer_map = guard.layer_map()?; let mut current_start = None; - // Split compaction job to about 2GB each - const GC_COMPACT_MAX_SIZE_MB: u64 = 4 * 1024; // 4GB, TODO: should be configuration in the future let ranges_num = split_key_ranges.len(); for (idx, (start, end)) in split_key_ranges.into_iter().enumerate() { if current_start.is_none() { @@ -1833,8 +1876,7 @@ impl Timeline { } let res = layer_map.range_search(start..end, compact_below_lsn); let total_size = res.found.keys().map(|x| x.layer.file_size()).sum::(); - if total_size > GC_COMPACT_MAX_SIZE_MB * 1024 * 1024 || ranges_num == idx + 1 { - let mut compact_options = options.clone(); + if total_size > sub_compaction_max_job_size_mb * 1024 * 1024 || ranges_num == idx + 1 { // Try to extend the compaction range so that we include at least one full layer file. let extended_end = res .found @@ -1852,10 +1894,11 @@ impl Timeline { "splitting compaction job: {}..{}, estimated_size={}", start, end, total_size ); - compact_options.compact_range = Some(CompactRange { start, end }); - compact_options.compact_below_lsn = Some(compact_below_lsn); - compact_options.sub_compaction = false; - compact_jobs.push(compact_options); + compact_jobs.push(GcCompactJob { + dry_run: job.dry_run, + compact_key_range: start..end, + compact_lsn_range: job.compact_lsn_range.start..compact_below_lsn, + }); current_start = Some(end); } } @@ -1877,7 +1920,7 @@ impl Timeline { /// Key::MIN..Key..MAX to the function indicates a full compaction, though technically, `Key::MAX` is not /// part of the range. /// - /// If `options.compact_below_lsn` is provided, the compaction will only compact layers below or intersect with + /// If `options.compact_lsn_range.end` is provided, the compaction will only compact layers below or intersect with /// the LSN. Otherwise, it will use the gc cutoff by default. pub(crate) async fn compact_with_gc( self: &Arc, @@ -1885,9 +1928,13 @@ impl Timeline { options: CompactOptions, ctx: &RequestContext, ) -> anyhow::Result<()> { - if options.sub_compaction { + let sub_compaction = options.sub_compaction; + let job = GcCompactJob::from_compact_options(options.clone()); + if sub_compaction { info!("running enhanced gc bottom-most compaction with sub-compaction, splitting compaction jobs"); - let jobs = self.gc_compaction_split_jobs(options).await?; + let jobs = self + .gc_compaction_split_jobs(job, options.sub_compaction_max_job_size_mb) + .await?; let jobs_len = jobs.len(); for (idx, job) in jobs.into_iter().enumerate() { info!( @@ -1902,19 +1949,15 @@ impl Timeline { } return Ok(()); } - self.compact_with_gc_inner(cancel, options, ctx).await + self.compact_with_gc_inner(cancel, job, ctx).await } async fn compact_with_gc_inner( self: &Arc, cancel: &CancellationToken, - options: CompactOptions, + job: GcCompactJob, ctx: &RequestContext, ) -> anyhow::Result<()> { - assert!( - !options.sub_compaction, - "sub-compaction should be handled by the outer function" - ); // Block other compaction/GC tasks from running for now. GC-compaction could run along // with legacy compaction tasks in the future. Always ensure the lock order is compaction -> gc. // Note that we already acquired the compaction lock when the outer `compact` function gets called. @@ -1934,19 +1977,11 @@ impl Timeline { ) .await?; - let flags = options.flags; - let compaction_key_range = options - .compact_range - .map(|range| range.start..range.end) - .unwrap_or_else(|| Key::MIN..Key::MAX); + let dry_run = job.dry_run; + let compact_key_range = job.compact_key_range; + let compact_lsn_range = job.compact_lsn_range; - let dry_run = flags.contains(CompactFlags::DryRun); - - if compaction_key_range == (Key::MIN..Key::MAX) { - info!("running enhanced gc bottom-most compaction, dry_run={dry_run}, compaction_key_range={}..{}", compaction_key_range.start, compaction_key_range.end); - } else { - info!("running enhanced gc bottom-most compaction, dry_run={dry_run}"); - } + info!("running enhanced gc bottom-most compaction, dry_run={dry_run}, compact_key_range={}..{}, compact_lsn_range={}..{}", compact_key_range.start, compact_key_range.end, compact_lsn_range.start, compact_lsn_range.end); scopeguard::defer! { info!("done enhanced gc bottom-most compaction"); @@ -1970,11 +2005,15 @@ impl Timeline { // to get the truth data. let real_gc_cutoff = *self.get_latest_gc_cutoff_lsn(); // The compaction algorithm will keep all keys above the gc_cutoff while keeping only necessary keys below the gc_cutoff for - // each of the retain_lsn. Therefore, if the user-provided `compact_below_lsn` is larger than the real gc cutoff, we will use + // each of the retain_lsn. Therefore, if the user-provided `compact_lsn_range.end` is larger than the real gc cutoff, we will use // the real cutoff. - let mut gc_cutoff = options.compact_below_lsn.unwrap_or(real_gc_cutoff); + let mut gc_cutoff = if compact_lsn_range.end == Lsn::MAX { + real_gc_cutoff + } else { + compact_lsn_range.end + }; if gc_cutoff > real_gc_cutoff { - warn!("provided compact_below_lsn={} is larger than the real_gc_cutoff={}, using the real gc cutoff", gc_cutoff, real_gc_cutoff); + warn!("provided compact_lsn_range.end={} is larger than the real_gc_cutoff={}, using the real gc cutoff", gc_cutoff, real_gc_cutoff); gc_cutoff = real_gc_cutoff; } gc_cutoff @@ -1991,7 +2030,7 @@ impl Timeline { } let mut selected_layers: Vec = Vec::new(); drop(gc_info); - // Pick all the layers intersect or below the gc_cutoff, get the largest LSN in the selected layers. + // Firstly, pick all the layers intersect or below the gc_cutoff, get the largest LSN in the selected layers. let Some(max_layer_lsn) = layers .iter_historic_layers() .filter(|desc| desc.get_lsn_range().start <= gc_cutoff) @@ -2001,27 +2040,45 @@ impl Timeline { info!("no layers to compact with gc: no historic layers below gc_cutoff, gc_cutoff={}", gc_cutoff); return Ok(()); }; + // Next, if the user specifies compact_lsn_range.start, we need to filter some layers out. All the layers (strictly) below + // the min_layer_lsn computed as below will be filtered out and the data will be accessed using the normal read path, as if + // it is a branch. + let Some(min_layer_lsn) = layers + .iter_historic_layers() + .filter(|desc| { + if compact_lsn_range.start == Lsn::INVALID { + true // select all layers below if start == Lsn(0) + } else { + desc.get_lsn_range().end > compact_lsn_range.start // strictly larger than compact_above_lsn + } + }) + .map(|desc| desc.get_lsn_range().start) + .min() + else { + info!("no layers to compact with gc: no historic layers above compact_above_lsn, compact_above_lsn={}", compact_lsn_range.end); + return Ok(()); + }; // Then, pick all the layers that are below the max_layer_lsn. This is to ensure we can pick all single-key // layers to compact. let mut rewrite_layers = Vec::new(); for desc in layers.iter_historic_layers() { if desc.get_lsn_range().end <= max_layer_lsn - && overlaps_with(&desc.get_key_range(), &compaction_key_range) + && desc.get_lsn_range().start >= min_layer_lsn + && overlaps_with(&desc.get_key_range(), &compact_key_range) { // If the layer overlaps with the compaction key range, we need to read it to obtain all keys within the range, // even if it might contain extra keys selected_layers.push(guard.get_from_desc(&desc)); // If the layer is not fully contained within the key range, we need to rewrite it if it's a delta layer (it's fine // to overlap image layers) - if desc.is_delta() - && !fully_contains(&compaction_key_range, &desc.get_key_range()) + if desc.is_delta() && !fully_contains(&compact_key_range, &desc.get_key_range()) { rewrite_layers.push(desc); } } } if selected_layers.is_empty() { - info!("no layers to compact with gc: no layers within the key range, gc_cutoff={}, key_range={}..{}", gc_cutoff, compaction_key_range.start, compaction_key_range.end); + info!("no layers to compact with gc: no layers within the key range, gc_cutoff={}, key_range={}..{}", gc_cutoff, compact_key_range.start, compact_key_range.end); return Ok(()); } retain_lsns_below_horizon.sort(); @@ -2029,13 +2086,20 @@ impl Timeline { selected_layers, gc_cutoff, retain_lsns_below_horizon, + min_layer_lsn, max_layer_lsn, - compaction_key_range, + compaction_key_range: compact_key_range, rewrite_layers, } }; - let lowest_retain_lsn = if self.ancestor_timeline.is_some() { - Lsn(self.ancestor_lsn.0 + 1) + let (has_data_below, lowest_retain_lsn) = if compact_lsn_range.start != Lsn::INVALID { + // If we only compact above some LSN, we should get the history from the current branch below the specified LSN. + // We use job_desc.min_layer_lsn as if it's the lowest branch point. + (true, job_desc.min_layer_lsn) + } else if self.ancestor_timeline.is_some() { + // In theory, we can also use min_layer_lsn here, but using ancestor LSN makes sure the delta layers cover the + // LSN ranges all the way to the ancestor timeline. + (true, self.ancestor_lsn) } else { let res = job_desc .retain_lsns_below_horizon @@ -2053,17 +2117,19 @@ impl Timeline { .unwrap_or(job_desc.gc_cutoff) ); } - res + (false, res) }; info!( - "picked {} layers for compaction ({} layers need rewriting) with max_layer_lsn={} gc_cutoff={} lowest_retain_lsn={}, key_range={}..{}", + "picked {} layers for compaction ({} layers need rewriting) with max_layer_lsn={} min_layer_lsn={} gc_cutoff={} lowest_retain_lsn={}, key_range={}..{}, has_data_below={}", job_desc.selected_layers.len(), job_desc.rewrite_layers.len(), job_desc.max_layer_lsn, + job_desc.min_layer_lsn, job_desc.gc_cutoff, lowest_retain_lsn, job_desc.compaction_key_range.start, - job_desc.compaction_key_range.end + job_desc.compaction_key_range.end, + has_data_below, ); for layer in &job_desc.selected_layers { @@ -2107,10 +2173,22 @@ impl Timeline { let mut delta_layers = Vec::new(); let mut image_layers = Vec::new(); let mut downloaded_layers = Vec::new(); + let mut total_downloaded_size = 0; + let mut total_layer_size = 0; for layer in &job_desc.selected_layers { + if layer.needs_download().await?.is_some() { + total_downloaded_size += layer.layer_desc().file_size; + } + total_layer_size += layer.layer_desc().file_size; let resident_layer = layer.download_and_keep_resident().await?; downloaded_layers.push(resident_layer); } + info!( + "finish downloading layers, downloaded={}, total={}, ratio={:.2}", + total_downloaded_size, + total_layer_size, + total_downloaded_size as f64 / total_layer_size as f64 + ); for resident_layer in &downloaded_layers { if resident_layer.layer_desc().is_delta() { let layer = resident_layer.get_as_delta(ctx).await?; @@ -2133,7 +2211,7 @@ impl Timeline { // Only create image layers when there is no ancestor branches. TODO: create covering image layer // when some condition meet. - let mut image_layer_writer = if self.ancestor_timeline.is_none() { + let mut image_layer_writer = if !has_data_below { Some( SplitImageLayerWriter::new( self.conf, @@ -2166,7 +2244,11 @@ impl Timeline { } let mut delta_layer_rewriters = HashMap::, RewritingLayers>::new(); - /// Returns None if there is no ancestor branch. Throw an error when the key is not found. + /// When compacting not at a bottom range (=`[0,X)`) of the root branch, we "have data below" (`has_data_below=true`). + /// The two cases are compaction in ancestor branches and when `compact_lsn_range.start` is set. + /// In those cases, we need to pull up data from below the LSN range we're compaction. + /// + /// This function unifies the cases so that later code doesn't have to think about it. /// /// Currently, we always get the ancestor image for each key in the child branch no matter whether the image /// is needed for reconstruction. This should be fixed in the future. @@ -2174,17 +2256,19 @@ impl Timeline { /// Furthermore, we should do vectored get instead of a single get, or better, use k-merge for ancestor /// images. async fn get_ancestor_image( - tline: &Arc, + this_tline: &Arc, key: Key, ctx: &RequestContext, + has_data_below: bool, + history_lsn_point: Lsn, ) -> anyhow::Result> { - if tline.ancestor_timeline.is_none() { + if !has_data_below { return Ok(None); }; // This function is implemented as a get of the current timeline at ancestor LSN, therefore reusing // as much existing code as possible. - let img = tline.get(key, tline.ancestor_lsn, ctx).await?; - Ok(Some((key, tline.ancestor_lsn, img))) + let img = this_tline.get(key, history_lsn_point, ctx).await?; + Ok(Some((key, history_lsn_point, img))) } // Actually, we can decide not to write to the image layer at all at this point because @@ -2268,7 +2352,8 @@ impl Timeline { job_desc.gc_cutoff, &job_desc.retain_lsns_below_horizon, COMPACTION_DELTA_THRESHOLD, - get_ancestor_image(self, *last_key, ctx).await?, + get_ancestor_image(self, *last_key, ctx, has_data_below, lowest_retain_lsn) + .await?, ) .await?; retention @@ -2297,7 +2382,7 @@ impl Timeline { job_desc.gc_cutoff, &job_desc.retain_lsns_below_horizon, COMPACTION_DELTA_THRESHOLD, - get_ancestor_image(self, last_key, ctx).await?, + get_ancestor_image(self, last_key, ctx, has_data_below, lowest_retain_lsn).await?, ) .await?; retention diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index 810a9723e0..88873c63c2 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -153,6 +153,7 @@ def test_pageserver_gc_compaction_smoke(neon_env_builder: NeonEnvBuilder): if i % 10 == 0: log.info(f"Running churn round {i}/{churn_rounds} ...") + if (i - 1) % 10 == 0: # Run gc-compaction every 10 rounds to ensure the test doesn't take too long time. ps_http.timeline_compact( tenant_id, @@ -161,10 +162,11 @@ def test_pageserver_gc_compaction_smoke(neon_env_builder: NeonEnvBuilder): body={ "scheduled": True, "sub_compaction": True, - "compact_range": { + "compact_key_range": { "start": "000000000000000000000000000000000000", "end": "030000000000000000000000000000000000", }, + "sub_compaction_max_job_size_mb": 16, }, ) From 2f3f98a3190a787b8420b8e969a7c5f6930de0b7 Mon Sep 17 00:00:00 2001 From: Peter Bendel Date: Thu, 12 Dec 2024 21:25:39 +0100 Subject: [PATCH 41/44] use OIDC role instead of AWS access keys for managing test runner (#10117) in periodic pagebench workflow ## Problem for background see https://github.com/neondatabase/cloud/issues/21545 ## Summary of changes use OIDC role to manage runners instead of AWS access key which needs to be periodically rotated ## logs seems to work in https://github.com/neondatabase/neon/actions/runs/12298575888/job/34322306127#step:6:1 --- .github/actionlint.yml | 1 + .github/workflows/periodic_pagebench.yml | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/actionlint.yml b/.github/actionlint.yml index 9d8389faa5..7a97e2ae55 100644 --- a/.github/actionlint.yml +++ b/.github/actionlint.yml @@ -24,3 +24,4 @@ config-variables: - PGREGRESS_PG16_PROJECT_ID - PGREGRESS_PG17_PROJECT_ID - SLACK_ON_CALL_QA_STAGING_STREAM + - DEV_AWS_OIDC_ROLE_MANAGE_BENCHMARK_EC2_VMS_ARN diff --git a/.github/workflows/periodic_pagebench.yml b/.github/workflows/periodic_pagebench.yml index a04ceb4a24..9f5a16feca 100644 --- a/.github/workflows/periodic_pagebench.yml +++ b/.github/workflows/periodic_pagebench.yml @@ -41,8 +41,6 @@ jobs: env: API_KEY: ${{ secrets.PERIODIC_PAGEBENCH_EC2_RUNNER_API_KEY }} RUN_ID: ${{ github.run_id }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_EC2_US_TEST_RUNNER_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY : ${{ secrets.AWS_EC2_US_TEST_RUNNER_ACCESS_KEY_SECRET }} AWS_DEFAULT_REGION : "eu-central-1" AWS_INSTANCE_ID : "i-02a59a3bf86bc7e74" steps: @@ -53,6 +51,13 @@ jobs: - name: Show my own (github runner) external IP address - usefull for IP allowlisting run: curl https://ifconfig.me + - name: Assume AWS OIDC role that allows to manage (start/stop/describe... EC machine) + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_MANAGE_BENCHMARK_EC2_VMS_ARN }} + role-duration-seconds: 3600 + - name: Start EC2 instance and wait for the instance to boot up run: | aws ec2 start-instances --instance-ids $AWS_INSTANCE_ID @@ -150,6 +155,14 @@ jobs: -H "Authorization: Bearer $API_KEY" \ -d '' + - name: Assume AWS OIDC role that allows to manage (start/stop/describe... EC machine) + if: always() && steps.poll_step.outputs.too_many_runs != 'true' + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: eu-central-1 + role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_MANAGE_BENCHMARK_EC2_VMS_ARN }} + role-duration-seconds: 3600 + - name: Stop EC2 instance and wait for the instance to be stopped if: always() && steps.poll_step.outputs.too_many_runs != 'true' run: | From ac04bad45751bd60f19e6ac51935f7de01e6b87d Mon Sep 17 00:00:00 2001 From: Alexander Bayandin Date: Thu, 12 Dec 2024 22:55:38 +0000 Subject: [PATCH 42/44] CI: don't run debug builds with LFC (#10123) ## Problem I've noticed that debug builds with LFC fail more frequently and for some reason ,their failure do block merging (but it should not) ## Summary of changes - Do not run Debug builds with LFC --- .github/workflows/build_and_test.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 4bfb5077c6..1bae23f7ef 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -254,16 +254,14 @@ jobs: build-tag: ${{ needs.tag.outputs.build-tag }} build-type: ${{ matrix.build-type }} # Run tests on all Postgres versions in release builds and only on the latest version in debug builds. - # Run without LFC on v17 release and debug builds only. For all the other cases LFC is enabled. Failure on the - # debug build with LFC enabled doesn't block merging. + # Run without LFC on v17 release and debug builds only. For all the other cases LFC is enabled. test-cfg: | ${{ matrix.build-type == 'release' && '[{"pg_version":"v14", "lfc_state": "with-lfc"}, {"pg_version":"v15", "lfc_state": "with-lfc"}, {"pg_version":"v16", "lfc_state": "with-lfc"}, {"pg_version":"v17", "lfc_state": "with-lfc"}, {"pg_version":"v17", "lfc_state": "without-lfc"}]' - || '[{"pg_version":"v17", "lfc_state": "without-lfc"}, - {"pg_version":"v17", "lfc_state": "with-lfc" }]' }} + || '[{"pg_version":"v17", "lfc_state": "without-lfc" }]' }} secrets: inherit # Keep `benchmarks` job outside of `build-and-test-locally` workflow to make job failures non-blocking From 59ef70192591c0fc2a5c2a69eea0d47f7bd8e033 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JC=20Gr=C3=BCnhage?= Date: Fri, 13 Dec 2024 00:38:20 +0100 Subject: [PATCH 43/44] CI(deploy): fix git tag/release creation (#10119) ## Problem When moving the comment on proxy-releases from the yaml doc into a javascript code block, I missed converting the comment marker from `#` to `//`. ## Summary of changes Correctly convert comment marker. --- .github/workflows/build_and_test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 1bae23f7ef..a3943cba91 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -1143,7 +1143,7 @@ jobs: console.log(`Tag ${tag} created successfully.`); } - # TODO: check how GitHub releases looks for proxy/compute releases and enable them if they're ok + // TODO: check how GitHub releases looks for proxy/compute releases and enable them if they're ok if (context.ref !== 'refs/heads/release') { console.log(`GitHub release skipped for ${context.ref}.`); return; From aeb79d1bb6da18448e31b56607a75a41e0a71c86 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 13 Dec 2024 06:02:24 +0000 Subject: [PATCH 44/44] Storage release 2024-12-13