diff --git a/.github/actions/neon-project-create/action.yml b/.github/actions/neon-project-create/action.yml index 02151b2e61..aa58876aa5 100644 --- a/.github/actions/neon-project-create/action.yml +++ b/.github/actions/neon-project-create/action.yml @@ -141,6 +141,8 @@ runs: fi # XXX # This is a workaround for project's settings which don't work well in public API now + # https://github.com/neondatabase/cloud/issues/27143 + # https://github.com/neondatabase/cloud/issues/27108 if ( [[ -n "${PROJECT_SETTINGS}" ]] && [[ "${PROJECT_SETTINGS}" != "{}" ]] ) || ( [[ -n "${DEFAULT_ENDPOINT_SETTINGS}" ]] && [[ "${DEFAULT_ENDPOINT_SETTINGS}" != "{}" ]] ); then PROJECT_DATA=$(curl -X GET \ "https://${API_HOST}/regions/${REGION_ID}/api/v1/admin/projects/${project_id}" \ diff --git a/.github/workflows/build-macos.yml b/.github/workflows/build-macos.yml index 148c1ef5af..ecd135cc3d 100644 --- a/.github/workflows/build-macos.yml +++ b/.github/workflows/build-macos.yml @@ -63,13 +63,8 @@ jobs: - name: Cache postgres ${{ matrix.postgres-version }} build id: cache_pg - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/${{ matrix.postgres-version }} key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ matrix.postgres-version }}-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} @@ -134,25 +129,15 @@ jobs: - name: Cache postgres v17 build id: cache_pg - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/v17 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache walproposer-lib id: cache_walproposer_lib - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/build/walproposer-lib key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }} @@ -218,57 +203,32 @@ jobs: - name: Cache postgres v14 build id: cache_pg - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/v14 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v14-${{ steps.pg_rev_v14.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache postgres v15 build id: cache_pg_v15 - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/v15 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v15-${{ steps.pg_rev_v15.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache postgres v16 build id: cache_pg_v16 - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/v16 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v16-${{ steps.pg_rev_v16.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache postgres v17 build id: cache_pg_v17 - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/v17 key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }} - name: Cache cargo deps (only for v17) - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: | ~/.cargo/registry !~/.cargo/registry/src @@ -278,13 +238,8 @@ jobs: - name: Cache walproposer-lib id: cache_walproposer_lib - uses: tespkg/actions-cache@b7bf5fcc2f98a52ac6080eb0fd282c2f752074b1 # v1.8.0 + uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 with: - endpoint: ${{ vars.HETZNER_CACHE_REGION }}.${{ vars.HETZNER_CACHE_ENDPOINT }} - bucket: ${{ vars.HETZNER_CACHE_BUCKET }} - accessKey: ${{ secrets.HETZNER_CACHE_ACCESS_KEY }} - secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }} - use-fallback: false path: pg_install/build/walproposer-lib key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev_v17.outputs.pg_rev }}-${{ hashFiles('Makefile') }} diff --git a/.github/workflows/cloud-extensions.yml b/.github/workflows/cloud-extensions.yml index 8d6427591c..7d60469f92 100644 --- a/.github/workflows/cloud-extensions.yml +++ b/.github/workflows/cloud-extensions.yml @@ -35,8 +35,9 @@ jobs: matrix: pg-version: [16, 17] - runs-on: small + runs-on: [ self-hosted, small ] container: + # We use the neon-test-extensions image here as it contains the source code for the extensions. image: ghcr.io/neondatabase/neon-test-extensions-v${{ matrix.pg-version }}:latest credentials: username: ${{ github.actor }} @@ -63,7 +64,6 @@ jobs: settings=$(jq -c -n --arg libs $LIBS '{preload_libraries:{use_defaults:false,enabled_libraries:($libs| split(":"))}}') echo settings=$settings >> $GITHUB_OUTPUT - - name: Create Neon Project id: create-neon-project uses: ./.github/actions/neon-project-create @@ -71,7 +71,18 @@ jobs: region_id: ${{ inputs.region_id }} postgres_version: ${{ matrix.pg-version }} project_settings: ${{ steps.project-settings.outputs.settings }} - default_endpoint_settings: '{"pg_settings":{"DateStyle":"Postgres,MDY","TimeZone":"America/Los_Angeles","compute_query_id":"off","max_worker_processes":"10","neon.allow_unstable_extensions":"on"}}' + # We need these settings to get the expected output results. + # We cannot use the environment variables e.g. PGTZ due to + # https://github.com/neondatabase/neon/issues/1287 + default_endpoint_settings: > + { + "pg_settings": { + "DateStyle": "Postgres,MDY", + "TimeZone": "America/Los_Angeles", + "compute_query_id": "off", + "neon.allow_unstable_extensions": "on" + } + } api_key: ${{ secrets.NEON_STAGING_API_KEY }} admin_api_key: ${{ secrets.NEON_STAGING_ADMIN_API_KEY }} diff --git a/Cargo.lock b/Cargo.lock index 4573629964..2cf260c88c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1323,7 +1323,6 @@ dependencies = [ "serde_json", "serde_with", "signal-hook", - "spki 0.7.3", "tar", "thiserror 1.0.69", "tokio", diff --git a/compute_tools/Cargo.toml b/compute_tools/Cargo.toml index d80ec41d34..8c1e7ad149 100644 --- a/compute_tools/Cargo.toml +++ b/compute_tools/Cargo.toml @@ -44,7 +44,6 @@ serde.workspace = true serde_with.workspace = true serde_json.workspace = true signal-hook.workspace = true -spki = { version = "0.7.3", features = ["std"] } tar.workspace = true tower.workspace = true tower-http.workspace = true diff --git a/compute_tools/src/tls.rs b/compute_tools/src/tls.rs index 8f465c7300..ab32a9459a 100644 --- a/compute_tools/src/tls.rs +++ b/compute_tools/src/tls.rs @@ -3,7 +3,6 @@ use std::{io::Write, os::unix::fs::OpenOptionsExt, path::Path, time::Duration}; use anyhow::{Context, Result, bail}; use compute_api::responses::TlsConfig; use ring::digest; -use spki::der::{Decode, PemReader}; use x509_cert::Certificate; #[derive(Clone, Copy)] @@ -52,7 +51,7 @@ pub fn update_key_path_blocking(pg_data: &Path, tls_config: &TlsConfig) { match try_update_key_path_blocking(pg_data, tls_config) { Ok(()) => break, Err(e) => { - tracing::error!("could not create key file {e:?}"); + tracing::error!(error = ?e, "could not create key file"); std::thread::sleep(Duration::from_secs(1)) } } @@ -92,8 +91,14 @@ fn try_update_key_path_blocking(pg_data: &Path, tls_config: &TlsConfig) -> Resul fn verify_key_cert(key: &str, cert: &str) -> Result<()> { use x509_cert::der::oid::db::rfc5912::ECDSA_WITH_SHA_256; - let cert = Certificate::decode(&mut PemReader::new(cert.as_bytes()).context("pem reader")?) - .context("decode cert")?; + let certs = Certificate::load_pem_chain(cert.as_bytes()) + .context("decoding PEM encoded certificates")?; + + // First certificate is our server-cert, + // all the rest of the certs are the CA cert chain. + let Some(cert) = certs.first() else { + bail!("no certificates found"); + }; match cert.signature_algorithm.oid { ECDSA_WITH_SHA_256 => { @@ -115,3 +120,82 @@ fn verify_key_cert(key: &str, cert: &str) -> Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::verify_key_cert; + + /// Real certificate chain file, generated by cert-manager in dev. + /// The server auth certificate has expired since 2025-04-24T15:41:35Z. + const CERT: &str = " +-----BEGIN CERTIFICATE----- +MIICCDCCAa+gAwIBAgIQKhLomFcNULbZA/bPdGzaSzAKBggqhkjOPQQDAjBEMQsw +CQYDVQQGEwJVUzESMBAGA1UEChMJTmVvbiBJbmMuMSEwHwYDVQQDExhOZW9uIEs4 +cyBJbnRlcm1lZGlhdGUgQ0EwHhcNMjUwNDIzMTU0MTM1WhcNMjUwNDI0MTU0MTM1 +WjBBMT8wPQYDVQQDEzZjb21wdXRlLXdpc3B5LWdyYXNzLXcwY21laWp3LmRlZmF1 +bHQuc3ZjLmNsdXN0ZXIubG9jYWwwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATF +QCcG2m/EVHAiZtSsYgVnHgoTjUL/Jtwfdrpvz2t0bVRZmBmSKhlo53uPV9Y5eKFG +AmR54p9/gT2eO3xU7vAgo4GFMIGCMA4GA1UdDwEB/wQEAwIFoDAMBgNVHRMBAf8E +AjAAMB8GA1UdIwQYMBaAFFR2JAhXkeiNQNEixTvAYIwxUu3QMEEGA1UdEQQ6MDiC +NmNvbXB1dGUtd2lzcHktZ3Jhc3MtdzBjbWVpancuZGVmYXVsdC5zdmMuY2x1c3Rl +ci5sb2NhbDAKBggqhkjOPQQDAgNHADBEAiBLG22wKG8XS9e9RxBT+kmUx/kIThcP +DIpp7jx0PrFcdQIgEMTdnXpx5Cv/Z0NIEDxtMHUD7G0vuRPfztki36JuakM= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICFzCCAb6gAwIBAgIUbbX98N2Ip6lWAONRk8dU9hSz+YIwCgYIKoZIzj0EAwIw +RDELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCU5lb24gSW5jLjEhMB8GA1UEAxMYTmVv +biBBV1MgSW50ZXJtZWRpYXRlIENBMB4XDTI1MDQyMjE1MTAxMFoXDTI1MDcyMTE1 +MTAxMFowRDELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCU5lb24gSW5jLjEhMB8GA1UE +AxMYTmVvbiBLOHMgSW50ZXJtZWRpYXRlIENBMFkwEwYHKoZIzj0CAQYIKoZIzj0D +AQcDQgAE5++m5owqNI4BPMTVNIUQH0qvU7pYhdpHGVGhdj/Lgars6ROvE6uSNQV4 +SAmJN5HBzj5/6kLQaTPWpXW7EHXjK6OBjTCBijAOBgNVHQ8BAf8EBAMCAQYwEgYD +VR0TAQH/BAgwBgEB/wIBADAdBgNVHQ4EFgQUVHYkCFeR6I1A0SLFO8BgjDFS7dAw +HwYDVR0jBBgwFoAUgHfNXfyKtHO0V9qoLOWCjkNiaI8wJAYDVR0eAQH/BBowGKAW +MBSCEi5zdmMuY2x1c3Rlci5sb2NhbDAKBggqhkjOPQQDAgNHADBEAiBObVFFdXaL +QpOXmN60dYUNnQRwjKreFduEkQgOdOlssgIgVAdJJQFgvlrvEOBhY8j5WyeKRwUN +k/ALs6KpgaFBCGY= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIB4jCCAYegAwIBAgIUFlxWFn/11yoGdmD+6gf+yQMToS0wCgYIKoZIzj0EAwIw +ODELMAkGA1UEBhMCVVMxEjAQBgNVBAoTCU5lb24gSW5jLjEVMBMGA1UEAxMMTmVv +biBSb290IENBMB4XDTI1MDQwMzA3MTUyMloXDTI2MDQwMzA3MTUyMlowRDELMAkG +A1UEBhMCVVMxEjAQBgNVBAoTCU5lb24gSW5jLjEhMB8GA1UEAxMYTmVvbiBBV1Mg +SW50ZXJtZWRpYXRlIENBMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEqonG/IQ6 +ZxtEtOUTkkoNopPieXDO5CBKUkNFTGeJEB7OxRlSpYJgsBpaYIaD6Vc4sVk3thIF +p+pLw52idQOIN6NjMGEwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8w +HQYDVR0OBBYEFIB3zV38irRztFfaqCzlgo5DYmiPMB8GA1UdIwQYMBaAFKh7M4/G +FHvr/ORDQZt4bMLlJvHCMAoGCCqGSM49BAMCA0kAMEYCIQCbS4x7QPslONzBYbjC +UQaQ0QLDW4CJHvQ4u4gbWFG87wIhAJMsHQHjP9qTT27Q65zQCR7O8QeLAfha1jrH +Ag/LsxSr +-----END CERTIFICATE----- +"; + + /// The key corresponding to [`CERT`] + const KEY: &str = " +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIDnAnrqmIJjndCLWP1iIO5X3X63Aia48TGpGuMXwvm6IoAoGCCqGSM49 +AwEHoUQDQgAExUAnBtpvxFRwImbUrGIFZx4KE41C/ybcH3a6b89rdG1UWZgZkioZ +aOd7j1fWOXihRgJkeeKff4E9njt8VO7wIA== +-----END EC PRIVATE KEY----- +"; + + /// An incorrect key. + const INCORRECT_KEY: &str = " +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIL6WqqBDyvM0HWz7Ir5M5+jhFWB7IzOClGn26OPrzHCXoAoGCCqGSM49 +AwEHoUQDQgAE7XVvdOy5lfwtNKb+gJEUtnG+DrnnXLY5LsHDeGQKV9PTRcEMeCrG +YZzHyML4P6Sr4yi2ts+4B9i47uvAG8+XwQ== +-----END EC PRIVATE KEY----- +"; + + #[test] + fn certificate_verification() { + verify_key_cert(KEY, CERT).unwrap(); + } + + #[test] + #[should_panic(expected = "private key file does not match certificate")] + fn certificate_verification_fail() { + verify_key_cert(INCORRECT_KEY, CERT).unwrap(); + } +} diff --git a/docker-compose/docker_compose_test.sh b/docker-compose/docker_compose_test.sh index ce031ca3f3..6edf90ca8d 100755 --- a/docker-compose/docker_compose_test.sh +++ b/docker-compose/docker_compose_test.sh @@ -74,7 +74,7 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do docker compose cp "${TMPDIR}/data" compute:/postgres/contrib/file_fdw/data rm -rf "${TMPDIR}" # Apply patches - docker compose exec -i neon-test-extensions bash -c "(cd /postgres && patch -p1)" <"../compute/patches/contrib_pg${pg_version}.patch" + docker compose exec -T neon-test-extensions bash -c "(cd /postgres && patch -p1)" <"../compute/patches/contrib_pg${pg_version}.patch" # We are running tests now rm -f testout.txt testout_contrib.txt docker compose exec -e USE_PGXS=1 -e SKIP=timescaledb-src,rdkit-src,pg_jsonschema-src,kq_imcx-src,wal2json_2_5-src,rag_jina_reranker_v1_tiny_en-src,rag_bge_small_en_v15-src \ diff --git a/docker-compose/ext-src/README.md b/docker-compose/ext-src/README.md new file mode 100644 index 0000000000..cea0fe7053 --- /dev/null +++ b/docker-compose/ext-src/README.md @@ -0,0 +1,99 @@ +# PostgreSQL Extensions for Testing + +This directory contains PostgreSQL extensions used primarily for: +1. Testing extension upgrades between different Compute versions +2. Running regression tests with regular users (mostly for cloud instances) + +## Directory Structure + +Each extension directory follows a standard structure: + +- `extension-name-src/` - Directory containing test files for the extension + - `test-upgrade.sh` - Script for testing upgrade scenarios + - `regular-test.sh` - Script for testing with regular users + - Additional test files depending on the extension + +## Available Extensions + +This directory includes the following extensions: + +- `hll-src` - HyperLogLog, a fixed-size data structure for approximating cardinality +- `hypopg-src` - Extension to create hypothetical indexes +- `ip4r-src` - IPv4/v6 and subnet data types +- `pg_cron-src` - Run periodic jobs in PostgreSQL +- `pg_graphql-src` - GraphQL support for PostgreSQL +- `pg_hint_plan-src` - Execution plan hints +- `pg_ivm-src` - Incremental view maintenance +- `pg_jsonschema-src` - JSON Schema validation +- `pg_repack-src` - Reorganize tables with minimal locks +- `pg_roaringbitmap-src` - Roaring bitmap implementation +- `pg_semver-src` - Semantic version data type +- `pg_session_jwt-src` - JWT authentication for PostgreSQL +- `pg_tiktoken-src` - OpenAI Tiktoken tokenizer +- `pg_uuidv7-src` - UUIDv7 implementation for PostgreSQL +- `pgjwt-src` - JWT tokens for PostgreSQL +- `pgrag-src` - Retrieval Augmented Generation for PostgreSQL +- `pgtap-src` - Unit testing framework for PostgreSQL +- `pgvector-src` - Vector similarity search +- `pgx_ulid-src` - ULID data type +- `plv8-src` - JavaScript language for PostgreSQL stored procedures +- `postgresql-unit-src` - SI units for PostgreSQL +- `prefix-src` - Prefix matching for strings +- `rag_bge_small_en_v15-src` - BGE embedding model for RAG +- `rag_jina_reranker_v1_tiny_en-src` - Jina reranker model for RAG +- `rum-src` - RUM access method for text search + +## Usage + +### Extension Upgrade Testing + +The extensions in this directory are used by the `test-upgrade.sh` script to test upgrading extensions between different versions of Neon Compute nodes. The script: + +1. Creates a database with extensions installed on an old Compute version +2. Creates timelines for each extension +3. Switches to a new Compute version and tests the upgrade process +4. Verifies extension functionality after upgrade + +### Regular User Testing + +For testing with regular users (particularly for cloud instances), each extension directory typically contains a `regular-test.sh` script that: + +1. Drops the database if it exists +2. Creates a fresh test database +3. Installs the extension +4. Runs regression tests + +A note about pg_regress: Since pg_regress attempts to set `lc_messages` for the database by default, which is forbidden for regular users, we create databases manually and use the `--use-existing` option to bypass this limitation. + +### CI Workflows + +Two main workflows use these extensions: + +1. **Cloud Extensions Test** - Tests extensions on Neon cloud projects +2. **Force Test Upgrading of Extension** - Tests upgrading extensions between different Compute versions + +These workflows are integrated into the build-and-test pipeline through shell scripts: + +- `docker_compose_test.sh` - Tests extensions in a Docker Compose environment + +- `test_extensions_upgrade.sh` - Tests extension upgrades between different Compute versions + +## Adding New Extensions + +To add a new extension for testing: + +1. Create a directory named `extension-name-src` in this directory +2. Add at minimum: + - `regular-test.sh` for testing with regular users + - If `regular-test.sh` doesn't exist, the system will look for `neon-test.sh` + - If neither exists, it will try to run `make installcheck` + - `test-upgrade.sh` is only needed if you want to test upgrade scenarios +3. Update the list of extensions in the `test_extensions_upgrade.sh` script if needed for upgrade testing + +### Patching Extension Sources + +If you need to patch the extension sources: + +1. Place the patch file in the extension's directory +2. Apply the patch in the appropriate script (`test-upgrade.sh`, `neon-test.sh`, `regular-test.sh`, or `Makefile`) +3. The patch will be applied during the testing process diff --git a/pageserver/src/tenant/storage_layer/delta_layer.rs b/pageserver/src/tenant/storage_layer/delta_layer.rs index a09d8b26ec..607b0d513c 100644 --- a/pageserver/src/tenant/storage_layer/delta_layer.rs +++ b/pageserver/src/tenant/storage_layer/delta_layer.rs @@ -1442,6 +1442,19 @@ impl DeltaLayerInner { } pub fn iter<'a>(&'a self, ctx: &'a RequestContext) -> DeltaLayerIterator<'a> { + self.iter_with_options( + ctx, + 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. + 1024, // The default value. Unit tests might use a different value + ) + } + + pub fn iter_with_options<'a>( + &'a self, + ctx: &'a RequestContext, + max_read_size: u64, + max_batch_size: usize, + ) -> DeltaLayerIterator<'a> { let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, block_reader); @@ -1451,10 +1464,7 @@ impl DeltaLayerInner { index_iter: tree_reader.iter(&[0; DELTA_KEY_SIZE], ctx), key_values_batch: std::collections::VecDeque::new(), is_end: false, - planner: StreamingVectoredReadPlanner::new( - 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. - 1024, // The default value. Unit tests might use a different value - ), + planner: StreamingVectoredReadPlanner::new(max_read_size, max_batch_size), } } diff --git a/pageserver/src/tenant/storage_layer/image_layer.rs b/pageserver/src/tenant/storage_layer/image_layer.rs index a617ffc308..2f7c5715bb 100644 --- a/pageserver/src/tenant/storage_layer/image_layer.rs +++ b/pageserver/src/tenant/storage_layer/image_layer.rs @@ -685,6 +685,19 @@ impl ImageLayerInner { } pub(crate) fn iter<'a>(&'a self, ctx: &'a RequestContext) -> ImageLayerIterator<'a> { + self.iter_with_options( + ctx, + 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. + 1024, // The default value. Unit tests might use a different value + ) + } + + pub(crate) fn iter_with_options<'a>( + &'a self, + ctx: &'a RequestContext, + max_read_size: u64, + max_batch_size: usize, + ) -> ImageLayerIterator<'a> { let block_reader = FileBlockReader::new(&self.file, self.file_id); let tree_reader = DiskBtreeReader::new(self.index_start_blk, self.index_root_blk, block_reader); @@ -694,10 +707,7 @@ impl ImageLayerInner { index_iter: tree_reader.iter(&[0; KEY_SIZE], ctx), key_values_batch: VecDeque::new(), is_end: false, - planner: StreamingVectoredReadPlanner::new( - 1024 * 8192, // The default value. Unit tests might use a different value. 1024 * 8K = 8MB buffer. - 1024, // The default value. Unit tests might use a different value - ), + planner: StreamingVectoredReadPlanner::new(max_read_size, max_batch_size), } } diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 55db9fe06a..e084e3d567 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -19,6 +19,7 @@ pub(crate) enum LayerRef<'a> { } impl<'a> LayerRef<'a> { + #[allow(dead_code)] fn iter(self, ctx: &'a RequestContext) -> LayerIterRef<'a> { match self { Self::Image(x) => LayerIterRef::Image(x.iter(ctx)), @@ -26,6 +27,22 @@ impl<'a> LayerRef<'a> { } } + fn iter_with_options( + self, + ctx: &'a RequestContext, + max_read_size: u64, + max_batch_size: usize, + ) -> LayerIterRef<'a> { + match self { + Self::Image(x) => { + LayerIterRef::Image(x.iter_with_options(ctx, max_read_size, max_batch_size)) + } + Self::Delta(x) => { + LayerIterRef::Delta(x.iter_with_options(ctx, max_read_size, max_batch_size)) + } + } + } + fn layer_dbg_info(&self) -> String { match self { Self::Image(x) => x.layer_dbg_info(), @@ -66,6 +83,8 @@ pub(crate) enum IteratorWrapper<'a> { first_key_lower_bound: (Key, Lsn), layer: LayerRef<'a>, source_desc: Arc, + max_read_size: u64, + max_batch_size: usize, }, Loaded { iter: PeekableLayerIterRef<'a>, @@ -146,6 +165,8 @@ impl<'a> IteratorWrapper<'a> { pub fn create_from_image_layer( image_layer: &'a ImageLayerInner, ctx: &'a RequestContext, + max_read_size: u64, + max_batch_size: usize, ) -> Self { Self::NotLoaded { layer: LayerRef::Image(image_layer), @@ -157,12 +178,16 @@ impl<'a> IteratorWrapper<'a> { is_delta: false, } .into(), + max_read_size, + max_batch_size, } } pub fn create_from_delta_layer( delta_layer: &'a DeltaLayerInner, ctx: &'a RequestContext, + max_read_size: u64, + max_batch_size: usize, ) -> Self { Self::NotLoaded { layer: LayerRef::Delta(delta_layer), @@ -174,6 +199,8 @@ impl<'a> IteratorWrapper<'a> { is_delta: true, } .into(), + max_read_size, + max_batch_size, } } @@ -204,11 +231,13 @@ impl<'a> IteratorWrapper<'a> { first_key_lower_bound, layer, source_desc, + max_read_size, + max_batch_size, } = self else { unreachable!() }; - let iter = layer.iter(ctx); + let iter = layer.iter_with_options(ctx, *max_read_size, *max_batch_size); let iter = PeekableLayerIterRef::create(iter).await?; if let Some((k1, l1, _)) = iter.peek() { let (k2, l2) = first_key_lower_bound; @@ -293,21 +322,41 @@ impl MergeIteratorItem for ((Key, Lsn, Value), Arc) { } impl<'a> MergeIterator<'a> { + pub fn create_with_options( + deltas: &[&'a DeltaLayerInner], + images: &[&'a ImageLayerInner], + ctx: &'a RequestContext, + max_read_size: u64, + max_batch_size: usize, + ) -> Self { + let mut heap = Vec::with_capacity(images.len() + deltas.len()); + for image in images { + heap.push(IteratorWrapper::create_from_image_layer( + image, + ctx, + max_read_size, + max_batch_size, + )); + } + for delta in deltas { + heap.push(IteratorWrapper::create_from_delta_layer( + delta, + ctx, + max_read_size, + max_batch_size, + )); + } + Self { + heap: BinaryHeap::from(heap), + } + } + pub fn create( deltas: &[&'a DeltaLayerInner], images: &[&'a ImageLayerInner], ctx: &'a RequestContext, ) -> Self { - let mut heap = Vec::with_capacity(images.len() + deltas.len()); - for image in images { - heap.push(IteratorWrapper::create_from_image_layer(image, ctx)); - } - for delta in deltas { - heap.push(IteratorWrapper::create_from_delta_layer(delta, ctx)); - } - Self { - heap: BinaryHeap::from(heap), - } + Self::create_with_options(deltas, images, ctx, 1024 * 8192, 1024) } pub(crate) async fn next_inner(&mut self) -> anyhow::Result> { diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index 47a07f929d..9086d29d50 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -2828,6 +2828,41 @@ impl Timeline { Ok(()) } + /// Check if the memory usage is within the limit. + async fn check_memory_usage( + self: &Arc, + layer_selection: &[Layer], + ) -> Result<(), CompactionError> { + let mut estimated_memory_usage_mb = 0.0; + let mut num_image_layers = 0; + let mut num_delta_layers = 0; + let target_layer_size_bytes = 256 * 1024 * 1024; + for layer in layer_selection { + let layer_desc = layer.layer_desc(); + if layer_desc.is_delta() { + // Delta layers at most have 1MB buffer; 3x to make it safe (there're deltas as large as 16KB). + // Multiply the layer size so that tests can pass. + estimated_memory_usage_mb += + 3.0 * (layer_desc.file_size / target_layer_size_bytes) as f64; + num_delta_layers += 1; + } else { + // Image layers at most have 1MB buffer but it might be compressed; assume 5x compression ratio. + estimated_memory_usage_mb += + 5.0 * (layer_desc.file_size / target_layer_size_bytes) as f64; + num_image_layers += 1; + } + } + if estimated_memory_usage_mb > 1024.0 { + return Err(CompactionError::Other(anyhow!( + "estimated memory usage is too high: {}MB, giving up compaction; num_image_layers={}, num_delta_layers={}", + estimated_memory_usage_mb, + num_image_layers, + num_delta_layers + ))); + } + Ok(()) + } + /// Get a watermark for gc-compaction, that is the lowest LSN that we can use as the `gc_horizon` for /// the compaction algorithm. It is min(space_cutoff, time_cutoff, latest_gc_cutoff, standby_horizon). /// Leases and retain_lsns are considered in the gc-compaction job itself so we don't need to account for them @@ -3264,6 +3299,17 @@ impl Timeline { self.check_compaction_space(&job_desc.selected_layers) .await?; + self.check_memory_usage(&job_desc.selected_layers).await?; + if job_desc.selected_layers.len() > 100 + && job_desc.rewrite_layers.len() as f64 >= job_desc.selected_layers.len() as f64 * 0.7 + { + return Err(CompactionError::Other(anyhow!( + "too many layers to rewrite: {} / {}, giving up compaction", + job_desc.rewrite_layers.len(), + job_desc.selected_layers.len() + ))); + } + // Generate statistics for the compaction for layer in &job_desc.selected_layers { let desc = layer.layer_desc(); @@ -3359,7 +3405,13 @@ impl Timeline { .context("failed to collect gc compaction keyspace") .map_err(CompactionError::Other)?; let mut merge_iter = FilterIterator::create( - MergeIterator::create(&delta_layers, &image_layers, ctx), + MergeIterator::create_with_options( + &delta_layers, + &image_layers, + ctx, + 128 * 8192, /* 1MB buffer for each of the inner iterators */ + 128, + ), dense_ks, sparse_ks, ) diff --git a/poetry.lock b/poetry.lock index 08732fd641..1a772d3415 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1274,14 +1274,14 @@ files = [ [[package]] name = "h11" -version = "0.14.0" +version = "0.16.0" description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" groups = ["main"] files = [ - {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, - {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, ] [[package]] @@ -1314,25 +1314,25 @@ files = [ [[package]] name = "httpcore" -version = "1.0.3" +version = "1.0.9" description = "A minimal low-level HTTP client." optional = false python-versions = ">=3.8" groups = ["main"] files = [ - {file = "httpcore-1.0.3-py3-none-any.whl", hash = "sha256:9a6a501c3099307d9fd76ac244e08503427679b1e81ceb1d922485e2f2462ad2"}, - {file = "httpcore-1.0.3.tar.gz", hash = "sha256:5c0f9546ad17dac4d0772b0808856eb616eb8b48ce94f49ed819fd6982a8a544"}, + {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, ] [package.dependencies] certifi = "*" -h11 = ">=0.13,<0.15" +h11 = ">=0.16" [package.extras] asyncio = ["anyio (>=4.0,<5.0)"] http2 = ["h2 (>=3,<5)"] socks = ["socksio (==1.*)"] -trio = ["trio (>=0.22.0,<0.24.0)"] +trio = ["trio (>=0.22.0,<1.0)"] [[package]] name = "httpx"