revert + add tests

Signed-off-by: Alex Chi Z <chi@neon.tech>

.config/hakari.toml

@@ -33,7 +33,6 @@ workspace-members = [
     "compute_api",
     "consumption_metrics",
     "desim",
-    "json",
     "metrics",
     "pageserver_api",
     "postgres_backend",

.dockerignore

@@ -27,4 +27,4 @@
 !storage_controller/
 !vendor/postgres-*/
 !workspace_hack/
-!build-tools/patches
+!build_tools/patches

.github/actionlint.yml

@@ -7,7 +7,6 @@ self-hosted-runner:
     - small-metal
     - small-arm64
     - unit-perf
-    - unit-perf-aws-arm
     - us-east-2
 config-variables:
   - AWS_ECR_REGION
@@ -31,7 +30,6 @@ config-variables:
   - NEON_PROD_AWS_ACCOUNT_ID
   - PGREGRESS_PG16_PROJECT_ID
   - PGREGRESS_PG17_PROJECT_ID
-  - PREWARM_PGBENCH_SIZE
   - REMOTE_STORAGE_AZURE_CONTAINER
   - REMOTE_STORAGE_AZURE_REGION
   - SLACK_CICD_CHANNEL_ID

.github/actions/run-python-test-set/action.yml

@@ -176,11 +176,7 @@ runs:
         fi
 
         if [[ $BUILD_TYPE == "debug" && $RUNNER_ARCH == 'X64' ]]; then
-          # We don't use code coverage for regression tests (the step is disabled),
-          # so there's no need to collect it.
-          # Ref https://github.com/neondatabase/neon/issues/4540
-          # cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
-          cov_prefix=()
+          cov_prefix=(scripts/coverage "--profraw-prefix=$GITHUB_JOB" --dir=/tmp/coverage run)
         else
           cov_prefix=()
         fi

.github/workflows/_build-and-test-locally.yml

@@ -150,7 +150,7 @@ jobs:
           secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
           use-fallback: false
           path: pg_install/v14
-          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools/Dockerfile') }}
+          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v14_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
 
       - name: Cache postgres v15 build
         id: cache_pg_15
@@ -162,7 +162,7 @@ jobs:
           secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
           use-fallback: false
           path: pg_install/v15
-          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools/Dockerfile') }}
+          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v15_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
 
       - name: Cache postgres v16 build
         id: cache_pg_16
@@ -174,7 +174,7 @@ jobs:
           secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
           use-fallback: false
           path: pg_install/v16
-          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools/Dockerfile') }}
+          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v16_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
 
       - name: Cache postgres v17 build
         id: cache_pg_17
@@ -186,7 +186,7 @@ jobs:
           secretKey: ${{ secrets.HETZNER_CACHE_SECRET_KEY }}
           use-fallback: false
           path: pg_install/v17
-          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools/Dockerfile') }}
+          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ inputs.build-type }}-pg-${{ steps.pg_v17_rev.outputs.pg_rev }}-bookworm-${{ hashFiles('Makefile', 'build-tools.Dockerfile') }}
 
       - name: Build all
         # Note: the Makefile picks up BUILD_TYPE and CARGO_PROFILE from the env variables

.github/workflows/benchmarking.yml

@@ -219,7 +219,6 @@ jobs:
           --ignore test_runner/performance/test_cumulative_statistics_persistence.py
           --ignore test_runner/performance/test_perf_many_relations.py
           --ignore test_runner/performance/test_perf_oltp_large_tenant.py
-          --ignore test_runner/performance/test_lfc_prewarm.py
       env:
         BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
         VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
@@ -411,77 +410,6 @@ jobs:
       env:
         SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
 
-  prewarm-test:
-    if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
-    permissions:
-      contents: write
-      statuses: write
-      id-token: write # aws-actions/configure-aws-credentials
-    env:
-      PGBENCH_SIZE: ${{ vars.PREWARM_PGBENCH_SIZE }}
-      POSTGRES_DISTRIB_DIR: /tmp/neon/pg_install
-      DEFAULT_PG_VERSION: 17
-      TEST_OUTPUT: /tmp/test_output
-      BUILD_TYPE: remote
-      SAVE_PERF_REPORT: ${{ github.event.inputs.save_perf_report || ( github.ref_name == 'main' ) }}
-      PLATFORM: "neon-staging"
-
-    runs-on: [ self-hosted, us-east-2, x64 ]
-    container:
-      image: ghcr.io/neondatabase/build-tools:pinned-bookworm
-      credentials:
-        username: ${{ github.actor }}
-        password: ${{ secrets.GITHUB_TOKEN }}
-      options: --init
-
-    steps:
-    - name: Harden the runner (Audit all outbound calls)
-      uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
-      with:
-        egress-policy: audit
-
-    - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-
-    - name: Configure AWS credentials
-      uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2
-      with:
-        aws-region: eu-central-1
-        role-to-assume: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-        role-duration-seconds: 18000 # 5 hours
-
-    - name: Download Neon artifact
-      uses: ./.github/actions/download
-      with:
-        name: neon-${{ runner.os }}-${{ runner.arch }}-release-artifact
-        path: /tmp/neon/
-        prefix: latest
-        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-
-    - name: Run prewarm benchmark
-      uses: ./.github/actions/run-python-test-set
-      with:
-        build_type: ${{ env.BUILD_TYPE }}
-        test_selection: performance/test_lfc_prewarm.py
-        run_in_parallel: false
-        save_perf_report: ${{ env.SAVE_PERF_REPORT }}
-        extra_params: -m remote_cluster --timeout 5400
-        pg_version: ${{ env.DEFAULT_PG_VERSION }}
-        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-      env:
-        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
-        PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
-        NEON_API_KEY: ${{ secrets.NEON_STAGING_API_KEY }}
-
-    - name: Create Allure report
-      id: create-allure-report
-      if: ${{ !cancelled() }}
-      uses: ./.github/actions/allure-report-generate
-      with:
-        store-test-results-into-db: true
-        aws-oidc-role-arn: ${{ vars.DEV_AWS_OIDC_ROLE_ARN }}
-      env:
-        REGRESS_TEST_RESULT_CONNSTR_NEW: ${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}
-
   generate-matrices:
     if: ${{ github.event.inputs.run_only_pgvector_tests == 'false' || github.event.inputs.run_only_pgvector_tests == null }}
     # Create matrices for the benchmarking jobs, so we run benchmarks on rds only once a week (on Saturday)

.github/workflows/build-build-tools-image.yml

@@ -72,7 +72,7 @@ jobs:
           ARCHS: ${{ inputs.archs || '["x64","arm64"]' }}
           DEBIANS: ${{ inputs.debians || '["bullseye","bookworm"]' }}
           IMAGE_TAG: |
-            ${{ hashFiles('build-tools/Dockerfile',
+            ${{ hashFiles('build-tools.Dockerfile',
                           '.github/workflows/build-build-tools-image.yml') }}
         run: |
           echo "archs=${ARCHS}"           | tee -a ${GITHUB_OUTPUT}
@@ -144,7 +144,7 @@ jobs:
 
       - uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
-          file: build-tools/Dockerfile
+          file: build-tools.Dockerfile
           context: .
           provenance: false
           push: true

.github/workflows/build-macos.yml

@@ -32,14 +32,162 @@ permissions:
   contents: read
 
 jobs:
-  make-all:
+  build-pgxn:
+    if: |
+      inputs.pg_versions != '[]' || inputs.rebuild_everything ||
+      contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos')  ||
+      contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') ||
+      github.ref_name == 'main'
+    timeout-minutes: 30
+    runs-on: macos-15
+    strategy:
+      matrix:
+        postgres-version: ${{ inputs.rebuild_everything && fromJSON('["v14", "v15", "v16", "v17"]') || fromJSON(inputs.pg_versions) }}
+    env:
+      # Use release build only, to have less debug info around
+      # Hence keeping target/ (and general cache size) smaller
+      BUILD_TYPE: release
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout main repo
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Set pg ${{ matrix.postgres-version }} for caching
+        id: pg_rev
+        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-${{ matrix.postgres-version }}) | tee -a "${GITHUB_OUTPUT}"
+
+      - name: Cache postgres ${{ matrix.postgres-version }} build
+        id: cache_pg
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: pg_install/${{ matrix.postgres-version }}
+          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-${{ matrix.postgres-version }}-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
+
+      - name: Checkout submodule vendor/postgres-${{ matrix.postgres-version }}
+        if: steps.cache_pg.outputs.cache-hit != 'true'
+        run: |
+          git submodule init vendor/postgres-${{ matrix.postgres-version }}
+          git submodule update --depth 1 --recursive
+
+      - name: Install build dependencies
+        if: steps.cache_pg.outputs.cache-hit != 'true'
+        run: |
+          brew install flex bison openssl protobuf icu4c
+
+      - name: Set extra env for macOS
+        if: steps.cache_pg.outputs.cache-hit != 'true'
+        run: |
+          echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
+          echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
+
+      - name: Build Postgres ${{ matrix.postgres-version }}
+        if: steps.cache_pg.outputs.cache-hit != 'true'
+        run: |
+          make postgres-${{ matrix.postgres-version }} -j$(sysctl -n hw.ncpu)
+
+      - name: Build Neon Pg Ext ${{ matrix.postgres-version }}
+        if: steps.cache_pg.outputs.cache-hit != 'true'
+        run: |
+          make "neon-pg-ext-${{ matrix.postgres-version }}" -j$(sysctl -n hw.ncpu)
+
+      - name: Upload "pg_install/${{ matrix.postgres-version }}" artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: pg_install--${{ matrix.postgres-version }}
+          path: pg_install/${{ matrix.postgres-version }}
+          # The artifact is supposed to be used by the next job in the same workflow,
+          # so there’s no need to store it for too long.
+          retention-days: 1
+
+  build-walproposer-lib:
+    if: |
+      contains(inputs.pg_versions, 'v17') || inputs.rebuild_everything ||
+      contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos')  ||
+      contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') ||
+      github.ref_name == 'main'
+    timeout-minutes: 30
+    runs-on: macos-15
+    needs: [build-pgxn]
+    env:
+      # Use release build only, to have less debug info around
+      # Hence keeping target/ (and general cache size) smaller
+      BUILD_TYPE: release
+    steps:
+      - name: Harden the runner (Audit all outbound calls)
+        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
+        with:
+          egress-policy: audit
+
+      - name: Checkout main repo
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+      - name: Set pg v17 for caching
+        id: pg_rev
+        run: echo pg_rev=$(git rev-parse HEAD:vendor/postgres-v17) | tee -a "${GITHUB_OUTPUT}"
+
+      - name: Download "pg_install/v17" artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: pg_install--v17
+          path: pg_install/v17
+
+      # `actions/download-artifact` doesn't preserve permissions:
+      # https://github.com/actions/download-artifact?tab=readme-ov-file#permission-loss
+      - name: Make pg_install/v*/bin/* executable
+        run: |
+          chmod +x pg_install/v*/bin/*
+
+      - name: Cache walproposer-lib
+        id: cache_walproposer_lib
+        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+        with:
+          path: build/walproposer-lib
+          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-walproposer_lib-v17-${{ steps.pg_rev.outputs.pg_rev }}-${{ hashFiles('Makefile') }}
+
+      - name: Checkout submodule vendor/postgres-v17
+        if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
+        run: |
+          git submodule init vendor/postgres-v17
+          git submodule update --depth 1 --recursive
+
+      - name: Install build dependencies
+        if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
+        run: |
+          brew install flex bison openssl protobuf icu4c
+
+      - name: Set extra env for macOS
+        if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
+        run: |
+          echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
+          echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
+
+      - name: Build walproposer-lib (only for v17)
+        if: steps.cache_walproposer_lib.outputs.cache-hit != 'true'
+        run:
+          make walproposer-lib -j$(sysctl -n hw.ncpu) PG_INSTALL_CACHED=1
+
+      - name: Upload "build/walproposer-lib" artifact
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        with:
+          name: build--walproposer-lib
+          path: build/walproposer-lib
+          # The artifact is supposed to be used by the next job in the same workflow,
+          # so there’s no need to store it for too long.
+          retention-days: 1
+
+  cargo-build:
     if: |
       inputs.pg_versions != '[]' || inputs.rebuild_rust_code || inputs.rebuild_everything ||
       contains(github.event.pull_request.labels.*.name, 'run-extra-build-macos') ||
       contains(github.event.pull_request.labels.*.name, 'run-extra-build-*') ||
       github.ref_name == 'main'
-    timeout-minutes: 60
+    timeout-minutes: 30
     runs-on: macos-15
+    needs: [build-pgxn, build-walproposer-lib]
     env:
       # Use release build only, to have less debug info around
       # Hence keeping target/ (and general cache size) smaller
@@ -55,53 +203,41 @@ jobs:
         with:
           submodules: true
 
-      - name: Install build dependencies
-        run: |
-          brew install flex bison openssl protobuf icu4c
-
-      - name: Set extra env for macOS
-        run: |
-          echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
-          echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
-
-      - name: Restore "pg_install/" cache
-        id: cache_pg
-        uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
+      - name: Download "pg_install/v14" artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
         with:
-          path: pg_install
-          key: v1-${{ runner.os }}-${{ runner.arch }}-${{ env.BUILD_TYPE }}-pg-install-v14-${{ hashFiles('Makefile', 'postgres.mk', 'vendor/revisions.json') }}
+          name: pg_install--v14
+          path: pg_install/v14
 
-      - name: Checkout vendor/postgres submodules
-        if: steps.cache_pg.outputs.cache-hit != 'true'
-        run: |
-          git submodule init
-          git submodule update --depth 1 --recursive
+      - name: Download "pg_install/v15" artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: pg_install--v15
+          path: pg_install/v15
 
-      - name: Build Postgres
-        if: steps.cache_pg.outputs.cache-hit != 'true'
-        run: |
-          make postgres -j$(sysctl -n hw.ncpu)
+      - name: Download "pg_install/v16" artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: pg_install--v16
+          path: pg_install/v16
 
-      # This isn't strictly necessary, but it makes the cached and non-cached builds more similar,
-      # When pg_install is restored from cache, there is no 'build/' directory. By removing it
-      # in a non-cached build too, we enforce that the rest of the steps don't depend on it,
-      # so that we notice any build caching bugs earlier.
-      - name: Remove build artifacts
-        if: steps.cache_pg.outputs.cache-hit != 'true'
-        run: |
-          rm -rf build
+      - name: Download "pg_install/v17" artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: pg_install--v17
+          path: pg_install/v17
 
-      # Explicitly update the rust toolchain before running 'make'. The parallel make build can
-      # invoke 'cargo build' more than once in parallel, for different crates.  That's OK, 'cargo'
-      # does its own locking to prevent concurrent builds from stepping on each other's
-      # toes. However, it will first try to update the toolchain, and that step is not locked the
-      # same way. To avoid two toolchain updates running in parallel and stepping on each other's
-      # toes, ensure that the toolchain is up-to-date beforehand.
-      - name: Update rust toolchain
+      - name: Download "build/walproposer-lib" artifact
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: build--walproposer-lib
+          path: build/walproposer-lib
+
+      # `actions/download-artifact` doesn't preserve permissions:
+      # https://github.com/actions/download-artifact?tab=readme-ov-file#permission-loss
+      - name: Make pg_install/v*/bin/* executable
         run: |
-          rustup --version &&
-          rustup update &&
-          rustup show
+          chmod +x pg_install/v*/bin/*
 
       - name: Cache cargo deps
         uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
@@ -113,12 +249,17 @@ jobs:
             target
           key: v1-${{ runner.os }}-${{ runner.arch }}-cargo-${{ hashFiles('./Cargo.lock') }}-${{ hashFiles('./rust-toolchain.toml') }}-rust
 
-      # Build the neon-specific postgres extensions, and all the Rust bits.
-      #
-      # Pass PG_INSTALL_CACHED=1 because PostgreSQL was already built and cached
-      # separately.
-      - name: Build all
-        run: PG_INSTALL_CACHED=1 BUILD_TYPE=release make -j$(sysctl -n hw.ncpu) all
+      - name: Install build dependencies
+        run: |
+          brew install flex bison openssl protobuf icu4c
+
+      - name: Set extra env for macOS
+        run: |
+          echo 'LDFLAGS=-L/usr/local/opt/openssl@3/lib' >> $GITHUB_ENV
+          echo 'CPPFLAGS=-I/usr/local/opt/openssl@3/include' >> $GITHUB_ENV
+
+      - name: Run cargo build
+        run: cargo build --all --release -j$(sysctl -n hw.ncpu)
 
       - name: Check that no warnings are produced
         run: ./run_clippy.sh

.github/workflows/build_and_test.yml

@@ -87,24 +87,6 @@ jobs:
     uses: ./.github/workflows/build-build-tools-image.yml
     secrets: inherit
 
-  lint-openapi-spec:
-    runs-on: ubuntu-22.04
-    needs: [ meta, check-permissions ]
-    # We do need to run this in `.*-rc-pr` because of hotfixes.
-    if: ${{ contains(fromJSON('["pr", "push-main", "storage-rc-pr", "proxy-rc-pr", "compute-rc-pr"]'), needs.meta.outputs.run-kind) }}
-    steps:
-      - name: Harden the runner (Audit all outbound calls)
-        uses: step-security/harden-runner@4d991eb9b905ef189e4c376166672c3f2f230481 # v2.11.0
-        with:
-          egress-policy: audit
-      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
-      - uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - run: make lint-openapi-spec
-
   check-codestyle-python:
     needs: [ meta, check-permissions, build-build-tools-image ]
     # No need to run on `main` because we this in the merge queue. We do need to run this in `.*-rc-pr` because of hotfixes.
@@ -324,14 +306,14 @@ jobs:
       statuses: write
       contents: write
       pull-requests: write
-    runs-on: [ self-hosted, unit-perf-aws-arm ]
+    runs-on: [ self-hosted, unit-perf ]
     container:
       image: ${{ needs.build-build-tools-image.outputs.image }}-bookworm
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
       # for changed limits, see comments on `options:` earlier in this file
-      options: --init --shm-size=512mb --ulimit memlock=67108864:67108864 --ulimit nofile=65536:65536 --security-opt seccomp=unconfined
+      options: --init --shm-size=512mb --ulimit memlock=67108864:67108864
     strategy:
       fail-fast: false
       matrix:
@@ -1004,7 +986,6 @@ jobs:
       - name: Verify docker-compose example and test extensions
         timeout-minutes: 60
         env:
-          PARALLEL_COMPUTES: 3
           TAG: >-
             ${{
               needs.meta.outputs.run-kind == 'compute-rc-pr'

.github/workflows/periodic_pagebench.yml

@@ -1,4 +1,4 @@
-name: Periodic pagebench performance test on unit-perf-aws-arm runners
+name: Periodic pagebench performance test on unit-perf hetzner runner
 
 on:
   schedule:
@@ -40,7 +40,7 @@ jobs:
       statuses: write
       contents: write
       pull-requests: write
-    runs-on: [ self-hosted, unit-perf-aws-arm ]
+    runs-on: [ self-hosted, unit-perf ]
     container:
       image: ghcr.io/neondatabase/build-tools:pinned-bookworm
       credentials:

.github/workflows/proxy-benchmark.yml

@@ -1,4 +1,4 @@
-name: Periodic proxy performance test on unit-perf-aws-arm runners
+name: Periodic proxy performance test on unit-perf hetzner runner
 
 on:
   push: # TODO: remove after testing
@@ -32,7 +32,7 @@ jobs:
       statuses: write
       contents: write
       pull-requests: write
-    runs-on: [self-hosted, unit-perf-aws-arm]
+    runs-on: [self-hosted, unit-perf]
     timeout-minutes: 60  # 1h timeout
     container:
       image: ghcr.io/neondatabase/build-tools:pinned-bookworm

.gitignore

@@ -15,7 +15,6 @@ neon.iml
 /.neon
 /integration_tests/.neon
 compaction-suite-results.*
-docker-compose/docker-compose-parallel.yml
 
 # Coverage
 *.profraw

.gitmodules

@@ -1,16 +1,16 @@
 [submodule "vendor/postgres-v14"]
 	path = vendor/postgres-v14
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_14_STABLE_neon
 [submodule "vendor/postgres-v15"]
 	path = vendor/postgres-v15
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_15_STABLE_neon
 [submodule "vendor/postgres-v16"]
 	path = vendor/postgres-v16
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_16_STABLE_neon
 [submodule "vendor/postgres-v17"]
 	path = vendor/postgres-v17
-	url = ../postgres.git
+	url = https://github.com/neondatabase/postgres.git
 	branch = REL_17_STABLE_neon

Cargo.lock

@@ -1083,25 +1083,6 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
-[[package]]
-name = "cbindgen"
-version = "0.29.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "975982cdb7ad6a142be15bdf84aea7ec6a9e5d4d797c004d43185b24cfe4e684"
-dependencies = [
- "clap",
- "heck",
- "indexmap 2.9.0",
- "log",
- "proc-macro2",
- "quote",
- "serde",
- "serde_json",
- "syn 2.0.100",
- "tempfile",
- "toml",
-]
-
 [[package]]
 name = "cc"
 version = "1.2.16"
@@ -1286,15 +1267,6 @@ dependencies = [
  "unicode-width",
 ]
 
-[[package]]
-name = "communicator"
-version = "0.1.0"
-dependencies = [
- "cbindgen",
- "neon-shmem",
- "workspace_hack",
-]
-
 [[package]]
 name = "compute_api"
 version = "0.1.0"
@@ -1348,7 +1320,6 @@ dependencies = [
  "p256 0.13.2",
  "pageserver_page_api",
  "postgres",
- "postgres-types",
  "postgres_initdb",
  "postgres_versioninfo",
  "regex",
@@ -3490,15 +3461,6 @@ dependencies = [
  "wasm-bindgen",
 ]
 
-[[package]]
-name = "json"
-version = "0.1.0"
-dependencies = [
- "futures",
- "itoa",
- "ryu",
-]
-
 [[package]]
 name = "json-structural-diff"
 version = "0.2.0"
@@ -4294,9 +4256,7 @@ dependencies = [
  "humantime-serde",
  "pageserver_api",
  "pageserver_client",
- "pageserver_client_grpc",
  "pageserver_page_api",
- "pprof",
  "rand 0.8.5",
  "reqwest",
  "serde",
@@ -4325,7 +4285,6 @@ dependencies = [
  "pageserver_api",
  "postgres_ffi",
  "remote_storage",
- "serde",
  "serde_json",
  "svg_fmt",
  "thiserror 1.0.69",
@@ -4343,7 +4302,6 @@ dependencies = [
  "arc-swap",
  "async-compression",
  "async-stream",
- "base64 0.22.1",
  "bincode",
  "bit_field",
  "byteorder",
@@ -4497,26 +4455,6 @@ dependencies = [
  "workspace_hack",
 ]
 
-[[package]]
-name = "pageserver_client_grpc"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "arc-swap",
- "bytes",
- "compute_api",
- "futures",
- "pageserver_api",
- "pageserver_page_api",
- "tokio",
- "tokio-stream",
- "tokio-util",
- "tonic 0.13.1",
- "tracing",
- "utils",
- "workspace_hack",
-]
-
 [[package]]
 name = "pageserver_compaction"
 version = "0.1.0"
@@ -5290,7 +5228,6 @@ dependencies = [
  "async-trait",
  "atomic-take",
  "aws-config",
- "aws-credential-types",
  "aws-sdk-iam",
  "aws-sigv4",
  "base64 0.22.1",
@@ -5330,7 +5267,6 @@ dependencies = [
  "itoa",
  "jose-jwa",
  "jose-jwk",
- "json",
  "lasso",
  "measured",
  "metrics",
@@ -5711,8 +5647,6 @@ dependencies = [
  "azure_identity",
  "azure_storage",
  "azure_storage_blobs",
- "base64 0.22.1",
- "byteorder",
  "bytes",
  "camino",
  "camino-tempfile",
@@ -6994,7 +6928,6 @@ dependencies = [
  "pageserver_api",
  "pageserver_client",
  "reqwest",
- "safekeeper_api",
  "serde_json",
  "storage_controller_client",
  "tokio",
@@ -7564,7 +7497,6 @@ dependencies = [
  "futures-core",
  "pin-project-lite",
  "tokio",
- "tokio-util",
 ]
 
 [[package]]
@@ -8733,10 +8665,8 @@ dependencies = [
  "fail",
  "form_urlencoded",
  "futures-channel",
- "futures-core",
  "futures-executor",
  "futures-io",
- "futures-sink",
  "futures-util",
  "generic-array",
  "getrandom 0.2.11",
@@ -8763,7 +8693,6 @@ dependencies = [
  "num-iter",
  "num-rational",
  "num-traits",
- "once_cell",
  "p256 0.13.2",
  "parquet",
  "prettyplease",

Cargo.toml

@@ -8,7 +8,6 @@ members = [
     "pageserver/compaction",
     "pageserver/ctl",
     "pageserver/client",
-    "pageserver/client_grpc",
     "pageserver/pagebench",
     "pageserver/page_api",
     "proxy",
@@ -43,12 +42,10 @@ members = [
     "libs/walproposer",
     "libs/wal_decoder",
     "libs/postgres_initdb",
-    "libs/proxy/json",
     "libs/proxy/postgres-protocol2",
     "libs/proxy/postgres-types2",
     "libs/proxy/tokio-postgres2",
     "endpoint_storage",
-    "pgxn/neon/communicator",
 ]
 
 [workspace.package]
@@ -201,7 +198,7 @@ tokio-epoll-uring = { git = "https://github.com/neondatabase/tokio-epoll-uring.g
 tokio-io-timeout = "1.2.0"
 tokio-postgres-rustls = "0.12.0"
 tokio-rustls = { version = "0.26.0", default-features = false, features = ["tls12", "ring"]}
-tokio-stream = { version = "0.1", features = ["sync"] }
+tokio-stream = "0.1"
 tokio-tar = "0.3"
 tokio-util = { version = "0.7.10", features = ["io", "io-util", "rt"] }
 toml = "0.8"
@@ -258,11 +255,9 @@ desim = { version = "0.1", path = "./libs/desim" }
 endpoint_storage = { version = "0.0.1", path = "./endpoint_storage/" }
 http-utils = { version = "0.1", path = "./libs/http-utils/" }
 metrics = { version = "0.1", path = "./libs/metrics/" }
-neon-shmem = { version = "0.1", path = "./libs/neon-shmem/" }
 pageserver = { path = "./pageserver" }
 pageserver_api = { version = "0.1", path = "./libs/pageserver_api/" }
 pageserver_client = { path = "./pageserver/client" }
-pageserver_client_grpc = { path = "./pageserver/client_grpc" }
 pageserver_compaction = { version = "0.1", path = "./pageserver/compaction/" }
 pageserver_page_api = { path = "./pageserver/page_api" }
 postgres_backend = { version = "0.1", path = "./libs/postgres_backend/" }
@@ -289,7 +284,6 @@ walproposer = { version = "0.1", path = "./libs/walproposer/" }
 workspace_hack = { version = "0.1", path = "./workspace_hack/" }
 
 ## Build dependencies
-cbindgen = "0.29.0"
 criterion = "0.5.1"
 rcgen = "0.13"
 rstest = "0.18"

Dockerfile

@@ -30,18 +30,7 @@ ARG BASE_IMAGE_SHA=debian:${DEBIAN_FLAVOR}
 ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bookworm-slim/debian@$BOOKWORM_SLIM_SHA}
 ARG BASE_IMAGE_SHA=${BASE_IMAGE_SHA/debian:bullseye-slim/debian@$BULLSEYE_SLIM_SHA}
 
-# Naive way:
-#
-# 1. COPY . .
-# 1. make neon-pg-ext
-# 2. cargo build <storage binaries>
-#
-# But to enable docker to cache intermediate layers, we perform a few preparatory steps:
-#
-# - Build all postgres versions, depending on just the contents of vendor/
-# - Use cargo chef to build all rust dependencies
-
-# 1. Build all postgres versions
+# Build Postgres
 FROM $REPOSITORY/$IMAGE:$TAG AS pg-build
 WORKDIR /home/nonroot
 
@@ -49,15 +38,17 @@ COPY --chown=nonroot vendor/postgres-v14 vendor/postgres-v14
 COPY --chown=nonroot vendor/postgres-v15 vendor/postgres-v15
 COPY --chown=nonroot vendor/postgres-v16 vendor/postgres-v16
 COPY --chown=nonroot vendor/postgres-v17 vendor/postgres-v17
+COPY --chown=nonroot pgxn pgxn
 COPY --chown=nonroot Makefile Makefile
 COPY --chown=nonroot postgres.mk postgres.mk
 COPY --chown=nonroot scripts/ninstall.sh scripts/ninstall.sh
 
 ENV BUILD_TYPE=release
 RUN set -e \
-    && mold -run make -j $(nproc) -s postgres
+    && mold -run make -j $(nproc) -s neon-pg-ext \
+    && tar -C pg_install -czf /home/nonroot/postgres_install.tar.gz .
 
-# 2. Prepare cargo-chef recipe
+# Prepare cargo-chef recipe
 FROM $REPOSITORY/$IMAGE:$TAG AS plan
 WORKDIR /home/nonroot
 
@@ -65,22 +56,23 @@ COPY --chown=nonroot . .
 
 RUN cargo chef prepare --recipe-path recipe.json
 
-# Main build image
+# Build neon binaries
 FROM $REPOSITORY/$IMAGE:$TAG AS build
 WORKDIR /home/nonroot
 ARG GIT_VERSION=local
 ARG BUILD_TAG
+
+COPY --from=pg-build /home/nonroot/pg_install/v14/include/postgresql/server pg_install/v14/include/postgresql/server
+COPY --from=pg-build /home/nonroot/pg_install/v15/include/postgresql/server pg_install/v15/include/postgresql/server
+COPY --from=pg-build /home/nonroot/pg_install/v16/include/postgresql/server pg_install/v16/include/postgresql/server
+COPY --from=pg-build /home/nonroot/pg_install/v17/include/postgresql/server pg_install/v17/include/postgresql/server
+COPY --from=plan     /home/nonroot/recipe.json                              recipe.json
+
 ARG ADDITIONAL_RUSTFLAGS=""
 
-# 3. Build cargo dependencies. Note that this step doesn't depend on anything else than
-# `recipe.json`, so the layer can be reused as long as none of the dependencies change.
-COPY --from=plan     /home/nonroot/recipe.json                              recipe.json
 RUN set -e \
     && RUSTFLAGS="-Clinker=clang -Clink-arg=-fuse-ld=mold -Clink-arg=-Wl,--no-rosegment -Cforce-frame-pointers=yes ${ADDITIONAL_RUSTFLAGS}" cargo chef cook --locked --release --recipe-path recipe.json
 
-# Perform the main build. We reuse the Postgres build artifacts from the intermediate 'pg-build'
-# layer, and the cargo dependencies built in the previous step.
-COPY --chown=nonroot --from=pg-build /home/nonroot/pg_install/ pg_install
 COPY --chown=nonroot . .
 
 RUN set -e \
@@ -95,10 +87,10 @@ RUN set -e \
       --bin endpoint_storage \
       --bin neon_local \
       --bin storage_scrubber \
-      --locked --release \
-    && mold -run make -j $(nproc) -s neon-pg-ext
+      --locked --release
 
-# Assemble the final image
+# Build final image
+#
 FROM $BASE_IMAGE_SHA
 WORKDIR /data
 
@@ -138,15 +130,12 @@ COPY --from=build --chown=neon:neon /home/nonroot/target/release/proxy
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/endpoint_storage    /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/neon_local          /usr/local/bin
 COPY --from=build --chown=neon:neon /home/nonroot/target/release/storage_scrubber    /usr/local/bin
-COPY --from=build /home/nonroot/pg_install/v14 /usr/local/v14/
-COPY --from=build /home/nonroot/pg_install/v15 /usr/local/v15/
-COPY --from=build /home/nonroot/pg_install/v16 /usr/local/v16/
-COPY --from=build /home/nonroot/pg_install/v17 /usr/local/v17/
 
-# Deprecated: Old deployment scripts use this tarball which contains all the Postgres binaries.
-# That's obsolete, since all the same files are also present under /usr/local/v*. But to keep the
-# old scripts working for now, create the tarball.
-RUN tar -C /usr/local -cvzf /data/postgres_install.tar.gz v14 v15 v16 v17
+COPY --from=pg-build /home/nonroot/pg_install/v14 /usr/local/v14/
+COPY --from=pg-build /home/nonroot/pg_install/v15 /usr/local/v15/
+COPY --from=pg-build /home/nonroot/pg_install/v16 /usr/local/v16/
+COPY --from=pg-build /home/nonroot/pg_install/v17 /usr/local/v17/
+COPY --from=pg-build /home/nonroot/postgres_install.tar.gz /data/
 
 # By default, pageserver uses `.neon/` working directory in WORKDIR, so create one and fill it with the dummy config.
 # Now, when `docker run ... pageserver` is run, it can start without errors, yet will have some default dummy values.

Makefile

@@ -30,18 +30,11 @@ ifeq ($(BUILD_TYPE),release)
 	PG_CFLAGS += -O2 -g3 $(CFLAGS)
 	PG_LDFLAGS = $(LDFLAGS)
 	CARGO_PROFILE ?= --profile=release
-	# NEON_CARGO_ARTIFACT_TARGET_DIR is the directory where `cargo build` places
-	# the final build artifacts. There is unfortunately no easy way of changing
-	# it to a fully predictable path, nor to extract the path with a simple
-	# command. See https://github.com/rust-lang/cargo/issues/9661 and
-	# https://github.com/rust-lang/cargo/issues/6790.
-	NEON_CARGO_ARTIFACT_TARGET_DIR = $(ROOT_PROJECT_DIR)/target/release
 else ifeq ($(BUILD_TYPE),debug)
 	PG_CONFIGURE_OPTS = --enable-debug --with-openssl --enable-cassert --enable-depend
 	PG_CFLAGS += -O0 -g3 $(CFLAGS)
 	PG_LDFLAGS = $(LDFLAGS)
 	CARGO_PROFILE ?= --profile=dev
-	NEON_CARGO_ARTIFACT_TARGET_DIR = $(ROOT_PROJECT_DIR)/target/debug
 else
 	$(error Bad build type '$(BUILD_TYPE)', see Makefile for options)
 endif
@@ -109,7 +102,7 @@ all: neon postgres-install neon-pg-ext
 
 ### Neon Rust bits
 #
-# The 'postgres_ffi' crate depends on the Postgres headers.
+# The 'postgres_ffi' depends on the Postgres headers.
 .PHONY: neon
 neon: postgres-headers-install walproposer-lib cargo-target-dir
 	+@echo "Compiling Neon"
@@ -122,13 +115,10 @@ cargo-target-dir:
 	test -e target/CACHEDIR.TAG || echo "$(CACHEDIR_TAG_CONTENTS)" > target/CACHEDIR.TAG
 
 .PHONY: neon-pg-ext-%
-neon-pg-ext-%: postgres-install-% cargo-target-dir
+neon-pg-ext-%: postgres-install-%
 	+@echo "Compiling neon-specific Postgres extensions for $*"
 	mkdir -p $(BUILD_DIR)/pgxn-$*
-	$(MAKE) PG_CONFIG="$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config" COPT='$(COPT)' \
-		NEON_CARGO_ARTIFACT_TARGET_DIR="$(NEON_CARGO_ARTIFACT_TARGET_DIR)" \
-		CARGO_BUILD_FLAGS="$(CARGO_BUILD_FLAGS)" \
-		CARGO_PROFILE="$(CARGO_PROFILE)" \
+	$(MAKE) PG_CONFIG=$(POSTGRES_INSTALL_DIR)/$*/bin/pg_config COPT='$(COPT)' \
 		-C $(BUILD_DIR)/pgxn-$*\
 		-f $(ROOT_PROJECT_DIR)/pgxn/Makefile  install
 
@@ -220,15 +210,6 @@ neon-pgindent: postgres-v17-pg-bsd-indent neon-pg-ext-v17
 setup-pre-commit-hook:
 	ln -s -f $(ROOT_PROJECT_DIR)/pre-commit.py .git/hooks/pre-commit
 
-.PHONY: lint-openapi-spec
-lint-openapi-spec:
-	# operation-2xx-response: pageserver timeline delete returns 404 on success
-	find . -iname "openapi_spec.y*ml" -exec\
-		docker run --rm -v ${PWD}:/spec ghcr.io/redocly/cli:1.34.4\
-			--skip-rule=operation-operationId --skip-rule=operation-summary --extends=minimal\
-			--skip-rule=no-server-example.com --skip-rule=operation-2xx-response\
-			lint {} \+
-
 # Targets for building PostgreSQL are defined in postgres.mk.
 #
 # But if the caller has indicated that PostgreSQL is already

build-tools.Dockerfile

@@ -35,7 +35,7 @@ RUN echo 'Acquire::Retries "5";' > /etc/apt/apt.conf.d/80-retries && \
     echo -e "retry_connrefused=on\ntimeout=15\ntries=5\nretry-on-host-error=on\n" > /root/.wgetrc && \
     echo -e "--retry-connrefused\n--connect-timeout 15\n--retry 5\n--max-time 300\n" > /root/.curlrc
 
-COPY build-tools/patches/pgcopydbv017.patch /pgcopydbv017.patch
+COPY build_tools/patches/pgcopydbv017.patch /pgcopydbv017.patch
 
 RUN if [ "${DEBIAN_VERSION}" = "bookworm" ]; then \
         set -e && \

clippy.toml

@@ -1,12 +1,9 @@
 disallowed-methods = [
     "tokio::task::block_in_place",
-
     # Allow this for now, to deny it later once we stop using Handle::block_on completely
     # "tokio::runtime::Handle::block_on",
-
-    # tokio-epoll-uring:
-    # - allow-invalid because the method doesn't exist on macOS
-    { path = "tokio_epoll_uring::thread_local_system", replacement = "tokio_epoll_uring_ext module inside pageserver crate", allow-invalid = true }
+    # use tokio_epoll_uring_ext instead
+    "tokio_epoll_uring::thread_local_system",
 ]
 
 disallowed-macros = [

compute/compute-node.Dockerfile

@@ -9,7 +9,7 @@
 #
 # build-tools:   This contains Rust compiler toolchain and other tools needed at compile
 #                time. This is also used for the storage builds. This image is defined in
-#                build-tools/Dockerfile.
+#                build-tools.Dockerfile.
 #
 # build-deps:    Contains C compiler, other build tools, and compile-time dependencies
 #                needed to compile PostgreSQL and most extensions. (Some extensions need
@@ -115,7 +115,7 @@ ARG EXTENSIONS=all
 FROM $BASE_IMAGE_SHA AS build-deps
 ARG DEBIAN_VERSION
 
-# Keep in sync with build-tools/Dockerfile
+# Keep in sync with build-tools.Dockerfile
 ENV PROTOC_VERSION=25.1
 
 # Use strict mode for bash to catch errors early
@@ -1636,14 +1636,11 @@ RUN make install USE_PGXS=1 -j $(getconf _NPROCESSORS_ONLN)
 # compile neon extensions
 #
 #########################################################################################
-FROM pg-build-with-cargo AS neon-ext-build
+FROM pg-build AS neon-ext-build
 ARG PG_VERSION
 
-USER root
-COPY . .
-
-RUN make -j $(getconf _NPROCESSORS_ONLN) -C pgxn -s install-compute \
-      BUILD_TYPE=release CARGO_BUILD_FLAGS="--locked --release" NEON_CARGO_ARTIFACT_TARGET_DIR="$(pwd)/target/release"
+COPY pgxn/ pgxn/
+RUN make -j $(getconf _NPROCESSORS_ONLN) -C pgxn -s install-compute
 
 #########################################################################################
 #
@@ -1790,7 +1787,7 @@ RUN set -e \
 #########################################################################################
 FROM build-deps AS exporters
 ARG TARGETARCH
-# Keep sql_exporter version same as in build-tools/Dockerfile and
+# Keep sql_exporter version same as in build-tools.Dockerfile and
 # test_runner/regress/test_compute_metrics.py
 # See comment on the top of the file regading `echo`, `-e` and `\n`
 RUN if [ "$TARGETARCH" = "amd64" ]; then\
@@ -1915,10 +1912,10 @@ RUN cd /ext-src/pg_repack-src && patch -p1 </ext-src/pg_repack.patch && rm -f /e
 
 COPY --chmod=755 docker-compose/run-tests.sh /run-tests.sh
 RUN echo /usr/local/pgsql/lib > /etc/ld.so.conf.d/00-neon.conf && /sbin/ldconfig
-RUN apt-get update && apt-get install -y libtap-parser-sourcehandler-pgtap-perl jq parallel \
+RUN apt-get update && apt-get install -y libtap-parser-sourcehandler-pgtap-perl jq \
    && apt clean && rm -rf /ext-src/*.tar.gz /ext-src/*.patch /var/lib/apt/lists/*
 ENV PATH=/usr/local/pgsql/bin:$PATH
-ENV PGHOST=compute1
+ENV PGHOST=compute
 ENV PGPORT=55433
 ENV PGUSER=cloud_admin
 ENV PGDATABASE=postgres

compute_tools/Cargo.toml

@@ -66,7 +66,7 @@ url.workspace = true
 uuid.workspace = true
 walkdir.workspace = true
 x509-cert.workspace = true
-postgres-types.workspace = true
+
 postgres_versioninfo.workspace = true
 postgres_initdb.workspace = true
 compute_api.workspace = true

compute_tools/README.md

@@ -46,14 +46,11 @@ stateDiagram-v2
   Configuration --> Failed : Failed to configure the compute
   Configuration --> Running : Compute has been configured
   Empty --> Init : Compute spec is immediately available
-  Empty --> TerminationPendingFast : Requested termination
-  Empty --> TerminationPendingImmediate : Requested termination
+  Empty --> TerminationPending : Requested termination
   Init --> Failed : Failed to start Postgres
   Init --> Running : Started Postgres
-  Running --> TerminationPendingFast : Requested termination
-  Running --> TerminationPendingImmediate : Requested termination
-  TerminationPendingFast --> Terminated compute with 30s delay for cplane to inspect status
-  TerminationPendingImmediate --> Terminated : Terminated compute immediately
+  Running --> TerminationPending : Requested termination
+  TerminationPending --> Terminated : Terminated compute
   Failed --> [*] : Compute exited
   Terminated --> [*] : Compute exited
 ```

compute_tools/src/compute.rs

@@ -3,7 +3,7 @@ use chrono::{DateTime, Utc};
 use compute_api::privilege::Privilege;
 use compute_api::responses::{
     ComputeConfig, ComputeCtlConfig, ComputeMetrics, ComputeStatus, LfcOffloadState,
-    LfcPrewarmState, PromoteState, TlsConfig,
+    LfcPrewarmState, TlsConfig,
 };
 use compute_api::spec::{
     ComputeAudit, ComputeFeature, ComputeMode, ComputeSpec, ExtVersion, PageserverProtocol, PgIdent,
@@ -29,7 +29,7 @@ use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
 use std::sync::{Arc, Condvar, Mutex, RwLock};
 use std::time::{Duration, Instant};
 use std::{env, fs};
-use tokio::{spawn, sync::watch, task::JoinHandle, time};
+use tokio::spawn;
 use tracing::{Instrument, debug, error, info, instrument, warn};
 use url::Url;
 use utils::id::{TenantId, TimelineId};
@@ -107,8 +107,6 @@ pub struct ComputeNodeParams {
     pub installed_extensions_collection_interval: Arc<AtomicU64>,
 }
 
-type TaskHandle = Mutex<Option<JoinHandle<()>>>;
-
 /// Compute node info shared across several `compute_ctl` threads.
 pub struct ComputeNode {
     pub params: ComputeNodeParams,
@@ -131,8 +129,7 @@ pub struct ComputeNode {
     pub compute_ctl_config: ComputeCtlConfig,
 
     /// Handle to the extension stats collection task
-    extension_stats_task: TaskHandle,
-    lfc_offload_task: TaskHandle,
+    extension_stats_task: Mutex<Option<tokio::task::JoinHandle<()>>>,
 }
 
 // store some metrics about download size that might impact startup time
@@ -174,7 +171,6 @@ pub struct ComputeState {
     /// WAL flush LSN that is set after terminating Postgres and syncing safekeepers if
     /// mode == ComputeMode::Primary. None otherwise
     pub terminate_flush_lsn: Option<Lsn>,
-    pub promote_state: Option<watch::Receiver<PromoteState>>,
 
     pub metrics: ComputeMetrics,
 }
@@ -192,7 +188,6 @@ impl ComputeState {
             lfc_prewarm_state: LfcPrewarmState::default(),
             lfc_offload_state: LfcOffloadState::default(),
             terminate_flush_lsn: None,
-            promote_state: None,
         }
     }
 
@@ -373,7 +368,7 @@ fn maybe_cgexec(cmd: &str) -> Command {
 
 struct PostgresHandle {
     postgres: std::process::Child,
-    log_collector: JoinHandle<Result<()>>,
+    log_collector: tokio::task::JoinHandle<Result<()>>,
 }
 
 impl PostgresHandle {
@@ -387,7 +382,7 @@ struct StartVmMonitorResult {
     #[cfg(target_os = "linux")]
     token: tokio_util::sync::CancellationToken,
     #[cfg(target_os = "linux")]
-    vm_monitor: Option<JoinHandle<Result<()>>>,
+    vm_monitor: Option<tokio::task::JoinHandle<Result<()>>>,
 }
 
 impl ComputeNode {
@@ -438,7 +433,6 @@ impl ComputeNode {
             ext_download_progress: RwLock::new(HashMap::new()),
             compute_ctl_config: config.compute_ctl_config,
             extension_stats_task: Mutex::new(None),
-            lfc_offload_task: Mutex::new(None),
         })
     }
 
@@ -526,8 +520,8 @@ impl ComputeNode {
             None
         };
 
+        // Terminate the extension stats collection task
         this.terminate_extension_stats_task();
-        this.terminate_lfc_offload_task();
 
         // Terminate the vm_monitor so it releases the file watcher on
         // /sys/fs/cgroup/neon-postgres.
@@ -857,15 +851,12 @@ impl ComputeNode {
         // Log metrics so that we can search for slow operations in logs
         info!(?metrics, postmaster_pid = %postmaster_pid, "compute start finished");
 
+        // Spawn the extension stats background task
         self.spawn_extension_stats_task();
 
         if pspec.spec.autoprewarm {
-            info!("autoprewarming on startup as requested");
             self.prewarm_lfc(None);
         }
-        if let Some(seconds) = pspec.spec.offload_lfc_interval_seconds {
-            self.spawn_lfc_offload_task(Duration::from_secs(seconds.into()));
-        };
         Ok(())
     }
 
@@ -956,20 +947,14 @@ impl ComputeNode {
             None
         };
 
+        let mut delay_exit = false;
         let mut state = self.state.lock().unwrap();
         state.terminate_flush_lsn = lsn;
-
-        let delay_exit = state.status == ComputeStatus::TerminationPendingFast;
-        if state.status == ComputeStatus::TerminationPendingFast
-            || state.status == ComputeStatus::TerminationPendingImmediate
-        {
-            info!(
-                "Changing compute status from {} to {}",
-                state.status,
-                ComputeStatus::Terminated
-            );
+        if let ComputeStatus::TerminationPending { mode } = state.status {
             state.status = ComputeStatus::Terminated;
             self.state_changed.notify_all();
+            // we were asked to terminate gracefully, don't exit to avoid restart
+            delay_exit = mode == compute_api::responses::TerminateMode::Fast
         }
         drop(state);
 
@@ -1040,8 +1025,6 @@ impl ComputeNode {
             PageserverProtocol::Grpc => self.try_get_basebackup_grpc(spec, lsn)?,
         };
 
-        self.fix_zenith_signal_neon_signal()?;
-
         let mut state = self.state.lock().unwrap();
         state.metrics.pageserver_connect_micros =
             connected.duration_since(started).as_micros() as u64;
@@ -1051,27 +1034,6 @@ impl ComputeNode {
         Ok(())
     }
 
-    /// Move the Zenith signal file to Neon signal file location.
-    /// This makes Compute compatible with older PageServers that don't yet
-    /// know about the Zenith->Neon rename.
-    fn fix_zenith_signal_neon_signal(&self) -> Result<()> {
-        let datadir = Path::new(&self.params.pgdata);
-
-        let neonsig = datadir.join("neon.signal");
-
-        if neonsig.is_file() {
-            return Ok(());
-        }
-
-        let zenithsig = datadir.join("zenith.signal");
-
-        if zenithsig.is_file() {
-            fs::copy(zenithsig, neonsig)?;
-        }
-
-        Ok(())
-    }
-
     /// Fetches a basebackup via gRPC. The connstring must use grpc://. Returns the timestamp when
     /// the connection was established, and the (compressed) size of the basebackup.
     fn try_get_basebackup_grpc(&self, spec: &ParsedSpec, lsn: Lsn) -> Result<(Instant, usize)> {
@@ -1087,7 +1049,7 @@ impl ComputeNode {
         };
 
         let (reader, connected) = tokio::runtime::Handle::current().block_on(async move {
-            let mut client = page_api::Client::connect(
+            let mut client = page_api::Client::new(
                 shard0_connstr,
                 spec.tenant_id,
                 spec.timeline_id,
@@ -1834,8 +1796,6 @@ impl ComputeNode {
             tls_config,
         )?;
 
-        self.pg_reload_conf()?;
-
         if !spec.skip_pg_catalog_updates {
             let max_concurrent_connections = spec.reconfigure_concurrency;
             // Temporarily reset max_cluster_size in config
@@ -1855,9 +1815,10 @@ impl ComputeNode {
 
                 Ok(())
             })?;
-            self.pg_reload_conf()?;
         }
 
+        self.pg_reload_conf()?;
+
         let unknown_op = "unknown".to_string();
         let op_id = spec.operation_uuid.as_ref().unwrap_or(&unknown_op);
         info!(
@@ -1930,8 +1891,7 @@ impl ComputeNode {
 
                             // exit loop
                             ComputeStatus::Failed
-                            | ComputeStatus::TerminationPendingFast
-                            | ComputeStatus::TerminationPendingImmediate
+                            | ComputeStatus::TerminationPending { .. }
                             | ComputeStatus::Terminated => break 'cert_update,
 
                             // wait
@@ -2397,7 +2357,10 @@ LIMIT 100",
     }
 
     pub fn spawn_extension_stats_task(&self) {
-        self.terminate_extension_stats_task();
+        // Cancel any existing task
+        if let Some(handle) = self.extension_stats_task.lock().unwrap().take() {
+            handle.abort();
+        }
 
         let conf = self.tokio_conn_conf.clone();
         let atomic_interval = self.params.installed_extensions_collection_interval.clone();
@@ -2408,23 +2371,24 @@ LIMIT 100",
             installed_extensions_collection_interval
         );
         let handle = tokio::spawn(async move {
+            // An initial sleep is added to ensure that two collections don't happen at the same time.
+            // The first collection happens during compute startup.
+            tokio::time::sleep(tokio::time::Duration::from_secs(
+                installed_extensions_collection_interval,
+            ))
+            .await;
+            let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(
+                installed_extensions_collection_interval,
+            ));
             loop {
-                info!(
-                    "[NEON_EXT_INT_SLEEP]: Interval: {}",
-                    installed_extensions_collection_interval
-                );
-                // Sleep at the start of the loop to ensure that two collections don't happen at the same time.
-                // The first collection happens during compute startup.
-                tokio::time::sleep(tokio::time::Duration::from_secs(
-                    installed_extensions_collection_interval,
-                ))
-                .await;
+                interval.tick().await;
                 let _ = installed_extensions(conf.clone()).await;
                 // Acquire a read lock on the compute spec and then update the interval if necessary
-                installed_extensions_collection_interval = std::cmp::max(
+                interval = tokio::time::interval(tokio::time::Duration::from_secs(std::cmp::max(
                     installed_extensions_collection_interval,
                     2 * atomic_interval.load(std::sync::atomic::Ordering::SeqCst),
-                );
+                )));
+                installed_extensions_collection_interval = interval.period().as_secs();
             }
         });
 
@@ -2433,30 +2397,8 @@ LIMIT 100",
     }
 
     fn terminate_extension_stats_task(&self) {
-        if let Some(h) = self.extension_stats_task.lock().unwrap().take() {
-            h.abort()
-        }
-    }
-
-    pub fn spawn_lfc_offload_task(self: &Arc<Self>, interval: Duration) {
-        self.terminate_lfc_offload_task();
-        let secs = interval.as_secs();
-        info!("spawning lfc offload worker with {secs}s interval");
-        let this = self.clone();
-        let handle = spawn(async move {
-            let mut interval = time::interval(interval);
-            interval.tick().await; // returns immediately
-            loop {
-                interval.tick().await;
-                this.offload_lfc_async().await;
-            }
-        });
-        *self.lfc_offload_task.lock().unwrap() = Some(handle);
-    }
-
-    fn terminate_lfc_offload_task(&self) {
-        if let Some(h) = self.lfc_offload_task.lock().unwrap().take() {
-            h.abort()
+        if let Some(handle) = self.extension_stats_task.lock().unwrap().take() {
+            handle.abort();
         }
     }
 
@@ -2465,11 +2407,19 @@ LIMIT 100",
         // If the value is -1, we never suspend so set the value to default collection.
         // If the value is 0, it means default, we will just continue to use the default.
         if spec.suspend_timeout_seconds == -1 || spec.suspend_timeout_seconds == 0 {
+            info!(
+                "[NEON_EXT_INT_UPD] Spec Timeout: {}, New Timeout: {}",
+                spec.suspend_timeout_seconds, DEFAULT_INSTALLED_EXTENSIONS_COLLECTION_INTERVAL
+            );
             self.params.installed_extensions_collection_interval.store(
                 DEFAULT_INSTALLED_EXTENSIONS_COLLECTION_INTERVAL,
                 std::sync::atomic::Ordering::SeqCst,
             );
         } else {
+            info!(
+                "[NEON_EXT_INT_UPD] Spec Timeout: {}",
+                spec.suspend_timeout_seconds
+            );
             self.params.installed_extensions_collection_interval.store(
                 spec.suspend_timeout_seconds as u64,
                 std::sync::atomic::Ordering::SeqCst,
@@ -2487,7 +2437,7 @@ pub async fn installed_extensions(conf: tokio_postgres::Config) -> Result<()> {
                 serde_json::to_string(&extensions).expect("failed to serialize extensions list")
             );
         }
-        Err(err) => error!("could not get installed extensions: {err}"),
+        Err(err) => error!("could not get installed extensions: {err:?}"),
     }
     Ok(())
 }

compute_tools/src/compute_prewarm.rs

@@ -5,7 +5,6 @@ use compute_api::responses::LfcOffloadState;
 use compute_api::responses::LfcPrewarmState;
 use http::StatusCode;
 use reqwest::Client;
-use std::mem::replace;
 use std::sync::Arc;
 use tokio::{io::AsyncReadExt, spawn};
 use tracing::{error, info};
@@ -70,7 +69,7 @@ impl ComputeNode {
             }
         };
         let row = match client
-            .query_one("select * from neon.get_prewarm_info()", &[])
+            .query_one("select * from get_prewarm_info()", &[])
             .await
         {
             Ok(row) => row,
@@ -89,15 +88,17 @@ impl ComputeNode {
         self.state.lock().unwrap().lfc_offload_state.clone()
     }
 
-    /// If there is a prewarm request ongoing, return false, true otherwise
+    /// Returns false if there is a prewarm request ongoing, true otherwise
     pub fn prewarm_lfc(self: &Arc<Self>, from_endpoint: Option<String>) -> bool {
+        crate::metrics::LFC_PREWARM_REQUESTS.inc();
         {
             let state = &mut self.state.lock().unwrap().lfc_prewarm_state;
-            if let LfcPrewarmState::Prewarming = replace(state, LfcPrewarmState::Prewarming) {
+            if let LfcPrewarmState::Prewarming =
+                std::mem::replace(state, LfcPrewarmState::Prewarming)
+            {
                 return false;
             }
         }
-        crate::metrics::LFC_PREWARMS.inc();
 
         let cloned = self.clone();
         spawn(async move {
@@ -105,8 +106,7 @@ impl ComputeNode {
                 cloned.state.lock().unwrap().lfc_prewarm_state = LfcPrewarmState::Completed;
                 return;
             };
-            crate::metrics::LFC_PREWARM_ERRORS.inc();
-            error!(%err, "prewarming lfc");
+            error!(%err);
             cloned.state.lock().unwrap().lfc_prewarm_state = LfcPrewarmState::Failed {
                 error: err.to_string(),
             };
@@ -146,46 +146,36 @@ impl ComputeNode {
         ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
             .await
             .context("connecting to postgres")?
-            .query_one("select neon.prewarm_local_cache($1)", &[&uncompressed])
+            .query_one("select prewarm_local_cache($1)", &[&uncompressed])
             .await
             .context("loading LFC state into postgres")
             .map(|_| ())
     }
 
-    /// If offload request is ongoing, return false, true otherwise
+    /// Returns false if there is an offload request ongoing, true otherwise
     pub fn offload_lfc(self: &Arc<Self>) -> bool {
+        crate::metrics::LFC_OFFLOAD_REQUESTS.inc();
         {
             let state = &mut self.state.lock().unwrap().lfc_offload_state;
-            if replace(state, LfcOffloadState::Offloading) == LfcOffloadState::Offloading {
+            if let LfcOffloadState::Offloading =
+                std::mem::replace(state, LfcOffloadState::Offloading)
+            {
                 return false;
             }
         }
+
         let cloned = self.clone();
-        spawn(async move { cloned.offload_lfc_with_state_update().await });
-        true
-    }
-
-    pub async fn offload_lfc_async(self: &Arc<Self>) {
-        {
-            let state = &mut self.state.lock().unwrap().lfc_offload_state;
-            if replace(state, LfcOffloadState::Offloading) == LfcOffloadState::Offloading {
+        spawn(async move {
+            let Err(err) = cloned.offload_lfc_impl().await else {
+                cloned.state.lock().unwrap().lfc_offload_state = LfcOffloadState::Completed;
                 return;
-            }
-        }
-        self.offload_lfc_with_state_update().await
-    }
-
-    async fn offload_lfc_with_state_update(&self) {
-        crate::metrics::LFC_OFFLOADS.inc();
-        let Err(err) = self.offload_lfc_impl().await else {
-            self.state.lock().unwrap().lfc_offload_state = LfcOffloadState::Completed;
-            return;
-        };
-        crate::metrics::LFC_OFFLOAD_ERRORS.inc();
-        error!(%err, "offloading lfc");
-        self.state.lock().unwrap().lfc_offload_state = LfcOffloadState::Failed {
-            error: err.to_string(),
-        };
+            };
+            error!(%err);
+            cloned.state.lock().unwrap().lfc_offload_state = LfcOffloadState::Failed {
+                error: err.to_string(),
+            };
+        });
+        true
     }
 
     async fn offload_lfc_impl(&self) -> Result<()> {
@@ -196,7 +186,7 @@ impl ComputeNode {
         ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
             .await
             .context("connecting to postgres")?
-            .query_one("select neon.get_local_cache_state()", &[])
+            .query_one("select get_local_cache_state()", &[])
             .await
             .context("querying LFC state")?
             .try_get::<usize, &[u8]>(0)

compute_tools/src/compute_promote.rs

@@ -1,132 +0,0 @@
-use crate::compute::ComputeNode;
-use anyhow::{Context, Result, bail};
-use compute_api::{
-    responses::{LfcPrewarmState, PromoteState, SafekeepersLsn},
-    spec::ComputeMode,
-};
-use std::{sync::Arc, time::Duration};
-use tokio::time::sleep;
-use utils::lsn::Lsn;
-
-impl ComputeNode {
-    /// Returns only when promote fails or succeeds. If a network error occurs
-    /// and http client disconnects, this does not stop promotion, and subsequent
-    /// calls block until promote finishes.
-    /// Called by control plane on secondary after primary endpoint is terminated
-    pub async fn promote(self: &Arc<Self>, safekeepers_lsn: SafekeepersLsn) -> PromoteState {
-        let cloned = self.clone();
-        let start_promotion = || {
-            let (tx, rx) = tokio::sync::watch::channel(PromoteState::NotPromoted);
-            tokio::spawn(async move {
-                tx.send(match cloned.promote_impl(safekeepers_lsn).await {
-                    Ok(_) => PromoteState::Completed,
-                    Err(err) => {
-                        tracing::error!(%err, "promoting");
-                        PromoteState::Failed {
-                            error: err.to_string(),
-                        }
-                    }
-                })
-            });
-            rx
-        };
-
-        let mut task;
-        // self.state is unlocked after block ends so we lock it in promote_impl
-        // and task.changed() is reached
-        {
-            task = self
-                .state
-                .lock()
-                .unwrap()
-                .promote_state
-                .get_or_insert_with(start_promotion)
-                .clone()
-        }
-        task.changed().await.expect("promote sender dropped");
-        task.borrow().clone()
-    }
-
-    // Why do we have to supply safekeepers?
-    // For secondary we use primary_connection_conninfo so safekeepers field is empty
-    async fn promote_impl(&self, safekeepers_lsn: SafekeepersLsn) -> Result<()> {
-        {
-            let state = self.state.lock().unwrap();
-            let mode = &state.pspec.as_ref().unwrap().spec.mode;
-            if *mode != ComputeMode::Replica {
-                bail!("{} is not replica", mode.to_type_str());
-            }
-
-            // we don't need to query Postgres so not self.lfc_prewarm_state()
-            match &state.lfc_prewarm_state {
-                LfcPrewarmState::NotPrewarmed | LfcPrewarmState::Prewarming => {
-                    bail!("prewarm not requested or pending")
-                }
-                LfcPrewarmState::Failed { error } => {
-                    tracing::warn!(%error, "replica prewarm failed")
-                }
-                _ => {}
-            }
-        }
-
-        let client = ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
-            .await
-            .context("connecting to postgres")?;
-
-        let primary_lsn = safekeepers_lsn.wal_flush_lsn;
-        let mut last_wal_replay_lsn: Lsn = Lsn::INVALID;
-        const RETRIES: i32 = 20;
-        for i in 0..=RETRIES {
-            let row = client
-                .query_one("SELECT pg_last_wal_replay_lsn()", &[])
-                .await
-                .context("getting last replay lsn")?;
-            let lsn: u64 = row.get::<usize, postgres_types::PgLsn>(0).into();
-            last_wal_replay_lsn = lsn.into();
-            if last_wal_replay_lsn >= primary_lsn {
-                break;
-            }
-            tracing::info!("Try {i}, replica lsn {last_wal_replay_lsn}, primary lsn {primary_lsn}");
-            sleep(Duration::from_secs(1)).await;
-        }
-        if last_wal_replay_lsn < primary_lsn {
-            bail!("didn't catch up with primary in {RETRIES} retries");
-        }
-
-        // using $1 doesn't work with ALTER SYSTEM SET
-        let safekeepers_sql = format!(
-            "ALTER SYSTEM SET neon.safekeepers='{}'",
-            safekeepers_lsn.safekeepers
-        );
-        client
-            .query(&safekeepers_sql, &[])
-            .await
-            .context("setting safekeepers")?;
-        client
-            .query("SELECT pg_reload_conf()", &[])
-            .await
-            .context("reloading postgres config")?;
-        let row = client
-            .query_one("SELECT * FROM pg_promote()", &[])
-            .await
-            .context("pg_promote")?;
-        if !row.get::<usize, bool>(0) {
-            bail!("pg_promote() returned false");
-        }
-
-        let client = ComputeNode::get_maintenance_client(&self.tokio_conn_conf)
-            .await
-            .context("connecting to postgres")?;
-        let row = client
-            .query_one("SHOW transaction_read_only", &[])
-            .await
-            .context("getting transaction_read_only")?;
-        if row.get::<usize, &str>(0) == "on" {
-            bail!("replica in read only mode after promotion");
-        }
-
-        let mut state = self.state.lock().unwrap();
-        state.pspec.as_mut().unwrap().spec.mode = ComputeMode::Primary;
-        Ok(())
-    }
-}

compute_tools/src/http/openapi_spec.yaml

@@ -83,87 +83,6 @@ paths:
               schema:
                 $ref: "#/components/schemas/DbsAndRoles"
 
-  /promote:
-    post:
-      tags:
-        - Promotion
-      summary: Promote secondary replica to primary
-      description: ""
-      operationId: promoteReplica
-      requestBody:
-        description: Promote requests data
-        required: true
-        content:
-          application/json:
-            schema:
-                $ref: "#/components/schemas/SafekeepersLsn"
-      responses:
-        200:
-          description: Promote succeeded or wasn't started
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/PromoteState"
-        500:
-          description: Promote failed
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/PromoteState"
-
-  /lfc/prewarm:
-    post:
-      summary: Request LFC Prewarm
-      parameters:
-        - name: from_endpoint
-          in: query
-          schema:
-            type: string
-      description: ""
-      operationId: lfcPrewarm
-      responses:
-        202:
-          description: LFC prewarm started
-        429:
-          description: LFC prewarm ongoing
-    get:
-      tags:
-        - Prewarm
-      summary: Get LFC prewarm state
-      description: ""
-      operationId: getLfcPrewarmState
-      responses:
-        200:
-          description: Prewarm state
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/LfcPrewarmState"
-
-  /lfc/offload:
-    post:
-      summary: Request LFC offload
-      description: ""
-      operationId: lfcOffload
-      responses:
-        202:
-          description: LFC offload started
-        429:
-          description: LFC offload ongoing
-    get:
-      tags:
-        - Prewarm
-      summary: Get LFC offloading state
-      description: ""
-      operationId: getLfcOffloadState
-      responses:
-        200:
-          description: Offload state
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/LfcOffloadState"
-
   /database_schema:
     get:
       tags:
@@ -371,28 +290,9 @@ paths:
       summary: Terminate Postgres and wait for it to exit
       description: ""
       operationId: terminate
-      parameters:
-        - name: mode
-          in: query
-          description: "Terminate mode: fast (wait 30s before returning) and immediate"
-          required: false
-          schema:
-            type: string
-            enum: ["fast", "immediate"]
-            default: fast
       responses:
         200:
           description: Result
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TerminateResponse"
-        201:
-          description: Result if compute is already terminated
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/TerminateResponse"
         412:
           description: "wrong state"
           content:
@@ -435,6 +335,15 @@ components:
         total_startup_ms:
           type: integer
 
+    Info:
+      type: object
+      description: Information about VM/Pod.
+      required:
+        - num_cpus
+      properties:
+        num_cpus:
+          type: integer
+
     DbsAndRoles:
       type: object
       description: Databases and Roles
@@ -549,14 +458,11 @@ components:
       type: string
       enum:
         - empty
-        - configuration_pending
         - init
-        - running
-        - configuration
         - failed
-        - termination_pending_fast
-        - termination_pending_immediate
-        - terminated
+        - running
+        - configuration_pending
+        - configuration
       example: running
 
     ExtensionInstallRequest:
@@ -591,69 +497,25 @@ components:
           type: string
           example: "1.0.0"
 
-    SafekeepersLsn:
+    InstalledExtensions:
       type: object
-      required:
-        - safekeepers
-        - wal_flush_lsn
       properties:
-        safekeepers:
-          description: Primary replica safekeepers
-          type: string
-        wal_flush_lsn:
-          description: Primary last WAL flush LSN
-          type: string
-
-    LfcPrewarmState:
-      type: object
-      required:
-        - status
-        - total
-        - prewarmed
-        - skipped
-      properties:
-        status:
-          description: Lfc prewarm status
-          enum: [not_prewarmed, prewarming, completed, failed]
-          type: string
-        error:
-          description: Lfc prewarm error, if any
-          type: string
-        total:
-          description: Total pages processed
-          type: integer
-        prewarmed:
-          description: Total pages prewarmed
-          type: integer
-        skipped:
-          description: Pages processed but not prewarmed
-          type: integer
-
-    LfcOffloadState:
-      type: object
-      required:
-        - status
-      properties:
-        status:
-          description: Lfc offload status
-          enum: [not_offloaded, offloading, completed, failed]
-          type: string
-        error:
-          description: Lfc offload error, if any
-          type: string
-
-    PromoteState:
-      type: object
-      required:
-        - status
-      properties:
-        status:
-          description: Promote result
-          enum: [not_promoted, completed, failed]
-          type: string
-        error:
-          description: Promote error, if any
-          type: string
+        extensions:
+          description: Contains list of installed extensions.
+          type: array
+          items:
+            type: object
+            properties:
+              extname:
+                type: string
+              version:
+                type: string
+                items:
+                  type: string
+              n_databases:
+                type: integer
+              owned_by_superuser:
+                type: integer
 
     SetRoleGrantsRequest:
       type: object
@@ -682,17 +544,6 @@ components:
           description: Role name.
           example: "neon"
 
-    TerminateResponse:
-      type: object
-      required:
-        - lsn
-      properties:
-        lsn:
-          type: string
-          nullable: true
-          description: "last WAL flush LSN"
-          example: "0/028F10D8"
-
     SetRoleGrantsResponse:
       type: object
       required:

compute_tools/src/http/routes/mod.rs

@@ -14,7 +14,6 @@ pub(in crate::http) mod insights;
 pub(in crate::http) mod lfc;
 pub(in crate::http) mod metrics;
 pub(in crate::http) mod metrics_json;
-pub(in crate::http) mod promote;
 pub(in crate::http) mod status;
 pub(in crate::http) mod terminate;
 

compute_tools/src/http/routes/promote.rs

@@ -1,14 +0,0 @@
-use crate::http::JsonResponse;
-use axum::Form;
-use http::StatusCode;
-
-pub(in crate::http) async fn promote(
-    compute: axum::extract::State<std::sync::Arc<crate::compute::ComputeNode>>,
-    Form(safekeepers_lsn): Form<compute_api::responses::SafekeepersLsn>,
-) -> axum::response::Response {
-    let state = compute.promote(safekeepers_lsn).await;
-    if let compute_api::responses::PromoteState::Failed { error } = state {
-        return JsonResponse::error(StatusCode::INTERNAL_SERVER_ERROR, error);
-    }
-    JsonResponse::success(StatusCode::OK, state)
-}

compute_tools/src/http/routes/terminate.rs

@@ -3,7 +3,7 @@ use crate::http::JsonResponse;
 use axum::extract::State;
 use axum::response::Response;
 use axum_extra::extract::OptionalQuery;
-use compute_api::responses::{ComputeStatus, TerminateMode, TerminateResponse};
+use compute_api::responses::{ComputeStatus, TerminateResponse};
 use http::StatusCode;
 use serde::Deserialize;
 use std::sync::Arc;
@@ -12,7 +12,7 @@ use tracing::info;
 
 #[derive(Deserialize, Default)]
 pub struct TerminateQuery {
-    mode: TerminateMode,
+    mode: compute_api::responses::TerminateMode,
 }
 
 /// Terminate the compute.
@@ -24,16 +24,16 @@ pub(in crate::http) async fn terminate(
     {
         let mut state = compute.state.lock().unwrap();
         if state.status == ComputeStatus::Terminated {
-            let response = TerminateResponse {
-                lsn: state.terminate_flush_lsn,
-            };
-            return JsonResponse::success(StatusCode::CREATED, response);
+            return JsonResponse::success(StatusCode::CREATED, state.terminate_flush_lsn);
         }
 
         if !matches!(state.status, ComputeStatus::Empty | ComputeStatus::Running) {
             return JsonResponse::invalid_status(state.status);
         }
-        state.set_status(mode.into(), &compute.state_changed);
+        state.set_status(
+            ComputeStatus::TerminationPending { mode },
+            &compute.state_changed,
+        );
     }
 
     forward_termination_signal(false);

compute_tools/src/http/server.rs

@@ -23,7 +23,7 @@ use super::{
     middleware::authorize::Authorize,
     routes::{
         check_writability, configure, database_schema, dbs_and_roles, extension_server, extensions,
-        grants, insights, lfc, metrics, metrics_json, promote, status, terminate,
+        grants, insights, lfc, metrics, metrics_json, status, terminate,
     },
 };
 use crate::compute::ComputeNode;
@@ -87,7 +87,6 @@ impl From<&Server> for Router<Arc<ComputeNode>> {
                 let authenticated_router = Router::<Arc<ComputeNode>>::new()
                     .route("/lfc/prewarm", get(lfc::prewarm_state).post(lfc::prewarm))
                     .route("/lfc/offload", get(lfc::offload_state).post(lfc::offload))
-                    .route("/promote", post(promote::promote))
                     .route("/check_writability", post(check_writability::is_writable))
                     .route("/configure", post(configure::configure))
                     .route("/database_schema", get(database_schema::get_schema_dump))

compute_tools/src/installed_extensions.rs

@@ -2,7 +2,6 @@ use std::collections::HashMap;
 
 use anyhow::Result;
 use compute_api::responses::{InstalledExtension, InstalledExtensions};
-use tokio_postgres::error::Error as PostgresError;
 use tokio_postgres::{Client, Config, NoTls};
 
 use crate::metrics::INSTALLED_EXTENSIONS;
@@ -11,7 +10,7 @@ use crate::metrics::INSTALLED_EXTENSIONS;
 /// and to make database listing query here more explicit.
 ///
 /// Limit the number of databases to 500 to avoid excessive load.
-async fn list_dbs(client: &mut Client) -> Result<Vec<String>, PostgresError> {
+async fn list_dbs(client: &mut Client) -> Result<Vec<String>> {
     // `pg_database.datconnlimit = -2` means that the database is in the
     // invalid state
     let databases = client
@@ -38,9 +37,7 @@ async fn list_dbs(client: &mut Client) -> Result<Vec<String>, PostgresError> {
 /// Same extension can be installed in multiple databases with different versions,
 /// so we report a separate metric (number of databases where it is installed)
 /// for each extension version.
-pub async fn get_installed_extensions(
-    mut conf: Config,
-) -> Result<InstalledExtensions, PostgresError> {
+pub async fn get_installed_extensions(mut conf: Config) -> Result<InstalledExtensions> {
     conf.application_name("compute_ctl:get_installed_extensions");
     let databases: Vec<String> = {
         let (mut client, connection) = conf.connect(NoTls).await?;

compute_tools/src/lib.rs

@@ -12,7 +12,6 @@ pub mod logger;
 pub mod catalog;
 pub mod compute;
 pub mod compute_prewarm;
-pub mod compute_promote;
 pub mod disk_quota;
 pub mod extension_server;
 pub mod installed_extensions;

compute_tools/src/lsn_lease.rs

@@ -192,7 +192,7 @@ fn acquire_lsn_lease_grpc(
     lsn: Lsn,
 ) -> Result<Option<SystemTime>> {
     tokio::runtime::Handle::current().block_on(async move {
-        let mut client = page_api::Client::connect(
+        let mut client = page_api::Client::new(
             connstring.to_string(),
             tenant_shard_id.tenant_id,
             timeline_id,

compute_tools/src/metrics.rs

@@ -97,34 +97,20 @@ pub(crate) static PG_TOTAL_DOWNTIME_MS: Lazy<GenericCounter<AtomicU64>> = Lazy::
     .expect("failed to define a metric")
 });
 
-pub(crate) static LFC_PREWARMS: Lazy<IntCounter> = Lazy::new(|| {
+/// Needed as neon.file_cache_prewarm_batch == 0 doesn't mean we never tried to prewarm.
+/// On the other hand, LFC_PREWARMED_PAGES is excessive as we can GET /lfc/prewarm
+pub(crate) static LFC_PREWARM_REQUESTS: Lazy<IntCounter> = Lazy::new(|| {
     register_int_counter!(
-        "compute_ctl_lfc_prewarms_total",
-        "Total number of LFC prewarms requested by compute_ctl or autoprewarm option",
+        "compute_ctl_lfc_prewarm_requests_total",
+        "Total number of LFC prewarm requests made by compute_ctl",
     )
     .expect("failed to define a metric")
 });
 
-pub(crate) static LFC_PREWARM_ERRORS: Lazy<IntCounter> = Lazy::new(|| {
+pub(crate) static LFC_OFFLOAD_REQUESTS: Lazy<IntCounter> = Lazy::new(|| {
     register_int_counter!(
-        "compute_ctl_lfc_prewarm_errors_total",
-        "Total number of LFC prewarm errors",
-    )
-    .expect("failed to define a metric")
-});
-
-pub(crate) static LFC_OFFLOADS: Lazy<IntCounter> = Lazy::new(|| {
-    register_int_counter!(
-        "compute_ctl_lfc_offloads_total",
-        "Total number of LFC offloads requested by compute_ctl or lfc_offload_period_seconds option",
-    )
-    .expect("failed to define a metric")
-});
-
-pub(crate) static LFC_OFFLOAD_ERRORS: Lazy<IntCounter> = Lazy::new(|| {
-    register_int_counter!(
-        "compute_ctl_lfc_offload_errors_total",
-        "Total number of LFC offload errors",
+        "compute_ctl_lfc_offload_requests_total",
+        "Total number of LFC offload requests made by compute_ctl",
     )
     .expect("failed to define a metric")
 });
@@ -138,9 +124,7 @@ pub fn collect() -> Vec<MetricFamily> {
     metrics.extend(AUDIT_LOG_DIR_SIZE.collect());
     metrics.extend(PG_CURR_DOWNTIME_MS.collect());
     metrics.extend(PG_TOTAL_DOWNTIME_MS.collect());
-    metrics.extend(LFC_PREWARMS.collect());
-    metrics.extend(LFC_PREWARM_ERRORS.collect());
-    metrics.extend(LFC_OFFLOADS.collect());
-    metrics.extend(LFC_OFFLOAD_ERRORS.collect());
+    metrics.extend(LFC_PREWARM_REQUESTS.collect());
+    metrics.extend(LFC_OFFLOAD_REQUESTS.collect());
     metrics
 }

compute_tools/src/migrations/0002-alter_roles.sql

@@ -1,16 +1,3 @@
--- On December 8th, 2023, an engineering escalation (INC-110) was opened after
--- it was found that BYPASSRLS was being applied to all roles.
---
--- PR that introduced the issue: https://github.com/neondatabase/neon/pull/5657
--- Subsequent commit on main: https://github.com/neondatabase/neon/commit/ad99fa5f0393e2679e5323df653c508ffa0ac072
---
--- NOBYPASSRLS and INHERIT are the defaults for a Postgres role, but because it
--- isn't easy to know if a Postgres cluster is affected by the issue, we need to
--- keep the migration around for a long time, if not indefinitely, so any
--- cluster can be fixed.
---
--- Branching is the gift that keeps on giving...
-
 DO $$
 DECLARE
     role_name text;

compute_tools/src/migrations/0012-grant_pg_signal_backend_to_neon_superuser.sql

@@ -1 +0,0 @@
-GRANT pg_signal_backend TO neon_superuser WITH ADMIN OPTION;

compute_tools/src/migrations/tests/0004-grant_pg_monitor_to_neon_superuser.sql

@@ -7,17 +7,13 @@ BEGIN
         INTO monitor
         FROM pg_auth_members
         WHERE roleid = 'pg_monitor'::regrole
-            AND member = 'neon_superuser'::regrole;
+            AND member = 'pg_monitor'::regrole;
 
-    IF monitor IS NULL THEN
-        RAISE EXCEPTION 'no entry in pg_auth_members for neon_superuser and pg_monitor';
-    END IF;
-
-    IF monitor.admin IS NULL OR NOT monitor.member THEN
+    IF NOT monitor.member THEN
         RAISE EXCEPTION 'neon_superuser is not a member of pg_monitor';
     END IF;
 
-    IF monitor.admin IS NULL OR NOT monitor.admin THEN
+    IF NOT monitor.admin THEN
         RAISE EXCEPTION 'neon_superuser cannot grant pg_monitor';
     END IF;
 END $$;

compute_tools/src/migrations/tests/0012-grant_pg_signal_backend_to_neon_superuser.sql

@@ -1,23 +0,0 @@
-DO $$
-DECLARE
-    signal_backend record;
-BEGIN
-    SELECT pg_has_role('neon_superuser', 'pg_signal_backend', 'member') AS member,
-            admin_option AS admin
-        INTO signal_backend
-        FROM pg_auth_members
-        WHERE roleid = 'pg_signal_backend'::regrole
-            AND member = 'neon_superuser'::regrole;
-
-    IF signal_backend IS NULL THEN
-        RAISE EXCEPTION 'no entry in pg_auth_members for neon_superuser and pg_signal_backend';
-    END IF;
-
-    IF signal_backend.member IS NULL OR NOT signal_backend.member THEN
-        RAISE EXCEPTION 'neon_superuser is not a member of pg_signal_backend';
-    END IF;
-
-    IF signal_backend.admin IS NULL OR NOT signal_backend.admin THEN
-        RAISE EXCEPTION 'neon_superuser cannot grant pg_signal_backend';
-    END IF;
-END $$;

compute_tools/src/monitor.rs

@@ -84,8 +84,7 @@ impl ComputeMonitor {
         if matches!(
             compute_status,
             ComputeStatus::Terminated
-                | ComputeStatus::TerminationPendingFast
-                | ComputeStatus::TerminationPendingImmediate
+                | ComputeStatus::TerminationPending { .. }
                 | ComputeStatus::Failed
         ) {
             info!(

compute_tools/src/spec.rs

@@ -197,7 +197,6 @@ pub async fn handle_migrations(client: &mut Client) -> Result<()> {
         include_str!(
             "./migrations/0011-grant_pg_show_replication_origin_status_to_neon_superuser.sql"
         ),
-        include_str!("./migrations/0012-grant_pg_signal_backend_to_neon_superuser.sql"),
     ];
 
     MigrationRunner::new(client, &migrations)

compute_tools/tests/pg_helpers_tests.rs

@@ -31,7 +31,6 @@ mod pg_helpers_tests {
 wal_level = logical
 hot_standby = on
 autoprewarm = off
-offload_lfc_interval_seconds = 20
 neon.safekeepers = '127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501'
 wal_log_hints = on
 log_connections = on

control_plane/src/bin/neon_local.rs

@@ -64,9 +64,7 @@ const DEFAULT_PAGESERVER_ID: NodeId = NodeId(1);
 const DEFAULT_BRANCH_NAME: &str = "main";
 project_git_version!(GIT_VERSION);
 
-#[allow(dead_code)]
 const DEFAULT_PG_VERSION: PgMajorVersion = PgMajorVersion::PG17;
-const DEFAULT_PG_VERSION_NUM: &str = "17";
 
 const DEFAULT_PAGESERVER_CONTROL_PLANE_API: &str = "http://127.0.0.1:1234/upcall/v1/";
 
@@ -169,7 +167,7 @@ struct TenantCreateCmdArgs {
     #[clap(short = 'c')]
     config: Vec<String>,
 
-    #[arg(default_value = DEFAULT_PG_VERSION_NUM)]
+    #[arg(default_value_t = DEFAULT_PG_VERSION)]
     #[clap(long, help = "Postgres version to use for the initial timeline")]
     pg_version: PgMajorVersion,
 
@@ -292,7 +290,7 @@ struct TimelineCreateCmdArgs {
     #[clap(long, help = "Human-readable alias for the new timeline")]
     branch_name: String,
 
-    #[arg(default_value = DEFAULT_PG_VERSION_NUM)]
+    #[arg(default_value_t = DEFAULT_PG_VERSION)]
     #[clap(long, help = "Postgres version")]
     pg_version: PgMajorVersion,
 }
@@ -324,7 +322,7 @@ struct TimelineImportCmdArgs {
     #[clap(long, help = "Lsn the basebackup ends at")]
     end_lsn: Option<Lsn>,
 
-    #[arg(default_value = DEFAULT_PG_VERSION_NUM)]
+    #[arg(default_value_t = DEFAULT_PG_VERSION)]
     #[clap(long, help = "Postgres version of the backup being imported")]
     pg_version: PgMajorVersion,
 }
@@ -603,7 +601,7 @@ struct EndpointCreateCmdArgs {
     )]
     config_only: bool,
 
-    #[arg(default_value = DEFAULT_PG_VERSION_NUM)]
+    #[arg(default_value_t = DEFAULT_PG_VERSION)]
     #[clap(long, help = "Postgres version")]
     pg_version: PgMajorVersion,
 
@@ -675,16 +673,6 @@ struct EndpointStartCmdArgs {
     #[arg(default_value = "90s")]
     start_timeout: Duration,
 
-    #[clap(
-        long,
-        help = "Download LFC cache from endpoint storage on endpoint startup",
-        default_value = "false"
-    )]
-    autoprewarm: bool,
-
-    #[clap(long, help = "Upload LFC cache to endpoint storage periodically")]
-    offload_lfc_interval_seconds: Option<std::num::NonZeroU64>,
-
     #[clap(
         long,
         help = "Run in development mode, skipping VM-specific operations like process termination",
@@ -1595,24 +1583,22 @@ async fn handle_endpoint(subcmd: &EndpointCmd, env: &local_env::LocalEnv) -> Res
             let endpoint_storage_token = env.generate_auth_token(&claims)?;
             let endpoint_storage_addr = env.endpoint_storage.listen_addr.to_string();
 
-            let args = control_plane::endpoint::EndpointStartArgs {
-                auth_token,
-                endpoint_storage_token,
-                endpoint_storage_addr,
-                safekeepers_generation,
-                safekeepers,
-                pageservers,
-                remote_ext_base_url: remote_ext_base_url.clone(),
-                shard_stripe_size: stripe_size.0 as usize,
-                create_test_user: args.create_test_user,
-                start_timeout: args.start_timeout,
-                autoprewarm: args.autoprewarm,
-                offload_lfc_interval_seconds: args.offload_lfc_interval_seconds,
-                dev: args.dev,
-            };
-
             println!("Starting existing endpoint {endpoint_id}...");
-            endpoint.start(args).await?;
+            endpoint
+                .start(
+                    &auth_token,
+                    endpoint_storage_token,
+                    endpoint_storage_addr,
+                    safekeepers_generation,
+                    safekeepers,
+                    pageservers,
+                    remote_ext_base_url.as_ref(),
+                    stripe_size.0 as usize,
+                    args.create_test_user,
+                    args.start_timeout,
+                    args.dev,
+                )
+                .await?;
         }
         EndpointCmd::Reconfigure(args) => {
             let endpoint_id = &args.endpoint_id;

control_plane/src/broker.rs

@@ -36,7 +36,7 @@ impl StorageBroker {
     pub async fn start(&self, retry_timeout: &Duration) -> anyhow::Result<()> {
         let broker = &self.env.broker;
 
-        println!("Starting neon broker at {}", broker.client_url());
+        print!("Starting neon broker at {}", broker.client_url());
 
         let mut args = Vec::new();
 

control_plane/src/endpoint.rs

@@ -32,8 +32,7 @@
 //!     config.json                 - passed to `compute_ctl`
 //!     pgdata/
 //!         postgresql.conf       - copy of postgresql.conf created by `compute_ctl`
-//!         neon.signal
-//!         zenith.signal         - copy of neon.signal, for backward compatibility
+//!         zenith.signal
 //!         <other PostgreSQL files>
 //! ```
 //!
@@ -374,22 +373,6 @@ impl std::fmt::Display for EndpointTerminateMode {
     }
 }
 
-pub struct EndpointStartArgs {
-    pub auth_token: Option<String>,
-    pub endpoint_storage_token: String,
-    pub endpoint_storage_addr: String,
-    pub safekeepers_generation: Option<SafekeeperGeneration>,
-    pub safekeepers: Vec<NodeId>,
-    pub pageservers: Vec<(PageserverProtocol, Host, u16)>,
-    pub remote_ext_base_url: Option<String>,
-    pub shard_stripe_size: usize,
-    pub create_test_user: bool,
-    pub start_timeout: Duration,
-    pub autoprewarm: bool,
-    pub offload_lfc_interval_seconds: Option<std::num::NonZeroU64>,
-    pub dev: bool,
-}
-
 impl Endpoint {
     fn from_dir_entry(entry: std::fs::DirEntry, env: &LocalEnv) -> Result<Endpoint> {
         if !entry.file_type()?.is_dir() {
@@ -694,7 +677,21 @@ impl Endpoint {
         })
     }
 
-    pub async fn start(&self, args: EndpointStartArgs) -> Result<()> {
+    #[allow(clippy::too_many_arguments)]
+    pub async fn start(
+        &self,
+        auth_token: &Option<String>,
+        endpoint_storage_token: String,
+        endpoint_storage_addr: String,
+        safekeepers_generation: Option<SafekeeperGeneration>,
+        safekeepers: Vec<NodeId>,
+        pageservers: Vec<(PageserverProtocol, Host, u16)>,
+        remote_ext_base_url: Option<&String>,
+        shard_stripe_size: usize,
+        create_test_user: bool,
+        start_timeout: Duration,
+        dev: bool,
+    ) -> Result<()> {
         if self.status() == EndpointStatus::Running {
             anyhow::bail!("The endpoint is already running");
         }
@@ -707,10 +704,10 @@ impl Endpoint {
             std::fs::remove_dir_all(self.pgdata())?;
         }
 
-        let pageserver_connstring = Self::build_pageserver_connstr(&args.pageservers);
+        let pageserver_connstring = Self::build_pageserver_connstr(&pageservers);
         assert!(!pageserver_connstring.is_empty());
 
-        let safekeeper_connstrings = self.build_safekeepers_connstrs(args.safekeepers)?;
+        let safekeeper_connstrings = self.build_safekeepers_connstrs(safekeepers)?;
 
         // check for file remote_extensions_spec.json
         // if it is present, read it and pass to compute_ctl
@@ -738,7 +735,7 @@ impl Endpoint {
                     cluster_id: None, // project ID: not used
                     name: None,       // project name: not used
                     state: None,
-                    roles: if args.create_test_user {
+                    roles: if create_test_user {
                         vec![Role {
                             name: PgIdent::from_str("test").unwrap(),
                             encrypted_password: None,
@@ -747,7 +744,7 @@ impl Endpoint {
                     } else {
                         Vec::new()
                     },
-                    databases: if args.create_test_user {
+                    databases: if create_test_user {
                         vec![Database {
                             name: PgIdent::from_str("neondb").unwrap(),
                             owner: PgIdent::from_str("test").unwrap(),
@@ -769,21 +766,20 @@ impl Endpoint {
                 endpoint_id: Some(self.endpoint_id.clone()),
                 mode: self.mode,
                 pageserver_connstring: Some(pageserver_connstring),
-                safekeepers_generation: args.safekeepers_generation.map(|g| g.into_inner()),
+                safekeepers_generation: safekeepers_generation.map(|g| g.into_inner()),
                 safekeeper_connstrings,
-                storage_auth_token: args.auth_token.clone(),
+                storage_auth_token: auth_token.clone(),
                 remote_extensions,
                 pgbouncer_settings: None,
-                shard_stripe_size: Some(args.shard_stripe_size),
+                shard_stripe_size: Some(shard_stripe_size),
                 local_proxy_config: None,
                 reconfigure_concurrency: self.reconfigure_concurrency,
                 drop_subscriptions_before_start: self.drop_subscriptions_before_start,
                 audit_log_level: ComputeAudit::Disabled,
                 logs_export_host: None::<String>,
-                endpoint_storage_addr: Some(args.endpoint_storage_addr),
-                endpoint_storage_token: Some(args.endpoint_storage_token),
-                autoprewarm: args.autoprewarm,
-                offload_lfc_interval_seconds: args.offload_lfc_interval_seconds,
+                endpoint_storage_addr: Some(endpoint_storage_addr),
+                endpoint_storage_token: Some(endpoint_storage_token),
+                autoprewarm: false,
                 suspend_timeout_seconds: -1, // Only used in neon_local.
             };
 
@@ -795,7 +791,7 @@ impl Endpoint {
                 debug!("spec.cluster {:?}", spec.cluster);
 
                 // fill missing fields again
-                if args.create_test_user {
+                if create_test_user {
                     spec.cluster.roles.push(Role {
                         name: PgIdent::from_str("test").unwrap(),
                         encrypted_password: None,
@@ -830,7 +826,7 @@ impl Endpoint {
         // Launch compute_ctl
         let conn_str = self.connstr("cloud_admin", "postgres");
         println!("Starting postgres node at '{conn_str}'");
-        if args.create_test_user {
+        if create_test_user {
             let conn_str = self.connstr("test", "neondb");
             println!("Also at '{conn_str}'");
         }
@@ -862,11 +858,11 @@ impl Endpoint {
         .stderr(logfile.try_clone()?)
         .stdout(logfile);
 
-        if let Some(remote_ext_base_url) = args.remote_ext_base_url {
-            cmd.args(["--remote-ext-base-url", &remote_ext_base_url]);
+        if let Some(remote_ext_base_url) = remote_ext_base_url {
+            cmd.args(["--remote-ext-base-url", remote_ext_base_url]);
         }
 
-        if args.dev {
+        if dev {
             cmd.arg("--dev");
         }
 
@@ -898,11 +894,10 @@ impl Endpoint {
                 Ok(state) => {
                     match state.status {
                         ComputeStatus::Init => {
-                            let timeout = args.start_timeout;
-                            if Instant::now().duration_since(start_at) > timeout {
+                            if Instant::now().duration_since(start_at) > start_timeout {
                                 bail!(
                                     "compute startup timed out {:?}; still in Init state",
-                                    timeout
+                                    start_timeout
                                 );
                             }
                             // keep retrying
@@ -923,18 +918,16 @@ impl Endpoint {
                         ComputeStatus::Empty
                         | ComputeStatus::ConfigurationPending
                         | ComputeStatus::Configuration
-                        | ComputeStatus::TerminationPendingFast
-                        | ComputeStatus::TerminationPendingImmediate
+                        | ComputeStatus::TerminationPending { .. }
                         | ComputeStatus::Terminated => {
                             bail!("unexpected compute status: {:?}", state.status)
                         }
                     }
                 }
                 Err(e) => {
-                    if Instant::now().duration_since(start_at) > args.start_timeout {
+                    if Instant::now().duration_since(start_at) > start_timeout {
                         return Err(e).context(format!(
-                            "timed out {:?} waiting to connect to compute_ctl HTTP",
-                            args.start_timeout
+                            "timed out {start_timeout:?} waiting to connect to compute_ctl HTTP",
                         ));
                     }
                 }

control_plane/src/local_env.rs

@@ -217,9 +217,6 @@ pub struct NeonStorageControllerConf {
     pub posthog_config: Option<PostHogConfig>,
 
     pub kick_secondary_downloads: Option<bool>,
-
-    #[serde(with = "humantime_serde")]
-    pub shard_split_request_timeout: Option<Duration>,
 }
 
 impl NeonStorageControllerConf {
@@ -253,7 +250,6 @@ impl Default for NeonStorageControllerConf {
             timeline_safekeeper_count: None,
             posthog_config: None,
             kick_secondary_downloads: None,
-            shard_split_request_timeout: None,
         }
     }
 }

control_plane/src/pageserver.rs

@@ -303,7 +303,7 @@ impl PageServerNode {
     async fn start_node(&self, retry_timeout: &Duration) -> anyhow::Result<()> {
         // TODO: using a thread here because start_process() is not async but we need to call check_status()
         let datadir = self.repo_path();
-        println!(
+        print!(
             "Starting pageserver node {} at '{}' in {:?}, retrying for {:?}",
             self.conf.id,
             self.pg_connection_config.raw_address(),
@@ -452,12 +452,6 @@ impl PageServerNode {
                 .map(|x| x.parse::<usize>())
                 .transpose()
                 .context("Failed to parse 'image_creation_threshold' as non zero integer")?,
-            // HADRON
-            image_layer_force_creation_period: settings
-                .remove("image_layer_force_creation_period")
-                .map(humantime::parse_duration)
-                .transpose()
-                .context("Failed to parse 'image_layer_force_creation_period' as duration")?,
             image_layer_creation_check_threshold: settings
                 .remove("image_layer_creation_check_threshold")
                 .map(|x| x.parse::<u8>())

control_plane/src/safekeeper.rs

@@ -127,7 +127,7 @@ impl SafekeeperNode {
         extra_opts: &[String],
         retry_timeout: &Duration,
     ) -> anyhow::Result<()> {
-        println!(
+        print!(
             "Starting safekeeper at '{}' in '{}', retrying for {:?}",
             self.pg_connection_config.raw_address(),
             self.datadir_path().display(),

control_plane/src/storage_controller.rs

@@ -648,13 +648,6 @@ impl StorageController {
             args.push(format!("--timeline-safekeeper-count={sk_cnt}"));
         }
 
-        if let Some(duration) = self.config.shard_split_request_timeout {
-            args.push(format!(
-                "--shard-split-request-timeout={}",
-                humantime::Duration::from(duration)
-            ));
-        }
-
         let mut envs = vec![
             ("LD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
             ("DYLD_LIBRARY_PATH".to_owned(), pg_lib_dir.to_string()),
@@ -667,7 +660,7 @@ impl StorageController {
             ));
         }
 
-        println!("Starting storage controller at {scheme}://{host}:{listen_port}");
+        println!("Starting storage controller");
 
         background_process::start_process(
             COMMAND,

control_plane/storcon_cli/Cargo.toml

@@ -14,7 +14,6 @@ humantime.workspace = true
 pageserver_api.workspace = true
 pageserver_client.workspace = true
 reqwest.workspace = true
-safekeeper_api.workspace=true
 serde_json = { workspace = true, features = ["raw_value"] }
 storage_controller_client.workspace = true
 tokio.workspace = true

control_plane/storcon_cli/src/main.rs

@@ -11,7 +11,7 @@ use pageserver_api::controller_api::{
     PlacementPolicy, SafekeeperDescribeResponse, SafekeeperSchedulingPolicyRequest,
     ShardSchedulingPolicy, ShardsPreferredAzsRequest, ShardsPreferredAzsResponse,
     SkSchedulingPolicy, TenantCreateRequest, TenantDescribeResponse, TenantPolicyRequest,
-    TenantShardMigrateRequest, TenantShardMigrateResponse, TimelineSafekeeperMigrateRequest,
+    TenantShardMigrateRequest, TenantShardMigrateResponse,
 };
 use pageserver_api::models::{
     EvictionPolicy, EvictionPolicyLayerAccessThreshold, ShardParameters, TenantConfig,
@@ -21,7 +21,6 @@ use pageserver_api::models::{
 use pageserver_api::shard::{ShardStripeSize, TenantShardId};
 use pageserver_client::mgmt_api::{self};
 use reqwest::{Certificate, Method, StatusCode, Url};
-use safekeeper_api::models::TimelineLocateResponse;
 use storage_controller_client::control_api::Client;
 use utils::id::{NodeId, TenantId, TimelineId};
 
@@ -66,27 +65,12 @@ enum Command {
         #[arg(long)]
         scheduling: Option<NodeSchedulingPolicy>,
     },
-    /// Exists for backup usage and will be removed in future.
-    /// Use [`Command::NodeStartDelete`] instead, if possible.
+    // Set a node status as deleted.
     NodeDelete {
         #[arg(long)]
         node_id: NodeId,
     },
-    /// Start deletion of the specified pageserver.
-    NodeStartDelete {
-        #[arg(long)]
-        node_id: NodeId,
-    },
-    /// Cancel deletion of the specified pageserver and wait for `timeout`
-    /// for the operation to be canceled. May be retried.
-    NodeCancelDelete {
-        #[arg(long)]
-        node_id: NodeId,
-        #[arg(long)]
-        timeout: humantime::Duration,
-    },
     /// Delete a tombstone of node from the storage controller.
-    /// This is used when we want to allow the node to be re-registered.
     NodeDeleteTombstone {
         #[arg(long)]
         node_id: NodeId,
@@ -280,23 +264,6 @@ enum Command {
         #[arg(long)]
         concurrency: Option<usize>,
     },
-    /// Locate safekeepers for a timeline from the storcon DB.
-    TimelineLocate {
-        #[arg(long)]
-        tenant_id: TenantId,
-        #[arg(long)]
-        timeline_id: TimelineId,
-    },
-    /// Migrate a timeline to a new set of safekeepers
-    TimelineSafekeeperMigrate {
-        #[arg(long)]
-        tenant_id: TenantId,
-        #[arg(long)]
-        timeline_id: TimelineId,
-        /// Example: --new-sk-set 1,2,3
-        #[arg(long, required = true, value_delimiter = ',')]
-        new_sk_set: Vec<NodeId>,
-    },
 }
 
 #[derive(Parser)]
@@ -476,7 +443,6 @@ async fn main() -> anyhow::Result<()> {
                         listen_http_port,
                         listen_https_port,
                         availability_zone_id: AvailabilityZone(availability_zone_id),
-                        node_ip_addr: None,
                     }),
                 )
                 .await?;
@@ -946,43 +912,10 @@ async fn main() -> anyhow::Result<()> {
                 .await?;
         }
         Command::NodeDelete { node_id } => {
-            eprintln!("Warning: This command is obsolete and will be removed in a future version");
-            eprintln!("Use `NodeStartDelete` instead, if possible");
             storcon_client
                 .dispatch::<(), ()>(Method::DELETE, format!("control/v1/node/{node_id}"), None)
                 .await?;
         }
-        Command::NodeStartDelete { node_id } => {
-            storcon_client
-                .dispatch::<(), ()>(
-                    Method::PUT,
-                    format!("control/v1/node/{node_id}/delete"),
-                    None,
-                )
-                .await?;
-            println!("Delete started for {node_id}");
-        }
-        Command::NodeCancelDelete { node_id, timeout } => {
-            storcon_client
-                .dispatch::<(), ()>(
-                    Method::DELETE,
-                    format!("control/v1/node/{node_id}/delete"),
-                    None,
-                )
-                .await?;
-
-            println!("Waiting for node {node_id} to quiesce on scheduling policy ...");
-
-            let final_policy =
-                wait_for_scheduling_policy(storcon_client, node_id, *timeout, |sched| {
-                    !matches!(sched, NodeSchedulingPolicy::Deleting)
-                })
-                .await?;
-
-            println!(
-                "Delete was cancelled for node {node_id}. Schedulling policy is now {final_policy:?}"
-            );
-        }
         Command::NodeDeleteTombstone { node_id } => {
             storcon_client
                 .dispatch::<(), ()>(
@@ -1343,7 +1276,7 @@ async fn main() -> anyhow::Result<()> {
             concurrency,
         } => {
             let mut path = format!(
-                "v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/download_heatmap_layers",
+                "/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/download_heatmap_layers",
             );
 
             if let Some(c) = concurrency {
@@ -1354,41 +1287,6 @@ async fn main() -> anyhow::Result<()> {
                 .dispatch::<(), ()>(Method::POST, path, None)
                 .await?;
         }
-        Command::TimelineLocate {
-            tenant_id,
-            timeline_id,
-        } => {
-            let path = format!("debug/v1/tenant/{tenant_id}/timeline/{timeline_id}/locate");
-
-            let resp = storcon_client
-                .dispatch::<(), TimelineLocateResponse>(Method::GET, path, None)
-                .await?;
-
-            let sk_set = resp.sk_set.iter().map(|id| id.0 as i64).collect::<Vec<_>>();
-            let new_sk_set = resp
-                .new_sk_set
-                .as_ref()
-                .map(|ids| ids.iter().map(|id| id.0 as i64).collect::<Vec<_>>());
-
-            println!("generation = {}", resp.generation);
-            println!("sk_set = {sk_set:?}");
-            println!("new_sk_set = {new_sk_set:?}");
-        }
-        Command::TimelineSafekeeperMigrate {
-            tenant_id,
-            timeline_id,
-            new_sk_set,
-        } => {
-            let path = format!("v1/tenant/{tenant_id}/timeline/{timeline_id}/safekeeper_migrate");
-
-            storcon_client
-                .dispatch::<_, ()>(
-                    Method::POST,
-                    path,
-                    Some(TimelineSafekeeperMigrateRequest { new_sk_set }),
-                )
-                .await?;
-        }
     }
 
     Ok(())

docker-compose/compute_wrapper/shell/compute.sh

@@ -54,16 +54,14 @@ else
     printf '%s\n' "${result}" | jq .
   fi
 
-  if [[ "${RUN_PARALLEL:-false}" != "true" ]]; then
-    echo "Check if a timeline present"
-    PARAMS=(
-         -X GET
-         -H "Content-Type: application/json"
-        "http://pageserver:9898/v1/tenant/${tenant_id}/timeline"
-    )
-    timeline_id=$(curl "${PARAMS[@]}" | jq -r .[0].timeline_id)
-  fi
-  if [[ -z "${timeline_id:-}" || "${timeline_id:-}" = null ]]; then
+  echo "Check if a timeline present"
+  PARAMS=(
+       -X GET
+       -H "Content-Type: application/json"
+       "http://pageserver:9898/v1/tenant/${tenant_id}/timeline"
+  )
+  timeline_id=$(curl "${PARAMS[@]}" | jq -r .[0].timeline_id)
+  if [[ -z "${timeline_id}" || "${timeline_id}" = null ]]; then
     generate_id timeline_id
     PARAMS=(
         -sbf

docker-compose/docker-compose.yml

@@ -142,7 +142,7 @@ services:
       - "storage_broker"
       - "--listen-addr=0.0.0.0:50051"
 
-  compute1:
+  compute:
     restart: always
     build:
       context: ./compute_wrapper/
@@ -152,7 +152,6 @@ services:
         - TAG=${COMPUTE_TAG:-${TAG:-latest}}
         - http_proxy=${http_proxy:-}
         - https_proxy=${https_proxy:-}
-    image: built-compute
     environment:
       - PG_VERSION=${PG_VERSION:-16}
       - TENANT_ID=${TENANT_ID:-}
@@ -167,11 +166,6 @@ services:
       - 3080:3080 # http endpoints
     entrypoint:
       - "/shell/compute.sh"
-    # Ad an alias for compute1 for compatibility
-    networks:
-      default:
-        aliases:
-            - compute
     depends_on:
       - safekeeper1
       - safekeeper2
@@ -180,20 +174,15 @@ services:
 
   compute_is_ready:
     image: postgres:latest
-    environment:
-      - PARALLEL_COMPUTES=1
     entrypoint:
-      - "/bin/sh"
+      - "/bin/bash"
       - "-c"
     command:
-      - "for i in $(seq 1 $${PARALLEL_COMPUTES}); do
-           until pg_isready -h compute$$i -p 55433 -U cloud_admin ; do
-             sleep 1;
-           done;
-         done;
-         echo All computes are started"
+      - "until pg_isready -h compute -p 55433 -U cloud_admin ; do
+            echo 'Waiting to start compute...' && sleep 1;
+         done"
     depends_on:
-      - compute1
+      - compute
 
   neon-test-extensions:
     profiles: ["test-extensions"]
@@ -207,4 +196,4 @@ services:
     command:
       - sleep 3600
     depends_on:
-      - compute1
+      - compute

docker-compose/docker_compose_test.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
 # A basic test to ensure Docker images are built correctly.
 # Build a wrapper around the compute, start all services and runs a simple SQL query.
@@ -13,36 +13,9 @@
 #
 set -eux -o pipefail
 
-cd "$(dirname "${0}")"
 export COMPOSE_FILE='docker-compose.yml'
 export COMPOSE_PROFILES=test-extensions
-export PARALLEL_COMPUTES=${PARALLEL_COMPUTES:-1}
-READY_MESSAGE="All computes are started"
-COMPUTES=()
-for i in $(seq 1 "${PARALLEL_COMPUTES}"); do
-  COMPUTES+=("compute${i}")
-done
-CURRENT_TMPDIR=$(mktemp -d)
-trap 'rm -rf ${CURRENT_TMPDIR} docker-compose-parallel.yml' EXIT
-if [[ ${PARALLEL_COMPUTES} -gt 1 ]]; then
-  export COMPOSE_FILE=docker-compose-parallel.yml
-  cp docker-compose.yml docker-compose-parallel.yml
-  # Replace the environment variable PARALLEL_COMPUTES with the actual value
-  yq eval -i ".services.compute_is_ready.environment |=  map(select(. | test(\"^PARALLEL_COMPUTES=\") | not)) + [\"PARALLEL_COMPUTES=${PARALLEL_COMPUTES}\"]" ${COMPOSE_FILE}
-  for i in $(seq 2 "${PARALLEL_COMPUTES}"); do
-    # Duplicate compute1 as compute${i} for parallel execution
-    yq eval -i ".services.compute${i} = .services.compute1" ${COMPOSE_FILE}
-    # We don't need these sections, so delete them
-    yq eval -i "(del .services.compute${i}.build) | (del .services.compute${i}.ports) | (del .services.compute${i}.networks)" ${COMPOSE_FILE}
-    # Let the compute 1 be the only dependence
-    yq eval -i ".services.compute${i}.depends_on = [\"compute1\"]" ${COMPOSE_FILE}
-    # Set RUN_PARALLEL=true for compute2. They will generate tenant_id and timeline_id to avoid using the same as other computes
-    yq eval -i ".services.compute${i}.environment += [\"RUN_PARALLEL=true\"]" ${COMPOSE_FILE}
-    # Remove TENANT_ID and TIMELINE_ID from the environment variables of the generated computes
-    # They will create new TENANT_ID and TIMELINE_ID anyway.
-    yq eval -i ".services.compute${i}.environment |= map(select(. | (test(\"^TENANT_ID=\") or test(\"^TIMELINE_ID=\")) | not))" ${COMPOSE_FILE}
-  done
-fi
+cd "$(dirname "${0}")"
 PSQL_OPTION="-h localhost -U cloud_admin -p 55433 -d postgres"
 
 function cleanup() {
@@ -54,11 +27,11 @@ function cleanup() {
 
 for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do
     pg_version=${pg_version/v/}
-    echo "clean up containers if exist"
+    echo "clean up containers if exists"
     cleanup
     PG_TEST_VERSION=$((pg_version < 16 ? 16 : pg_version))
-    PG_VERSION=${pg_version} PG_TEST_VERSION=${PG_TEST_VERSION} docker compose build compute1
-    PG_VERSION=${pg_version} PG_TEST_VERSION=${PG_TEST_VERSION} docker compose up --quiet-pull -d
+    PG_VERSION=${pg_version} PG_TEST_VERSION=${PG_TEST_VERSION} docker compose up --quiet-pull --build -d
+
     echo "wait until the compute is ready. timeout after 60s. "
     cnt=0
     while sleep 3; do
@@ -68,50 +41,45 @@ for pg_version in ${TEST_VERSION_ONLY-14 15 16 17}; do
             echo "timeout before the compute is ready."
             exit 1
         fi
-        if docker compose logs compute_is_ready | grep -q "${READY_MESSAGE}"; then
+        if docker compose logs "compute_is_ready" | grep -q "accepting connections"; then
             echo "OK. The compute is ready to connect."
             echo "execute simple queries."
-            for compute in "${COMPUTES[@]}"; do
-              docker compose exec "${compute}" /bin/bash -c "psql ${PSQL_OPTION} -c 'SELECT 1'"
-            done
+            docker compose exec compute /bin/bash -c "psql ${PSQL_OPTION} -c 'SELECT 1'"
             break
         fi
     done
 
     if [[ ${pg_version} -ge 16 ]]; then
-        mkdir "${CURRENT_TMPDIR}"/{pg_hint_plan-src,file_fdw,postgis-src}
-        docker compose cp neon-test-extensions:/ext-src/postgis-src/raster/test "${CURRENT_TMPDIR}/postgis-src/test"
-        docker compose cp neon-test-extensions:/ext-src/postgis-src/regress/00-regress-install "${CURRENT_TMPDIR}/postgis-src/00-regress-install"
-        docker compose cp neon-test-extensions:/ext-src/pg_hint_plan-src/data "${CURRENT_TMPDIR}/pg_hint_plan-src/data"
-        docker compose cp neon-test-extensions:/postgres/contrib/file_fdw/data "${CURRENT_TMPDIR}/file_fdw/data"
-
-        for compute in "${COMPUTES[@]}"; do
-          # This is required for the pg_hint_plan test, to prevent flaky log message causing the test to fail
-          # It cannot be moved to Dockerfile now because the database directory is created after the start of the container
-          echo Adding dummy config on "${compute}"
-          docker compose exec "${compute}" touch /var/db/postgres/compute/compute_ctl_temp_override.conf
-          # Prepare for the PostGIS test
-          docker compose exec "${compute}" mkdir -p /tmp/pgis_reg/pgis_reg_tmp /ext-src/postgis-src/raster /ext-src/postgis-src/regress /ext-src/postgis-src/regress/00-regress-install
-          docker compose cp "${CURRENT_TMPDIR}/postgis-src/test" "${compute}":/ext-src/postgis-src/raster/test
-          docker compose cp "${CURRENT_TMPDIR}/postgis-src/00-regress-install" "${compute}":/ext-src/postgis-src/regress
-          # The following block copies the files for the pg_hintplan test to the compute node for the extension test in an isolated docker-compose environment
-          docker compose cp "${CURRENT_TMPDIR}/pg_hint_plan-src/data" "${compute}":/ext-src/pg_hint_plan-src/
-          # The following block does the same for the contrib/file_fdw test
-          docker compose cp "${CURRENT_TMPDIR}/file_fdw/data" "${compute}":/postgres/contrib/file_fdw/data
-        done
+        # This is required for the pg_hint_plan test, to prevent flaky log message causing the test to fail
+        # It cannot be moved to Dockerfile now because the database directory is created after the start of the container
+        echo Adding dummy config
+        docker compose exec compute touch /var/db/postgres/compute/compute_ctl_temp_override.conf
+        # Prepare for the PostGIS test
+        docker compose exec compute mkdir -p /tmp/pgis_reg/pgis_reg_tmp
+        TMPDIR=$(mktemp -d)
+        docker compose cp neon-test-extensions:/ext-src/postgis-src/raster/test "${TMPDIR}"
+        docker compose cp neon-test-extensions:/ext-src/postgis-src/regress/00-regress-install "${TMPDIR}"
+        docker compose exec compute mkdir -p /ext-src/postgis-src/raster /ext-src/postgis-src/regress /ext-src/postgis-src/regress/00-regress-install
+        docker compose cp "${TMPDIR}/test" compute:/ext-src/postgis-src/raster/test
+        docker compose cp "${TMPDIR}/00-regress-install" compute:/ext-src/postgis-src/regress
+        rm -rf "${TMPDIR}"
+        # The following block copies the files for the pg_hintplan test to the compute node for the extension test in an isolated docker-compose environment
+        TMPDIR=$(mktemp -d)
+        docker compose cp neon-test-extensions:/ext-src/pg_hint_plan-src/data "${TMPDIR}/data"
+        docker compose cp "${TMPDIR}/data" compute:/ext-src/pg_hint_plan-src/
+        rm -rf "${TMPDIR}"
+        # The following block does the same for the contrib/file_fdw test
+        TMPDIR=$(mktemp -d)
+        docker compose cp neon-test-extensions:/postgres/contrib/file_fdw/data "${TMPDIR}/data"
+        docker compose cp "${TMPDIR}/data" compute:/postgres/contrib/file_fdw/data
+        rm -rf "${TMPDIR}"
         # Apply patches
         docker compose exec -T neon-test-extensions bash -c "(cd /postgres && patch -p1)" <"../compute/patches/contrib_pg${pg_version}.patch"
         # We are running tests now
         rm -f testout.txt testout_contrib.txt
-        # We want to run the longest tests first to better utilize parallelization and reduce overall test time.
-        # Tests listed in the RUN_FIRST variable will be run before others.
-        # If parallelization is not used, this environment variable will be ignored.
-
         docker compose exec -e USE_PGXS=1 -e SKIP=timescaledb-src,rdkit-src,pg_jsonschema-src,kq_imcx-src,wal2json_2_5-src,rag_jina_reranker_v1_tiny_en-src,rag_bge_small_en_v15-src \
-        -e RUN_FIRST=hll-src,postgis-src,pgtap-src -e PARALLEL_COMPUTES="${PARALLEL_COMPUTES}" \
         neon-test-extensions /run-tests.sh /ext-src | tee testout.txt && EXT_SUCCESS=1 || EXT_SUCCESS=0
         docker compose exec -e SKIP=start-scripts,postgres_fdw,ltree_plpython,jsonb_plpython,jsonb_plperl,hstore_plpython,hstore_plperl,dblink,bool_plperl \
-        -e PARALLEL_COMPUTES="${PARALLEL_COMPUTES}" \
         neon-test-extensions /run-tests.sh /postgres/contrib | tee testout_contrib.txt && CONTRIB_SUCCESS=1 || CONTRIB_SUCCESS=0
         if [[ ${EXT_SUCCESS} -eq 0 || ${CONTRIB_SUCCESS} -eq 0 ]]; then
             CONTRIB_FAILED=

docker-compose/run-tests.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 set -x
 
 if [[ -v BENCHMARK_CONNSTR ]]; then
@@ -26,9 +26,8 @@ if [[ -v BENCHMARK_CONNSTR ]]; then
   fi
 fi
 REGULAR_USER=false
-PARALLEL_COMPUTES=${PARALLEL_COMPUTES:-1}
-while getopts pr arg; do
-  case ${arg} in
+while getopts r arg; do
+  case $arg in
   r)
     REGULAR_USER=true
     shift $((OPTIND-1))
@@ -42,49 +41,26 @@ extdir=${1}
 
 cd "${extdir}" || exit 2
 FAILED=
-export FAILED_FILE=/tmp/failed
-rm -f ${FAILED_FILE}
-mapfile -t LIST < <( (echo -e "${SKIP//","/"\n"}"; ls) | sort | uniq -u)
-if [[ ${PARALLEL_COMPUTES} -gt 1 ]]; then
-  # Avoid errors if RUN_FIRST is not defined
-  RUN_FIRST=${RUN_FIRST:-}
-  # Move entries listed in the RUN_FIRST variable to the beginning
-  ORDERED_LIST=$(printf "%s\n" "${LIST[@]}" | grep -x -Ff <(echo -e "${RUN_FIRST//,/$'\n'}"); printf "%s\n" "${LIST[@]}" | grep -vx -Ff <(echo -e "${RUN_FIRST//,/$'\n'}"))
-  parallel -j"${PARALLEL_COMPUTES}" "[[ -d {} ]] || exit 0
-                export PGHOST=compute{%}
-                if ! psql -c 'select 1'>/dev/null; then
-                  exit 1
-                fi
-                echo Running on \${PGHOST}
-                if [[ -f ${extdir}/{}/neon-test.sh ]]; then
-                  echo Running from script
-                  ${extdir}/{}/neon-test.sh || echo {} >> ${FAILED_FILE};
-                else
-                  echo Running using make;
-                  USE_PGXS=1 make -C {} installcheck || echo {} >> ${FAILED_FILE};
-                fi" ::: ${ORDERED_LIST}
-  [[ ! -f ${FAILED_FILE} ]] && exit 0
-else
-  for d in "${LIST[@]}"; do
-      [ -d "${d}" ] || continue
-      if ! psql -w -c "select 1" >/dev/null; then
-        FAILED="${d} ${FAILED}"
-        break
-      fi
-      if [[ ${REGULAR_USER} = true ]] && [ -f "${d}"/regular-test.sh ]; then
-        "${d}/regular-test.sh" || FAILED="${d} ${FAILED}"
-        continue
-      fi
+LIST=$( (echo -e "${SKIP//","/"\n"}"; ls) | sort | uniq -u)
+for d in ${LIST}; do
+    [ -d "${d}" ] || continue
+    if ! psql -w -c "select 1" >/dev/null; then
+      FAILED="${d} ${FAILED}"
+      break
+    fi
+    if [[ ${REGULAR_USER} = true ]] && [ -f "${d}"/regular-test.sh ]; then
+       "${d}/regular-test.sh" || FAILED="${d} ${FAILED}"
+       continue
+    fi
 
-      if [ -f "${d}/neon-test.sh" ]; then
-        "${d}/neon-test.sh" || FAILED="${d} ${FAILED}"
-      else
-        USE_PGXS=1 make -C "${d}" installcheck || FAILED="${d} ${FAILED}"
-      fi
-  done
-  [[ -z ${FAILED} ]]  && exit 0
-fi
-for d in ${FAILED} $([[ ! -f ${FAILED_FILE} ]] || cat ${FAILED_FILE}); do
+    if [ -f "${d}/neon-test.sh" ]; then
+       "${d}/neon-test.sh" || FAILED="${d} ${FAILED}"
+    else
+       USE_PGXS=1 make -C "${d}" installcheck || FAILED="${d} ${FAILED}"
+    fi
+done
+[ -z "${FAILED}" ] && exit 0
+for d in ${FAILED}; do
   cat "$(find $d -name regression.diffs)"
 done
 for postgis_diff in /tmp/pgis_reg/*_diff; do
@@ -92,5 +68,4 @@ for postgis_diff in /tmp/pgis_reg/*_diff; do
   cat "${postgis_diff}"
 done
 echo "${FAILED}"
-cat ${FAILED_FILE}
 exit 1

docker-compose/test_extensions_upgrade.sh

@@ -1,4 +1,4 @@
-#!/usr/bin/env bash
+#!/bin/bash
 set -eux -o pipefail
 cd "$(dirname "${0}")"
 # Takes a variable name as argument. The result is stored in that variable.
@@ -60,8 +60,8 @@ function check_timeline() {
 # Restarts the compute node with the required compute tag and timeline.
 # Accepts the tag for the compute node and the timeline as parameters.
 function restart_compute() {
-  docker compose down compute1 compute_is_ready
-  COMPUTE_TAG=${1} TENANT_ID=${tenant_id} TIMELINE_ID=${2} docker compose up --quiet-pull -d --build compute1 compute_is_ready
+  docker compose down compute compute_is_ready
+  COMPUTE_TAG=${1} TENANT_ID=${tenant_id} TIMELINE_ID=${2} docker compose up --quiet-pull -d --build compute compute_is_ready
   wait_for_ready
   check_timeline ${2}
 }

docs/core_changes.md

@@ -129,10 +129,9 @@ segment to bootstrap the WAL writing, but it doesn't contain the checkpoint reco
 changes in xlog.c, to allow starting the compute node without reading the last checkpoint record
 from WAL.
 
-This includes code to read the `neon.signal` (also `zenith.signal`) file, which tells the startup 
-code the LSN to start at. When the `neon.signal` file is present, the startup uses that LSN
-instead of the last checkpoint's LSN. The system is known to be consistent at that LSN, without 
-any WAL redo.
+This includes code to read the `zenith.signal` file, which tells the startup code the LSN to start
+at. When the `zenith.signal` file is present, the startup uses that LSN instead of the last
+checkpoint's LSN. The system is known to be consistent at that LSN, without any WAL redo.
 
 
 ### How to get rid of the patch

docs/rfcs/035-safekeeper-dynamic-membership-change.md

@@ -20,7 +20,7 @@ In our case consensus leader is compute (walproposer), and we don't want to wake
 up all computes for the change. Neither we want to fully reimplement the leader
 logic second time outside compute. Because of that the proposed algorithm relies
 for issuing configurations on the external fault tolerant (distributed) strongly
-consistent storage with simple API: CAS (compare-and-swap) on the single key.
+consisent storage with simple API: CAS (compare-and-swap) on the single key.
 Properly configured postgres suits this.
 
 In the system consensus is implemented at the timeline level, so algorithm below
@@ -34,7 +34,7 @@ A configuration is
 
 ```
 struct Configuration {
-    generation: SafekeeperGeneration, // a number uniquely identifying configuration
+    generation: Generation, // a number uniquely identifying configuration
     sk_set: Vec<NodeId>, // current safekeeper set
     new_sk_set: Optional<Vec<NodeId>>,
 }
@@ -81,11 +81,11 @@ configuration generation in them is less than its current one. Namely, it
 refuses to vote, to truncate WAL in `handle_elected` and to accept WAL. In
 response it sends its current configuration generation to let walproposer know.
 
-Safekeeper gets `PUT /v1/tenants/{tenant_id}/timelines/{timeline_id}/membership`
-accepting `Configuration`. Safekeeper switches to the given conf if it is higher than its
+Safekeeper gets `PUT /v1/tenants/{tenant_id}/timelines/{timeline_id}/configuration`
+accepting `Configuration`. Safekeeper switches to the given conf it is higher than its
 current one and ignores it otherwise. In any case it replies with
 ```
-struct TimelineMembershipSwitchResponse {
+struct ConfigurationSwitchResponse {
     conf: Configuration,
     term: Term,
     last_log_term: Term,
@@ -108,7 +108,7 @@ establishes this configuration as its own and moves to voting.
 It should stop talking to safekeepers not listed in the configuration at this
 point, though it is not unsafe to continue doing so.
 
-To be elected it must receive votes from both majorities if `new_sk_set` is present.
+To be elected it must receive votes from both majorites if `new_sk_set` is present.
 Similarly, to commit WAL it must receive flush acknowledge from both majorities.
 
 If walproposer hears from safekeeper configuration higher than his own (i.e.
@@ -130,7 +130,7 @@ storage are reachable.
 1) Fetch current timeline configuration from the configuration storage.
 2) If it is already joint one and `new_set` is different from `desired_set`
    refuse to change. However, assign join conf to (in memory) var
-   `joint_conf` and proceed to step 4 to finish the ongoing change.
+   `join_conf` and proceed to step 4 to finish the ongoing change.
 3) Else, create joint `joint_conf: Configuration`: increment current conf number
    `n` and put `desired_set` to `new_sk_set`. Persist it in the configuration
    storage by doing CAS on the current generation: change happens only if
@@ -161,11 +161,11 @@ storage are reachable.
    because `pull_timeline` already includes it and plus additionally would be
    broadcast by compute. More importantly, we may proceed to the next step
    only when `<last_log_term, flush_lsn>` on the majority of the new set reached
-   `sync_position`. Similarly, on the happy path no waiting is needed because
+   `sync_position`. Similarly, on the happy path no waiting is not needed because
    `pull_timeline` already includes it. However, we should double
     check to be safe. For example, timeline could have been created earlier e.g.
     manually or after try-to-migrate, abort, try-to-migrate-again sequence.
-7) Create `new_conf: Configuration` incrementing `joint_conf` generation and having new
+7) Create `new_conf: Configuration` incrementing `join_conf` generation and having new
    safekeeper set as `sk_set` and None `new_sk_set`. Write it to configuration
    storage under one more CAS.
 8) Call `PUT` `configuration` on safekeepers from the new set,
@@ -178,12 +178,12 @@ spec of it.
 
 Description above focuses on safety. To make the flow practical and live, here a few more
 considerations.
-1) It makes sense to ping new set to ensure we are migrating to live node(s) before
+1) It makes sense to ping new set to ensure it we are migrating to live node(s) before
   step 3.
 2) If e.g. accidentally wrong new sk set has been specified, before CAS in step `6` is completed
    it is safe to rollback to the old conf with one more CAS.
 3) On step 4 timeline might be already created on members of the new set for various reasons;
-   the simplest is the procedure restart. There are more complicated scenarios like mentioned
+   the simplest is the procedure restart. There are more complicated scenarious like mentioned
    in step 5. Deleting and re-doing `pull_timeline` is generally unsafe without involving
    generations, so seems simpler to treat existing timeline as success. However, this also
    has a disadvantage: you might imagine an surpassingly unlikely schedule where condition in
@@ -192,7 +192,7 @@ considerations.
 4) In the end timeline should be locally deleted on the safekeeper(s) which are
    in the old set but not in the new one, unless they are unreachable. To be
    safe this also should be done under generation number (deletion proceeds only if
-   current configuration is <= than one in request and safekeeper is not member of it).
+   current configuration is <= than one in request and safekeeper is not memeber of it).
 5) If current conf fetched on step 1 is already not joint and members equal to `desired_set`,
    jump to step 7, using it as `new_conf`.
 
@@ -261,14 +261,14 @@ Timeline (branch) creation in cplane should call storage_controller POST
 Response should be augmented with `safekeepers_generation` and `safekeepers`
 fields like described in `/notify-safekeepers` above. Initially (currently)
 these fields may be absent; in this case cplane chooses safekeepers on its own
-like it currently does. The call should be retried until it succeeds.
+like it currently does. The call should be retried until succeeds.
 
 Timeline deletion and tenant deletion in cplane should call appropriate
 storage_controller endpoints like it currently does for sharded tenants. The
 calls should be retried until they succeed.
 
-When compute receives safekeeper list from control plane it needs to know the
-generation to check whether it should be updated (note that compute may get
+When compute receives safekeepers list from control plane it needs to know the
+generation to checked whether it should be updated (note that compute may get
 safekeeper list from either cplane or safekeepers). Currently `neon.safekeepers`
 GUC is just a comma separates list of `host:port`. Let's prefix it with
 `g#<generation>:` to this end, so it will look like
@@ -305,8 +305,8 @@ enum MigrationRequest {
 ```
 
 `FinishPending` requests to run the procedure to ensure state is clean: current
-configuration is not joint and the majority of safekeepers are aware of it, but do
-not attempt to migrate anywhere. If the current configuration fetched on step 1 is
+configuration is not joint and majority of safekeepers are aware of it, but do
+not attempt to migrate anywhere. If current configuration fetched on step 1 is
 not joint it jumps to step 7. It should be run at startup for all timelines (but
 similarly, in the first version it is ok to trigger it manually).
 
@@ -315,7 +315,7 @@ similarly, in the first version it is ok to trigger it manually).
 `safekeepers` table mirroring current `nodes` should be added, except that for
 `scheduling_policy`: it is enough to have at least in the beginning only 3
 fields: 1) `active` 2) `paused` (initially means only not assign new tlis there
-3) `decommissioned` (node is removed).
+3) `decomissioned` (node is removed).
 
 `timelines` table:
 ```
@@ -326,10 +326,9 @@ table! {
         tenant_id -> Varchar,
         start_lsn -> pg_lsn,
         generation -> Int4,
-        sk_set -> Array<Int8>, // list of safekeeper ids
+        sk_set -> Array<Int4>, // list of safekeeper ids
         new_sk_set -> Nullable<Array<Int8>>, // list of safekeeper ids, null if not joint conf
         cplane_notified_generation -> Int4,
-        sk_set_notified_generation -> Int4, // the generation a quorum of sk_set knows about
         deleted_at -> Nullable<Timestamptz>,
     }
 }
@@ -339,23 +338,13 @@ table! {
 might also want to add ancestor_timeline_id to preserve the hierarchy, but for
 this RFC it is not needed.
 
-`cplane_notified_generation` and `sk_set_notified_generation` fields are used to
-track the last stage of the algorithm, when we need to notify safekeeper set and cplane
-with the final configuration after it's already committed to DB.
-
-The timeline is up-to-date (no migration in progress) if `new_sk_set` is null and
-`*_notified_generation` fields are up to date with `generation`. 
-
-It's possible to replace `*_notified_generation` with one boolean field `migration_completed`,
-but for better observability it's nice to have them separately.
-
 #### API
 
 Node management is similar to pageserver:
-1) POST `/control/v1/safekeeper` inserts safekeeper.
-2) GET `/control/v1/safekeeper` lists safekeepers.
-3) GET `/control/v1/safekeeper/:node_id` gets safekeeper.
-4) PUT `/control/v1/safekeper/:node_id/scheduling_policy` changes status to e.g.
+1) POST `/control/v1/safekeepers` inserts safekeeper.
+2) GET `/control/v1/safekeepers` lists safekeepers.
+3) GET `/control/v1/safekeepers/:node_id` gets safekeeper.
+4) PUT `/control/v1/safekepers/:node_id/status` changes status to e.g.
    `offline` or `decomissioned`. Initially it is simpler not to schedule any
     migrations here.
 
@@ -379,8 +368,8 @@ Migration API: the first version is the simplest and the most imperative:
 all timelines from one safekeeper to another. It accepts json
 ```
 {
-    "src_sk": NodeId,
-    "dst_sk": NodeId,
+    "src_sk": u32,
+    "dst_sk": u32,
     "limit": Optional<u32>,
 }
 ```
@@ -390,15 +379,12 @@ Returns list of scheduled requests.
 2) PUT `/control/v1/tenant/:tenant_id/timeline/:timeline_id/safekeeper_migrate` schedules `MigrationRequest`
    to move single timeline to given set of safekeepers:
 ```
-struct TimelineSafekeeperMigrateRequest {
-    "new_sk_set": Vec<NodeId>,
+{
+    "desired_set": Vec<u32>,
 }
 ```
 
-In the first version the handler migrates the timeline to `new_sk_set` synchronously.
-Should be retried until success.
-
-In the future we might change it to asynchronous API and return scheduled request.
+Returns scheduled request.
 
 Similar call should be added for the tenant.
 
@@ -448,9 +434,6 @@ table! {
 }
 ```
 
-We load all pending ops from the table on startup into the memory.
-The table is needed only to preserve the state between restarts.
-
 `op_type` can be `include` (seed from peers and ensure generation is up to
 date), `exclude` (remove locally) and `delete`. Field is actually not strictly
 needed as it can be computed from current configuration, but gives more explicit
@@ -491,7 +474,7 @@ actions must be idempotent. Now, a tricky point here is timeline start LSN. For
 the initial (tenant creation) call cplane doesn't know it. However, setting
 start_lsn on safekeepers during creation is a good thing -- it provides a
 guarantee that walproposer can always find a common point in WAL histories of
-safekeeper and its own, and so absence of it would be a clear sign of
+safekeeper and its own, and so absense of it would be a clear sign of
 corruption. The following sequence works:
 1) Create timeline (or observe that it exists) on pageserver,
    figuring out last_record_lsn in response.
@@ -514,9 +497,11 @@ corruption. The following sequence works:
    retries the call until 200 response.
 
    There is a small question how request handler (timeline creation in this
-   case) would interact with per sk reconciler. In the current implementation
-   we first persist the request in the DB, and then send an in-memory request
-   to each safekeeper reconciler to process it.
+   case) would interact with per sk reconciler. As always I prefer to do the
+   simplest possible thing and here it seems to be just waking it up so it
+   re-reads the db for work to do. Passing work in memory is faster, but
+   that shouldn't matter, and path to scan db for work will exist anyway, 
+   simpler to reuse it.
 
 For pg version / wal segment size: while we may persist them in `timelines`
 table, it is not necessary as initial creation at step 3 can take them from
@@ -524,40 +509,30 @@ pageserver or cplane creation call and later pull_timeline will carry them
 around.
 
 Timeline migration.
-1) CAS to the db to create joint conf. Since this moment the migration is considered to be 
-   "in progress". We can detect all "in-progress" migrations looking into the database.
-2) Do steps 4-6 from the algorithm, including `pull_timeline` onto `new_sk_set`, update membership
-   configuration on all safekeepers, notify cplane, etc. All operations are idempotent,
-   so we don't need to persist anything in the database at this stage. If any errors occur,
-   it's safe to retry or abort the migration.
-3) Once it becomes possible per alg description above, get out of joint conf
-   with another CAS. Also should insert `exclude` entries into `safekeeper_timeline_pending_ops`
-   in the same DB transaction. Adding `exclude` entries atomically is nesessary because after
-   CAS we don't have the list of excluded safekeepers in the `timelines` table anymore, but we
-   need to have them persisted somewhere in case the migration is interrupted right after the CAS.
-4) Finish the migration. The final membership configuration is committed to the DB at this stage.
-   So, the migration can not be aborted anymore. But it can still be retried if the migration fails
-   past stage 3. To finish the migration we need to send the new membership configuration to
-   a new quorum of safekeepers, notify cplane with the new safekeeper list and schedule the `exclude`
-   requests to in-memory queue for safekeeper reconciler. If the algrorithm is retried, it's
-   possible that we have already committed `exclude` requests to DB, but didn't send them to
-   the in-memory queue. In this case we need to read them from `safekeeper_timeline_pending_ops`
-   because it's the only place where they are persistent. The fields `sk_set_notified_generation`
-   and `cplane_notified_generation` are updated after each step. The migration is considered
-   fully completed when they match the `generation` field.
-
-In practice, we can report "success" after stage 3 and do the "finish" step in per-timeline
-reconciler (if we implement it). But it's wise to at least try to finish them synchronously,
-so the timeline is always in a "good state" and doesn't require an old quorum to commit
-WAL after the migration reported "success".
+1) CAS to the db to create joint conf, and in the same transaction create
+   `safekeeper_timeline_pending_ops` `include` entries to initialize new members
+   as well as deliver this conf to current ones; poke per sk reconcilers to work
+   on it. Also any conf change should also poke cplane notifier task(s).
+2) Once it becomes possible per alg description above, get out of joint conf
+   with another CAS. Task should get wakeups from per sk reconcilers because 
+   conf switch is required for advancement; however retries should be sleep
+   based as well as LSN advancement might be needed, though in happy path 
+   it isn't. To see whether further transition is possible on wakup migration
+   executor polls safekeepers per the algorithm. CAS creating new conf with only
+   new members should again insert entries to `safekeeper_timeline_pending_ops`
+   to switch them there, as well as `exclude` rows to remove timeline from 
+   old members.
 
 Timeline deletion: just set `deleted_at` on the timeline row and insert
 `safekeeper_timeline_pending_ops` entries in the same xact, the rest is done by
 per sk reconcilers.
 
-When node is removed (set to `decommissioned`), `safekeeper_timeline_pending_ops`
+When node is removed (set to `decomissioned`), `safekeeper_timeline_pending_ops`
 for it must be cleared in the same transaction.
 
+One more task pool should infinitely retry notifying control plane about changed
+safekeeper sets (trying making `cplane_notified_generation` equal `generation`).
+
 #### Dealing with multiple instances of storage_controller
 
 Operations described above executed concurrently might create some errors but do
@@ -566,7 +541,7 @@ of storage_controller it is fine to have it temporarily, e.g. during redeploy.
 
 To harden against some controller instance creating some work in
 `safekeeper_timeline_pending_ops` and then disappearing without anyone pickup up
-the job per sk reconcilers apart from explicit wakeups should scan for work
+the job per sk reconcilers apart from explicit wakups should scan for work
 periodically. It is possible to remove that though if all db updates are
 protected with leadership token/term -- then such scans are needed only after
 leadership is acquired.
@@ -588,7 +563,7 @@ There should be following layers of tests:
    safekeeper communication and pull_timeline need to be mocked and main switch
    procedure wrapped to as a node (thread) in simulation tests, using these
    mocks. Test would inject migrations like it currently injects
-   safekeeper/walproposer restarts. Main assert is the same -- committed WAL must
+   safekeeper/walproposer restars. Main assert is the same -- committed WAL must
    not be lost.
 
 3) Since simulation testing injects at relatively high level points (not
@@ -638,7 +613,7 @@ Let's have the following implementation bits for gradual rollout:
   `notify-safekeepers`.
 
 Then the rollout for a region would be:
-- Current situation: safekeepers are chosen by control_plane.
+- Current situation: safekeepers are choosen by control_plane.
 - We manually migrate some timelines, test moving them around.
 - Then we enable `--set-safekeepers` so that all new timelines
   are on storage controller.

endpoint_storage/src/app.rs

@@ -13,8 +13,6 @@ use utils::backoff::retry;
 pub fn app(state: Arc<Storage>) -> Router<()> {
     use axum::routing::{delete as _delete, get as _get};
     let delete_prefix = _delete(delete_prefix);
-    // NB: On any changes do not forget to update the OpenAPI spec
-    // in /endpoint_storage/src/openapi_spec.yml.
     Router::new()
         .route(
             "/{tenant_id}/{timeline_id}/{endpoint_id}/{*path}",

endpoint_storage/src/openapi_spec.yml

@@ -1,146 +0,0 @@
-openapi: "3.0.2"
-info:
-  title: Endpoint Storage API
-  description: Endpoint Storage API
-  version: "1.0"
-  license:
-    name: "Apache"
-    url: https://github.com/neondatabase/neon/blob/main/LICENSE
-servers:
-  - url: ""
-paths:
-  /status:
-    description: Healthcheck endpoint
-    get:
-      description: Healthcheck
-      security: []
-      responses:
-        "200":
-          description: OK
-
-  /{tenant_id}/{timeline_id}/{endpoint_id}/{key}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-      - name: endpoint_id
-        in: path
-        required: true
-        schema:
-          type: string
-      - name: key
-        in: path
-        required: true
-        schema:
-          type: string
-    get:
-      description: Get file from blob storage
-      responses:
-        "200":
-          description: "File stream from blob storage"
-          content:
-            application/octet-stream:
-              schema:
-                type: string
-                format: binary
-        "400":
-          description: File was not found
-        "403":
-          description: JWT does not authorize request to this route
-    put:
-      description: Insert file into blob storage. If file exists, override it
-      requestBody:
-        content:
-          application/octet-stream:
-            schema:
-              type: string
-              format: binary
-      responses:
-        "200":
-          description: File was inserted successfully
-        "403":
-          description: JWT does not authorize request to this route
-    delete:
-      description: Delete file from blob storage
-      responses:
-        "200":
-          description: File was successfully deleted or not found
-        "403":
-          description: JWT does not authorize request to this route
-
-  /{tenant_id}/{timeline_id}/{endpoint_id}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-      - name: endpoint_id
-        in: path
-        required: true
-        schema:
-          type: string
-    delete:
-      description: Delete endpoint data from blob storage
-      responses:
-        "200":
-          description: Endpoint data was deleted
-        "403":
-          description: JWT does not authorize request to this route
-
-  /{tenant_id}/{timeline_id}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-    delete:
-      description: Delete timeline data from blob storage
-      responses:
-        "200":
-          description: Timeline data was deleted
-        "403":
-          description: JWT does not authorize request to this route
-
-  /{tenant_id}:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-    delete:
-      description: Delete tenant data from blob storage
-      responses:
-        "200":
-          description: Tenant data was deleted
-        "403":
-          description: JWT does not authorize request to this route
-
-components:
-  securitySchemes:
-    JWT:
-      type: http
-      scheme: bearer
-      bearerFormat: JWT
-
-security:
-  - JWT: []

libs/compute_api/src/responses.rs

@@ -46,7 +46,7 @@ pub struct ExtensionInstallResponse {
     pub version: ExtVersion,
 }
 
-#[derive(Serialize, Default, Debug, Clone, PartialEq)]
+#[derive(Serialize, Default, Debug, Clone)]
 #[serde(tag = "status", rename_all = "snake_case")]
 pub enum LfcPrewarmState {
     #[default]
@@ -58,18 +58,7 @@ pub enum LfcPrewarmState {
     },
 }
 
-impl Display for LfcPrewarmState {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            LfcPrewarmState::NotPrewarmed => f.write_str("NotPrewarmed"),
-            LfcPrewarmState::Prewarming => f.write_str("Prewarming"),
-            LfcPrewarmState::Completed => f.write_str("Completed"),
-            LfcPrewarmState::Failed { error } => write!(f, "Error({error})"),
-        }
-    }
-}
-
-#[derive(Serialize, Default, Debug, Clone, PartialEq)]
+#[derive(Serialize, Default, Debug, Clone)]
 #[serde(tag = "status", rename_all = "snake_case")]
 pub enum LfcOffloadState {
     #[default]
@@ -81,23 +70,6 @@ pub enum LfcOffloadState {
     },
 }
 
-#[derive(Serialize, Debug, Clone, PartialEq)]
-#[serde(tag = "status", rename_all = "snake_case")]
-/// Response of /promote
-pub enum PromoteState {
-    NotPromoted,
-    Completed,
-    Failed { error: String },
-}
-
-#[derive(Deserialize, Serialize, Default, Debug, Clone)]
-#[serde(rename_all = "snake_case")]
-/// Result of /safekeepers_lsn
-pub struct SafekeepersLsn {
-    pub safekeepers: String,
-    pub wal_flush_lsn: utils::lsn::Lsn,
-}
-
 /// Response of the /status API
 #[derive(Serialize, Debug, Deserialize)]
 #[serde(rename_all = "snake_case")]
@@ -121,15 +93,6 @@ pub enum TerminateMode {
     Immediate,
 }
 
-impl From<TerminateMode> for ComputeStatus {
-    fn from(mode: TerminateMode) -> Self {
-        match mode {
-            TerminateMode::Fast => ComputeStatus::TerminationPendingFast,
-            TerminateMode::Immediate => ComputeStatus::TerminationPendingImmediate,
-        }
-    }
-}
-
 #[derive(Serialize, Clone, Copy, Debug, Deserialize, PartialEq, Eq)]
 #[serde(rename_all = "snake_case")]
 pub enum ComputeStatus {
@@ -150,9 +113,7 @@ pub enum ComputeStatus {
     // control-plane to terminate it.
     Failed,
     // Termination requested
-    TerminationPendingFast,
-    // Termination requested, without waiting 30s before returning from /terminate
-    TerminationPendingImmediate,
+    TerminationPending { mode: TerminateMode },
     // Terminated Postgres
     Terminated,
 }
@@ -171,10 +132,7 @@ impl Display for ComputeStatus {
             ComputeStatus::Running => f.write_str("running"),
             ComputeStatus::Configuration => f.write_str("configuration"),
             ComputeStatus::Failed => f.write_str("failed"),
-            ComputeStatus::TerminationPendingFast => f.write_str("termination-pending-fast"),
-            ComputeStatus::TerminationPendingImmediate => {
-                f.write_str("termination-pending-immediate")
-            }
+            ComputeStatus::TerminationPending { .. } => f.write_str("termination-pending"),
             ComputeStatus::Terminated => f.write_str("terminated"),
         }
     }

libs/compute_api/src/spec.rs

@@ -181,14 +181,10 @@ pub struct ComputeSpec {
     /// JWT for authorizing requests to endpoint storage service
     pub endpoint_storage_token: Option<String>,
 
+    /// Download LFC state from endpoint_storage and pass it to Postgres on startup
     #[serde(default)]
-    /// Download LFC state from endpoint storage and pass it to Postgres on compute startup
     pub autoprewarm: bool,
 
-    #[serde(default)]
-    /// Upload LFC state to endpoint storage periodically. Default value (None) means "don't upload"
-    pub offload_lfc_interval_seconds: Option<std::num::NonZeroU64>,
-
     /// Suspend timeout in seconds.
     ///
     /// We use this value to derive other values, such as the installed extensions metric.
@@ -442,7 +438,7 @@ pub struct JwksSettings {
 }
 
 /// Protocol used to connect to a Pageserver. Parsed from the connstring scheme.
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Copy, Debug, Default)]
 pub enum PageserverProtocol {
     /// The original protocol based on libpq and COPY. Uses postgresql:// or postgres:// scheme.
     #[default]

libs/compute_api/tests/cluster_spec.json

@@ -90,11 +90,6 @@
                 "value": "off",
                 "vartype": "bool"
             },
-            {
-                "name": "offload_lfc_interval_seconds",
-                "value": "20",
-                "vartype": "integer"
-            },
             {
                 "name": "neon.safekeepers",
                 "value": "127.0.0.1:6502,127.0.0.1:6503,127.0.0.1:6501",

libs/http-utils/src/endpoint.rs

@@ -20,7 +20,6 @@ use tokio_stream::wrappers::ReceiverStream;
 use tokio_util::io::ReaderStream;
 use tracing::{Instrument, debug, info, info_span, warn};
 use utils::auth::{AuthError, Claims, SwappableJwtAuth};
-use utils::metrics_collector::{METRICS_COLLECTOR, METRICS_STALE_MILLIS};
 
 use crate::error::{ApiError, api_error_handler, route_error_handler};
 use crate::request::{get_query_param, parse_query_param};
@@ -251,28 +250,9 @@ impl std::io::Write for ChannelWriter {
     }
 }
 
-pub async fn prometheus_metrics_handler(
-    req: Request<Body>,
-    force_metric_collection_on_scrape: bool,
-) -> Result<Response<Body>, ApiError> {
+pub async fn prometheus_metrics_handler(_req: Request<Body>) -> Result<Response<Body>, ApiError> {
     SERVE_METRICS_COUNT.inc();
 
-    // HADRON
-    let requested_use_latest = parse_query_param(&req, "use_latest")?;
-
-    let use_latest = match requested_use_latest {
-        None => force_metric_collection_on_scrape,
-        Some(true) => true,
-        Some(false) => {
-            if force_metric_collection_on_scrape {
-                // We don't cache in this case
-                true
-            } else {
-                false
-            }
-        }
-    };
-
     let started_at = std::time::Instant::now();
 
     let (tx, rx) = mpsc::channel(1);
@@ -297,18 +277,12 @@ pub async fn prometheus_metrics_handler(
 
         let _span = span.entered();
 
-        // HADRON
-        let collected = if use_latest {
-            // Skip caching the results if we always force metric collection on scrape.
-            METRICS_COLLECTOR.run_once(!force_metric_collection_on_scrape)
-        } else {
-            METRICS_COLLECTOR.last_collected()
-        };
+        let metrics = metrics::gather();
 
         let gathered_at = std::time::Instant::now();
 
         let res = encoder
-            .encode(&collected.metrics, &mut writer)
+            .encode(&metrics, &mut writer)
             .and_then(|_| writer.flush().map_err(|e| e.into()));
 
         // this instant is not when we finally got the full response sent, sending is done by hyper
@@ -321,10 +295,6 @@ pub async fn prometheus_metrics_handler(
         let encoded_in = encoded_at - gathered_at - writer.wait_time();
         let total = encoded_at - started_at;
 
-        // HADRON
-        let staleness_ms = (encoded_at - collected.collected_at).as_millis();
-        METRICS_STALE_MILLIS.set(staleness_ms as i64);
-
         match res {
             Ok(()) => {
                 tracing::info!(
@@ -333,7 +303,6 @@ pub async fn prometheus_metrics_handler(
                     spawning_ms = spawned_in.as_millis(),
                     collection_ms = collected_in.as_millis(),
                     encoding_ms = encoded_in.as_millis(),
-                    stalenss_ms = staleness_ms,
                     "responded /metrics"
                 );
             }

libs/http-utils/src/request.rs

@@ -41,35 +41,17 @@ pub fn get_query_param<'a>(
         Some(q) => q,
         None => return Ok(None),
     };
-    let values = url::form_urlencoded::parse(query.as_bytes())
+    let mut values = url::form_urlencoded::parse(query.as_bytes())
         .filter_map(|(k, v)| if k == param_name { Some(v) } else { None })
         // we call .next() twice below. If it's None the first time, .fuse() ensures it's None afterwards
         .fuse();
 
-    // Work around an issue with Alloy's pyroscope scrape where the "seconds"
-    // parameter is added several times. https://github.com/grafana/alloy/issues/3026
-    // TODO: revert after Alloy is fixed.
-    let value1 = values
-        .map(Ok)
-        .reduce(|acc, i| {
-            match acc {
-                Err(_) => acc,
-
-                // It's okay to have duplicates as along as they have the same value.
-                Ok(ref a) if a == &i.unwrap() => acc,
-
-                _ => Err(ApiError::BadRequest(anyhow!(
-                    "param {param_name} specified more than once"
-                ))),
-            }
-        })
-        .transpose()?;
-    // if values.next().is_some() {
-    //     return Err(ApiError::BadRequest(anyhow!(
-    //         "param {param_name} specified more than once"
-    //     )));
-    // }
-
+    let value1 = values.next();
+    if values.next().is_some() {
+        return Err(ApiError::BadRequest(anyhow!(
+            "param {param_name} specified more than once"
+        )));
+    }
     Ok(value1)
 }
 
@@ -110,39 +92,3 @@ pub async fn ensure_no_body(request: &mut Request<Body>) -> Result<(), ApiError>
         None => Ok(()),
     }
 }
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[test]
-    fn test_get_query_param_duplicate() {
-        let req = Request::builder()
-            .uri("http://localhost:12345/testuri?testparam=1")
-            .body(hyper::Body::empty())
-            .unwrap();
-        let value = get_query_param(&req, "testparam").unwrap();
-        assert_eq!(value.unwrap(), "1");
-
-        let req = Request::builder()
-            .uri("http://localhost:12345/testuri?testparam=1&testparam=1")
-            .body(hyper::Body::empty())
-            .unwrap();
-        let value = get_query_param(&req, "testparam").unwrap();
-        assert_eq!(value.unwrap(), "1");
-
-        let req = Request::builder()
-            .uri("http://localhost:12345/testuri")
-            .body(hyper::Body::empty())
-            .unwrap();
-        let value = get_query_param(&req, "testparam").unwrap();
-        assert!(value.is_none());
-
-        let req = Request::builder()
-            .uri("http://localhost:12345/testuri?testparam=1&testparam=2&testparam=3")
-            .body(hyper::Body::empty())
-            .unwrap();
-        let value = get_query_param(&req, "testparam");
-        assert!(value.is_err());
-    }
-}

libs/pageserver_api/src/config.rs

@@ -5,7 +5,6 @@ mod tests;
 
 use const_format::formatcp;
 use posthog_client_lite::PostHogClientConfig;
-use utils::serde_percent::Percent;
 pub const DEFAULT_PG_LISTEN_PORT: u16 = 64000;
 pub const DEFAULT_PG_LISTEN_ADDR: &str = formatcp!("127.0.0.1:{DEFAULT_PG_LISTEN_PORT}");
 pub const DEFAULT_HTTP_LISTEN_PORT: u16 = 9898;
@@ -224,9 +223,8 @@ pub struct ConfigToml {
     pub metric_collection_bucket: Option<RemoteStorageConfig>,
     #[serde(with = "humantime_serde")]
     pub synthetic_size_calculation_interval: Duration,
-    pub disk_usage_based_eviction: DiskUsageEvictionTaskConfig,
+    pub disk_usage_based_eviction: Option<DiskUsageEvictionTaskConfig>,
     pub test_remote_failures: u64,
-    pub test_remote_failures_probability: u64,
     pub ondemand_download_behavior_treat_error_as_warn: bool,
     #[serde(with = "humantime_serde")]
     pub background_task_maximum_delay: Duration,
@@ -272,13 +270,9 @@ pub struct ConfigToml {
     pub timeline_import_config: TimelineImportConfig,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub basebackup_cache_config: Option<BasebackupCacheConfig>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image_layer_generation_large_timeline_threshold: Option<u64>,
-    pub force_metric_collection_on_scrape: bool,
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
-#[serde(default)]
 pub struct DiskUsageEvictionTaskConfig {
     pub max_usage_pct: utils::serde_percent::Percent,
     pub min_avail_bytes: u64,
@@ -289,21 +283,6 @@ pub struct DiskUsageEvictionTaskConfig {
     /// Select sorting for evicted layers
     #[serde(default)]
     pub eviction_order: EvictionOrder,
-    pub enabled: bool,
-}
-
-impl Default for DiskUsageEvictionTaskConfig {
-    fn default() -> Self {
-        Self {
-            max_usage_pct: Percent::new(80).unwrap(),
-            min_avail_bytes: 2_000_000_000,
-            period: Duration::from_secs(60),
-            #[cfg(feature = "testing")]
-            mock_statvfs: None,
-            eviction_order: EvictionOrder::default(),
-            enabled: true,
-        }
-    }
 }
 
 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
@@ -564,11 +543,6 @@ pub struct TenantConfigToml {
     pub gc_period: Duration,
     // Delta layer churn threshold to create L1 image layers.
     pub image_creation_threshold: usize,
-    // HADRON
-    // When the timeout is reached, PageServer will (1) force compact any remaining L0 deltas and
-    // (2) create image layers if there are any L1 deltas.
-    #[serde(with = "humantime_serde")]
-    pub image_layer_force_creation_period: Option<Duration>,
     // Determines how much history is retained, to allow
     // branching and read replicas at an older point in time.
     // The unit is time.
@@ -764,10 +738,9 @@ impl Default for ConfigToml {
 
             metric_collection_bucket: (None),
 
-            disk_usage_based_eviction: DiskUsageEvictionTaskConfig::default(),
+            disk_usage_based_eviction: (None),
 
             test_remote_failures: (0),
-            test_remote_failures_probability: (100),
 
             ondemand_download_behavior_treat_error_as_warn: (false),
 
@@ -831,8 +804,6 @@ impl Default for ConfigToml {
             },
             basebackup_cache_config: None,
             posthog_config: None,
-            image_layer_generation_large_timeline_threshold: Some(2 * 1024 * 1024 * 1024),
-            force_metric_collection_on_scrape: true,
         }
     }
 }
@@ -926,7 +897,6 @@ impl Default for TenantConfigToml {
             gc_period: humantime::parse_duration(DEFAULT_GC_PERIOD)
                 .expect("cannot parse default gc period"),
             image_creation_threshold: DEFAULT_IMAGE_CREATION_THRESHOLD,
-            image_layer_force_creation_period: None,
             pitr_interval: humantime::parse_duration(DEFAULT_PITR_INTERVAL)
                 .expect("cannot parse default PITR interval"),
             walreceiver_connect_timeout: humantime::parse_duration(

libs/pageserver_api/src/controller_api.rs

@@ -1,6 +1,5 @@
 use std::collections::{HashMap, HashSet};
 use std::fmt::Display;
-use std::net::IpAddr;
 use std::str::FromStr;
 use std::time::{Duration, Instant};
 
@@ -11,7 +10,7 @@ use serde::{Deserialize, Serialize};
 use utils::id::{NodeId, TenantId, TimelineId};
 use utils::lsn::Lsn;
 
-use crate::models::{PageserverUtilization, ShardParameters, TenantConfig, TimelineInfo};
+use crate::models::{PageserverUtilization, ShardParameters, TenantConfig};
 use crate::shard::{ShardStripeSize, TenantShardId};
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -61,11 +60,6 @@ pub struct NodeRegisterRequest {
     pub listen_https_port: Option<u16>,
 
     pub availability_zone_id: AvailabilityZone,
-
-    // Reachable IP address of the PS/SK registering, if known.
-    // Hadron Cluster Coordiantor will update the DNS record of the registering node
-    // with this IP address.
-    pub node_ip_addr: Option<IpAddr>,
 }
 
 #[derive(Serialize, Deserialize)]
@@ -132,13 +126,6 @@ pub struct TenantDescribeResponse {
     pub config: TenantConfig,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
-pub struct TenantTimelineDescribeResponse {
-    pub shards: Vec<TimelineInfo>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image_consistent_lsn: Option<Lsn>,
-}
-
 #[derive(Serialize, Deserialize, Debug)]
 pub struct NodeShardResponse {
     pub node_id: NodeId,
@@ -399,7 +386,6 @@ pub enum NodeSchedulingPolicy {
     Pause,
     PauseForRestart,
     Draining,
-    Deleting,
 }
 
 impl FromStr for NodeSchedulingPolicy {
@@ -412,7 +398,6 @@ impl FromStr for NodeSchedulingPolicy {
             "pause" => Ok(Self::Pause),
             "pause_for_restart" => Ok(Self::PauseForRestart),
             "draining" => Ok(Self::Draining),
-            "deleting" => Ok(Self::Deleting),
             _ => Err(anyhow::anyhow!("Unknown scheduling state '{s}'")),
         }
     }
@@ -427,7 +412,6 @@ impl From<NodeSchedulingPolicy> for String {
             Pause => "pause",
             PauseForRestart => "pause_for_restart",
             Draining => "draining",
-            Deleting => "deleting",
         }
         .to_string()
     }
@@ -436,7 +420,6 @@ impl From<NodeSchedulingPolicy> for String {
 #[derive(Serialize, Deserialize, Clone, Copy, Eq, PartialEq, Debug)]
 pub enum SkSchedulingPolicy {
     Active,
-    Activating,
     Pause,
     Decomissioned,
 }
@@ -447,7 +430,6 @@ impl FromStr for SkSchedulingPolicy {
     fn from_str(s: &str) -> Result<Self, Self::Err> {
         Ok(match s {
             "active" => Self::Active,
-            "activating" => Self::Activating,
             "pause" => Self::Pause,
             "decomissioned" => Self::Decomissioned,
             _ => {
@@ -464,7 +446,6 @@ impl From<SkSchedulingPolicy> for String {
         use SkSchedulingPolicy::*;
         match value {
             Active => "active",
-            Activating => "activating",
             Pause => "pause",
             Decomissioned => "decomissioned",
         }
@@ -551,39 +532,6 @@ pub struct SafekeeperDescribeResponse {
     pub scheduling_policy: SkSchedulingPolicy,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct TimelineSafekeeperPeer {
-    pub node_id: NodeId,
-    pub listen_http_addr: String,
-    pub http_port: i32,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct SCSafekeeperTimeline {
-    // SC does not know the tenant id.
-    pub timeline_id: TimelineId,
-    pub peers: Vec<NodeId>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct SCSafekeeperTimelinesResponse {
-    pub timelines: Vec<SCSafekeeperTimeline>,
-    pub safekeeper_peers: Vec<TimelineSafekeeperPeer>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct SafekeeperTimeline {
-    pub tenant_id: TenantId,
-    pub timeline_id: TimelineId,
-    pub peers: Vec<NodeId>,
-}
-
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct SafekeeperTimelinesResponse {
-    pub timelines: Vec<SafekeeperTimeline>,
-    pub safekeeper_peers: Vec<TimelineSafekeeperPeer>,
-}
-
 #[derive(Serialize, Deserialize, Clone)]
 pub struct SafekeeperSchedulingPolicyRequest {
     pub scheduling_policy: SkSchedulingPolicy,

libs/pageserver_api/src/models.rs

@@ -384,7 +384,7 @@ pub struct SafekeepersInfo {
     pub safekeepers: Vec<SafekeeperInfo>,
 }
 
-#[derive(Serialize, Deserialize, Clone, Debug)]
+#[derive(Serialize, Deserialize, Clone)]
 pub struct SafekeeperInfo {
     pub id: NodeId,
     pub hostname: String,
@@ -597,9 +597,6 @@ pub struct TenantConfigPatch {
     pub gc_period: FieldPatch<String>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub image_creation_threshold: FieldPatch<usize>,
-    // HADRON
-    #[serde(skip_serializing_if = "FieldPatch::is_noop")]
-    pub image_layer_force_creation_period: FieldPatch<String>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
     pub pitr_interval: FieldPatch<String>,
     #[serde(skip_serializing_if = "FieldPatch::is_noop")]
@@ -703,11 +700,6 @@ pub struct TenantConfig {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub image_creation_threshold: Option<usize>,
 
-    // HADRON
-    #[serde(skip_serializing_if = "Option::is_none")]
-    #[serde(with = "humantime_serde")]
-    pub image_layer_force_creation_period: Option<Duration>,
-
     #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(with = "humantime_serde")]
     pub pitr_interval: Option<Duration>,
@@ -806,7 +798,6 @@ impl TenantConfig {
             mut gc_horizon,
             mut gc_period,
             mut image_creation_threshold,
-            mut image_layer_force_creation_period,
             mut pitr_interval,
             mut walreceiver_connect_timeout,
             mut lagging_wal_timeout,
@@ -870,11 +861,6 @@ impl TenantConfig {
         patch
             .image_creation_threshold
             .apply(&mut image_creation_threshold);
-        // HADRON
-        patch
-            .image_layer_force_creation_period
-            .map(|v| humantime::parse_duration(&v))?
-            .apply(&mut image_layer_force_creation_period);
         patch
             .pitr_interval
             .map(|v| humantime::parse_duration(&v))?
@@ -956,7 +942,6 @@ impl TenantConfig {
             gc_horizon,
             gc_period,
             image_creation_threshold,
-            image_layer_force_creation_period,
             pitr_interval,
             walreceiver_connect_timeout,
             lagging_wal_timeout,
@@ -1031,9 +1016,6 @@ impl TenantConfig {
             image_creation_threshold: self
                 .image_creation_threshold
                 .unwrap_or(global_conf.image_creation_threshold),
-            image_layer_force_creation_period: self
-                .image_layer_force_creation_period
-                .or(global_conf.image_layer_force_creation_period),
             pitr_interval: self.pitr_interval.unwrap_or(global_conf.pitr_interval),
             walreceiver_connect_timeout: self
                 .walreceiver_connect_timeout
@@ -1622,9 +1604,6 @@ pub struct TimelineInfo {
 
     /// Whether the timeline is invisible in synthetic size calculations.
     pub is_invisible: Option<bool>,
-    // HADRON: the largest LSN below which all page updates have been included in the image layers.
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub image_consistent_lsn: Option<Lsn>,
 }
 
 #[derive(Debug, Clone, Serialize, Deserialize)]

libs/pageserver_api/src/shard.rs

@@ -332,11 +332,7 @@ fn hash_combine(mut a: u32, mut b: u32) -> u32 {
 ///
 /// The mapping of key to shard is not stable across changes to ShardCount: this is intentional
 /// and will be handled at higher levels when shards are split.
-pub fn key_to_shard_number(
-    count: ShardCount,
-    stripe_size: ShardStripeSize,
-    key: &Key,
-) -> ShardNumber {
+fn key_to_shard_number(count: ShardCount, stripe_size: ShardStripeSize, key: &Key) -> ShardNumber {
     // Fast path for un-sharded tenants or broadcast keys
     if count < ShardCount(2) || key_is_shard0(key) {
         return ShardNumber(0);

libs/postgres_backend/src/lib.rs

@@ -78,13 +78,7 @@ pub fn is_expected_io_error(e: &io::Error) -> bool {
     use io::ErrorKind::*;
     matches!(
         e.kind(),
-        HostUnreachable
-            | NetworkUnreachable
-            | BrokenPipe
-            | ConnectionRefused
-            | ConnectionAborted
-            | ConnectionReset
-            | TimedOut,
+        BrokenPipe | ConnectionRefused | ConnectionAborted | ConnectionReset | TimedOut
     )
 }
 

libs/proxy/json/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-name = "json"
-version = "0.1.0"
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-ryu = "1"
-itoa = "1"
-
-[dev-dependencies]
-futures = "0.3"

libs/proxy/json/src/lib.rs

@@ -1,412 +0,0 @@
-//! A JSON serialization lib, designed for more flexibility than `serde_json` offers.
-//!
-//! Features:
-//!
-//! ## Dynamic construction
-//!
-//! Sometimes you have dynamic values you want to serialize, that are not already in a serde-aware model like a struct or a Vec etc.
-//! To achieve this with serde, you need to implement a lot of different traits on a lot of different new-types.
-//! Because of this, it's often easier to give-in and pull all the data into a serde-aware model (`serde_json::Value` or some intermediate struct),
-//! but that is often not very efficient.
-//!
-//! This crate allows full control over the JSON encoding without needing to implement any extra traits. Just call the
-//! relevant functions, and it will guarantee a correctly encoded JSON value.
-//!
-//! ## Async construction
-//!
-//! Similar to the above, sometimes the values arrive asynchronously. Often collecting those values in memory
-//! is more expensive than writing them as JSON, since the overheads of `Vec` and `String` is much higher, however
-//! there are exceptions.
-//!
-//! Serializing to JSON all in one go is also more CPU intensive and can cause lag spikes,
-//! whereas serializing values incrementally spreads out the CPU load and reduces lag.
-//!
-//! ## Examples
-//!
-//! To represent the following JSON as a compact string
-//!
-//! ```json
-//! {
-//!   "results": {
-//!     "rows": [
-//!       {
-//!         "id": 1,
-//!         "value": null
-//!       },
-//!       {
-//!         "id": 2,
-//!         "value": "hello"
-//!       }
-//!     ]
-//!   }
-//! }
-//! ```
-//!
-//! We can use the following code:
-//!
-//! ```
-//! // create the outer object
-//! let s = json::value_to_string!(|v| json::value_as_object!(|v| {
-//!     // create an entry with key "results" and start an object value associated with it.
-//!     let results = v.key("results");
-//!     json::value_as_object!(|results| {
-//!         // create an entry with key "rows" and start an list value associated with it.
-//!         let rows = results.key("rows");
-//!         json::value_as_list!(|rows| {
-//!             // create a list entry and start an object value associated with it.
-//!             let row = rows.entry();
-//!             json::value_as_object!(|row| {
-//!                 // add entry "id": 1
-//!                 row.entry("id", 1);
-//!                 // add entry "value": null
-//!                 row.entry("value", json::Null);
-//!             });
-//!
-//!             // create a list entry and start an object value associated with it.
-//!             let row = rows.entry();
-//!             json::value_as_object!(|row| {
-//!                 // add entry "id": 2
-//!                 row.entry("id", 2);
-//!                 // add entry "value": "hello"
-//!                 row.entry("value", "hello");
-//!             });
-//!         });
-//!     });
-//! }));
-//!
-//! assert_eq!(s, r#"{"results":{"rows":[{"id":1,"value":null},{"id":2,"value":"hello"}]}}"#);
-//! ```
-
-mod macros;
-mod str;
-mod value;
-
-pub use value::{Null, ValueEncoder};
-
-#[must_use]
-/// Serialize a single json value.
-pub struct ValueSer<'buf> {
-    buf: &'buf mut Vec<u8>,
-    start: usize,
-}
-
-impl<'buf> ValueSer<'buf> {
-    /// Create a new json value serializer.
-    pub fn new(buf: &'buf mut Vec<u8>) -> Self {
-        Self { buf, start: 0 }
-    }
-
-    /// Borrow the underlying buffer
-    pub fn as_buffer(&self) -> &[u8] {
-        self.buf
-    }
-
-    #[inline]
-    pub fn value(self, e: impl ValueEncoder) {
-        e.encode(self);
-    }
-
-    /// Write raw bytes to the buf. This must be already JSON encoded.
-    #[inline]
-    pub fn write_raw_json(self, data: &[u8]) {
-        self.buf.extend_from_slice(data);
-        self.finish();
-    }
-
-    /// Start a new object serializer.
-    #[inline]
-    pub fn object(self) -> ObjectSer<'buf> {
-        ObjectSer::new(self)
-    }
-
-    /// Start a new list serializer.
-    #[inline]
-    pub fn list(self) -> ListSer<'buf> {
-        ListSer::new(self)
-    }
-
-    /// Finish the value ser.
-    #[inline]
-    fn finish(self) {
-        // don't trigger the drop handler which triggers a rollback.
-        // this won't cause memory leaks because `ValueSet` owns no allocations.
-        std::mem::forget(self);
-    }
-}
-
-impl Drop for ValueSer<'_> {
-    fn drop(&mut self) {
-        self.buf.truncate(self.start);
-    }
-}
-
-#[must_use]
-/// Serialize a json object.
-pub struct ObjectSer<'buf> {
-    value: ValueSer<'buf>,
-    start: usize,
-}
-
-impl<'buf> ObjectSer<'buf> {
-    /// Start a new object serializer.
-    #[inline]
-    pub fn new(value: ValueSer<'buf>) -> Self {
-        value.buf.push(b'{');
-        let start = value.buf.len();
-        Self { value, start }
-    }
-
-    /// Borrow the underlying buffer
-    pub fn as_buffer(&self) -> &[u8] {
-        self.value.as_buffer()
-    }
-
-    /// Start a new object entry with the given string key, returning a [`ValueSer`] for the associated value.
-    #[inline]
-    pub fn key(&mut self, key: impl KeyEncoder) -> ValueSer<'_> {
-        key.write_key(self)
-    }
-
-    /// Write an entry (key-value pair) to the object.
-    #[inline]
-    pub fn entry(&mut self, key: impl KeyEncoder, val: impl ValueEncoder) {
-        self.key(key).value(val);
-    }
-
-    #[inline]
-    fn entry_inner(&mut self, f: impl FnOnce(&mut Vec<u8>)) -> ValueSer<'_> {
-        // track before the separator so we the value is rolled back it also removes the separator.
-        let start = self.value.buf.len();
-
-        // push separator if necessary
-        if self.value.buf.len() > self.start {
-            self.value.buf.push(b',');
-        }
-        // push key
-        f(self.value.buf);
-        // push value separator
-        self.value.buf.push(b':');
-
-        // return value writer.
-        ValueSer {
-            buf: self.value.buf,
-            start,
-        }
-    }
-
-    /// Reset the buffer back to before this object was started.
-    #[inline]
-    pub fn rollback(self) -> ValueSer<'buf> {
-        // Do not fully reset the value, only reset it to before the `{`.
-        // This ensures any `,` before this value are not clobbered.
-        self.value.buf.truncate(self.start - 1);
-        self.value
-    }
-
-    /// Finish the object ser.
-    #[inline]
-    pub fn finish(self) {
-        self.value.buf.push(b'}');
-        self.value.finish();
-    }
-}
-
-pub trait KeyEncoder {
-    fn write_key<'a>(self, obj: &'a mut ObjectSer) -> ValueSer<'a>;
-}
-
-#[must_use]
-/// Serialize a json object.
-pub struct ListSer<'buf> {
-    value: ValueSer<'buf>,
-    start: usize,
-}
-
-impl<'buf> ListSer<'buf> {
-    /// Start a new list serializer.
-    #[inline]
-    pub fn new(value: ValueSer<'buf>) -> Self {
-        value.buf.push(b'[');
-        let start = value.buf.len();
-        Self { value, start }
-    }
-
-    /// Borrow the underlying buffer
-    pub fn as_buffer(&self) -> &[u8] {
-        self.value.as_buffer()
-    }
-
-    /// Write an value to the list.
-    #[inline]
-    pub fn push(&mut self, val: impl ValueEncoder) {
-        self.entry().value(val);
-    }
-
-    /// Start a new value entry in this list.
-    #[inline]
-    pub fn entry(&mut self) -> ValueSer<'_> {
-        // track before the separator so we the value is rolled back it also removes the separator.
-        let start = self.value.buf.len();
-
-        // push separator if necessary
-        if self.value.buf.len() > self.start {
-            self.value.buf.push(b',');
-        }
-
-        // return value writer.
-        ValueSer {
-            buf: self.value.buf,
-            start,
-        }
-    }
-
-    /// Reset the buffer back to before this object was started.
-    #[inline]
-    pub fn rollback(self) -> ValueSer<'buf> {
-        // Do not fully reset the value, only reset it to before the `[`.
-        // This ensures any `,` before this value are not clobbered.
-        self.value.buf.truncate(self.start - 1);
-        self.value
-    }
-
-    /// Finish the object ser.
-    #[inline]
-    pub fn finish(self) {
-        self.value.buf.push(b']');
-        self.value.finish();
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use crate::{Null, ValueSer};
-
-    #[test]
-    fn object() {
-        let mut buf = vec![];
-        let mut object = ValueSer::new(&mut buf).object();
-        object.entry("foo", "bar");
-        object.entry("baz", Null);
-        object.finish();
-
-        assert_eq!(buf, br#"{"foo":"bar","baz":null}"#);
-    }
-
-    #[test]
-    fn list() {
-        let mut buf = vec![];
-        let mut list = ValueSer::new(&mut buf).list();
-        list.entry().value("bar");
-        list.entry().value(Null);
-        list.finish();
-
-        assert_eq!(buf, br#"["bar",null]"#);
-    }
-
-    #[test]
-    fn object_macro() {
-        let res = crate::value_to_string!(|obj| {
-            crate::value_as_object!(|obj| {
-                obj.entry("foo", "bar");
-                obj.entry("baz", Null);
-            })
-        });
-
-        assert_eq!(res, r#"{"foo":"bar","baz":null}"#);
-    }
-
-    #[test]
-    fn list_macro() {
-        let res = crate::value_to_string!(|list| {
-            crate::value_as_list!(|list| {
-                list.entry().value("bar");
-                list.entry().value(Null);
-            })
-        });
-
-        assert_eq!(res, r#"["bar",null]"#);
-    }
-
-    #[test]
-    fn rollback_on_drop() {
-        let res = crate::value_to_string!(|list| {
-            crate::value_as_list!(|list| {
-                list.entry().value("bar");
-
-                'cancel: {
-                    let nested_list = list.entry();
-                    crate::value_as_list!(|nested_list| {
-                        nested_list.entry().value(1);
-
-                        assert_eq!(nested_list.as_buffer(), br#"["bar",[1"#);
-                        if true {
-                            break 'cancel;
-                        }
-                    })
-                }
-
-                assert_eq!(list.as_buffer(), br#"["bar""#);
-
-                list.entry().value(Null);
-            })
-        });
-
-        assert_eq!(res, r#"["bar",null]"#);
-    }
-
-    #[test]
-    fn rollback_object() {
-        let res = crate::value_to_string!(|obj| {
-            crate::value_as_object!(|obj| {
-                let entry = obj.key("1");
-                entry.value(1_i32);
-
-                let entry = obj.key("2");
-                let entry = {
-                    let mut nested_obj = entry.object();
-                    nested_obj.entry("foo", "bar");
-                    nested_obj.rollback()
-                };
-
-                entry.value(2_i32);
-            })
-        });
-
-        assert_eq!(res, r#"{"1":1,"2":2}"#);
-    }
-
-    #[test]
-    fn rollback_list() {
-        let res = crate::value_to_string!(|list| {
-            crate::value_as_list!(|list| {
-                let entry = list.entry();
-                entry.value(1_i32);
-
-                let entry = list.entry();
-                let entry = {
-                    let mut nested_list = entry.list();
-                    nested_list.push("foo");
-                    nested_list.rollback()
-                };
-
-                entry.value(2_i32);
-            })
-        });
-
-        assert_eq!(res, r#"[1,2]"#);
-    }
-
-    #[test]
-    fn string_escaping() {
-        let mut buf = vec![];
-        let mut object = ValueSer::new(&mut buf).object();
-
-        let key = "hello";
-        let value = "\n world";
-
-        object.entry(format_args!("{key:?}"), value);
-        object.finish();
-
-        assert_eq!(buf, br#"{"\"hello\"":"\n world"}"#);
-    }
-}

libs/proxy/json/src/macros.rs

@@ -1,86 +0,0 @@
-//! # Examples
-//!
-//! ```
-//! use futures::{StreamExt, TryStream, TryStreamExt};
-//!
-//! async fn stream_to_json_list<S, T, E>(mut s: S) -> Result<String, E>
-//! where
-//!     S: TryStream<Ok = T, Error = E> + Unpin,
-//!     T: json::ValueEncoder
-//! {
-//!     Ok(json::value_to_string!(|val| json::value_as_list!(|val| {
-//!         // note how we can use `.await` and `?` in here.
-//!         while let Some(value) = s.try_next().await? {
-//!             val.push(value);
-//!         }
-//!     })))
-//! }
-//!
-//! let stream = futures::stream::iter([1, 2, 3]).map(Ok::<i32, ()>);
-//! let json_string = futures::executor::block_on(stream_to_json_list(stream)).unwrap();
-//! assert_eq!(json_string, "[1,2,3]");
-//! ```
-
-/// A helper to create a new JSON vec.
-///
-/// Implemented as a macro to preserve all control flow.
-#[macro_export]
-macro_rules! value_to_vec {
-    (|$val:ident| $body:expr) => {{
-        let mut buf = vec![];
-        let $val = $crate::ValueSer::new(&mut buf);
-        let _: () = $body;
-        buf
-    }};
-}
-
-/// A helper to create a new JSON string.
-///
-/// Implemented as a macro to preserve all control flow.
-#[macro_export]
-macro_rules! value_to_string {
-    (|$val:ident| $body:expr) => {{
-        ::std::string::String::from_utf8($crate::value_to_vec!(|$val| $body))
-            .expect("json should be valid utf8")
-    }};
-}
-
-/// A helper that ensures the [`ObjectSer::finish`](crate::ObjectSer::finish) method is called on completion.
-///
-/// Consumes `$val` and assigns it as an [`ObjectSer`](crate::ObjectSer) serializer.
-/// The serializer is only 'finished' if the body completes.
-/// The serializer is rolled back if `break`/`return` escapes the body.
-///
-/// Implemented as a macro to preserve all control flow.
-#[macro_export]
-macro_rules! value_as_object {
-    (|$val:ident| $body:expr) => {{
-        let mut obj = $crate::ObjectSer::new($val);
-
-        let $val = &mut obj;
-        let res = $body;
-
-        obj.finish();
-        res
-    }};
-}
-
-/// A helper that ensures the [`ListSer::finish`](crate::ListSer::finish) method is called on completion.
-///
-/// Consumes `$val` and assigns it as an [`ListSer`](crate::ListSer) serializer.
-/// The serializer is only 'finished' if the body completes.
-/// The serializer is rolled back if `break`/`return` escapes the body.
-///
-/// Implemented as a macro to preserve all control flow.
-#[macro_export]
-macro_rules! value_as_list {
-    (|$val:ident| $body:expr) => {{
-        let mut list = $crate::ListSer::new($val);
-
-        let $val = &mut list;
-        let res = $body;
-
-        list.finish();
-        res
-    }};
-}

libs/proxy/json/src/str.rs

@@ -1,166 +0,0 @@
-//! Helpers for serializing escaped strings.
-//!
-//! ## License
-//!
-//! <https://github.com/serde-rs/json/blob/c1826ebcccb1a520389c6b78ad3da15db279220d/src/ser.rs#L1514-L1552>
-//! <https://github.com/serde-rs/json/blob/c1826ebcccb1a520389c6b78ad3da15db279220d/src/ser.rs#L2081-L2157>
-//! Licensed by David Tolnay under MIT or Apache-2.0.
-//!
-//! With modifications by Conrad Ludgate on behalf of Databricks.
-
-use std::fmt::{self, Write};
-
-/// Represents a character escape code in a type-safe manner.
-pub enum CharEscape {
-    /// An escaped quote `"`
-    Quote,
-    /// An escaped reverse solidus `\`
-    ReverseSolidus,
-    // /// An escaped solidus `/`
-    // Solidus,
-    /// An escaped backspace character (usually escaped as `\b`)
-    Backspace,
-    /// An escaped form feed character (usually escaped as `\f`)
-    FormFeed,
-    /// An escaped line feed character (usually escaped as `\n`)
-    LineFeed,
-    /// An escaped carriage return character (usually escaped as `\r`)
-    CarriageReturn,
-    /// An escaped tab character (usually escaped as `\t`)
-    Tab,
-    /// An escaped ASCII plane control character (usually escaped as
-    /// `\u00XX` where `XX` are two hex characters)
-    AsciiControl(u8),
-}
-
-impl CharEscape {
-    #[inline]
-    fn from_escape_table(escape: u8, byte: u8) -> CharEscape {
-        match escape {
-            self::BB => CharEscape::Backspace,
-            self::TT => CharEscape::Tab,
-            self::NN => CharEscape::LineFeed,
-            self::FF => CharEscape::FormFeed,
-            self::RR => CharEscape::CarriageReturn,
-            self::QU => CharEscape::Quote,
-            self::BS => CharEscape::ReverseSolidus,
-            self::UU => CharEscape::AsciiControl(byte),
-            _ => unreachable!(),
-        }
-    }
-}
-
-pub(crate) fn format_escaped_str(writer: &mut Vec<u8>, value: &str) {
-    writer.reserve(2 + value.len());
-
-    writer.push(b'"');
-
-    let rest = format_escaped_str_contents(writer, value);
-    writer.extend_from_slice(rest);
-
-    writer.push(b'"');
-}
-
-pub(crate) fn format_escaped_fmt(writer: &mut Vec<u8>, args: fmt::Arguments) {
-    writer.push(b'"');
-
-    Collect { buf: writer }
-        .write_fmt(args)
-        .expect("formatting should not error");
-
-    writer.push(b'"');
-}
-
-struct Collect<'buf> {
-    buf: &'buf mut Vec<u8>,
-}
-
-impl fmt::Write for Collect<'_> {
-    fn write_str(&mut self, s: &str) -> fmt::Result {
-        let last = format_escaped_str_contents(self.buf, s);
-        self.buf.extend(last);
-        Ok(())
-    }
-}
-
-// writes any escape sequences, and returns the suffix still needed to be written.
-fn format_escaped_str_contents<'a>(writer: &mut Vec<u8>, value: &'a str) -> &'a [u8] {
-    let bytes = value.as_bytes();
-
-    let mut start = 0;
-
-    for (i, &byte) in bytes.iter().enumerate() {
-        let escape = ESCAPE[byte as usize];
-        if escape == 0 {
-            continue;
-        }
-
-        writer.extend_from_slice(&bytes[start..i]);
-
-        let char_escape = CharEscape::from_escape_table(escape, byte);
-        write_char_escape(writer, char_escape);
-
-        start = i + 1;
-    }
-
-    &bytes[start..]
-}
-
-const BB: u8 = b'b'; // \x08
-const TT: u8 = b't'; // \x09
-const NN: u8 = b'n'; // \x0A
-const FF: u8 = b'f'; // \x0C
-const RR: u8 = b'r'; // \x0D
-const QU: u8 = b'"'; // \x22
-const BS: u8 = b'\\'; // \x5C
-const UU: u8 = b'u'; // \x00...\x1F except the ones above
-const __: u8 = 0;
-
-// Lookup table of escape sequences. A value of b'x' at index i means that byte
-// i is escaped as "\x" in JSON. A value of 0 means that byte i is not escaped.
-static ESCAPE: [u8; 256] = [
-    //   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F
-    UU, UU, UU, UU, UU, UU, UU, UU, BB, TT, NN, UU, FF, RR, UU, UU, // 0
-    UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, // 1
-    __, __, QU, __, __, __, __, __, __, __, __, __, __, __, __, __, // 2
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 3
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 4
-    __, __, __, __, __, __, __, __, __, __, __, __, BS, __, __, __, // 5
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 6
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 7
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 8
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 9
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // A
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // B
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // C
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // D
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // E
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // F
-];
-
-fn write_char_escape(writer: &mut Vec<u8>, char_escape: CharEscape) {
-    let s = match char_escape {
-        CharEscape::Quote => b"\\\"",
-        CharEscape::ReverseSolidus => b"\\\\",
-        // CharEscape::Solidus => b"\\/",
-        CharEscape::Backspace => b"\\b",
-        CharEscape::FormFeed => b"\\f",
-        CharEscape::LineFeed => b"\\n",
-        CharEscape::CarriageReturn => b"\\r",
-        CharEscape::Tab => b"\\t",
-        CharEscape::AsciiControl(byte) => {
-            static HEX_DIGITS: [u8; 16] = *b"0123456789abcdef";
-            let bytes = &[
-                b'\\',
-                b'u',
-                b'0',
-                b'0',
-                HEX_DIGITS[(byte >> 4) as usize],
-                HEX_DIGITS[(byte & 0xF) as usize],
-            ];
-            return writer.extend_from_slice(bytes);
-        }
-    };
-
-    writer.extend_from_slice(s);
-}

libs/proxy/json/src/value.rs

@@ -1,168 +0,0 @@
-use core::fmt;
-use std::collections::{BTreeMap, HashMap};
-
-use crate::str::{format_escaped_fmt, format_escaped_str};
-use crate::{KeyEncoder, ObjectSer, ValueSer, value_as_list, value_as_object};
-
-/// Write a value to the underlying json representation.
-pub trait ValueEncoder {
-    fn encode(self, v: ValueSer<'_>);
-}
-
-pub(crate) fn write_int(x: impl itoa::Integer, b: &mut Vec<u8>) {
-    b.extend_from_slice(itoa::Buffer::new().format(x).as_bytes());
-}
-
-pub(crate) fn write_float(x: impl ryu::Float, b: &mut Vec<u8>) {
-    b.extend_from_slice(ryu::Buffer::new().format(x).as_bytes());
-}
-
-impl<T: Copy + ValueEncoder> ValueEncoder for &T {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        T::encode(*self, v);
-    }
-}
-
-impl ValueEncoder for &str {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        format_escaped_str(v.buf, self);
-        v.finish();
-    }
-}
-
-impl ValueEncoder for fmt::Arguments<'_> {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        if let Some(s) = self.as_str() {
-            format_escaped_str(v.buf, s);
-        } else {
-            format_escaped_fmt(v.buf, self);
-        }
-        v.finish();
-    }
-}
-
-macro_rules! int {
-    [$($t:ty),*] => {
-        $(
-            impl ValueEncoder for $t {
-                #[inline]
-                fn encode(self, v: ValueSer<'_>) {
-                    write_int(self, v.buf);
-                    v.finish();
-                }
-            }
-        )*
-    };
-}
-
-int![u8, u16, u32, u64, usize, u128];
-int![i8, i16, i32, i64, isize, i128];
-
-macro_rules! float {
-    [$($t:ty),*] => {
-        $(
-            impl ValueEncoder for $t {
-                #[inline]
-                fn encode(self, v: ValueSer<'_>) {
-                    write_float(self, v.buf);
-                    v.finish();
-                }
-            }
-        )*
-    };
-}
-
-float![f32, f64];
-
-impl ValueEncoder for bool {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        v.write_raw_json(if self { b"true" } else { b"false" });
-    }
-}
-
-impl<T: ValueEncoder> ValueEncoder for Option<T> {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        match self {
-            Some(value) => value.encode(v),
-            None => Null.encode(v),
-        }
-    }
-}
-
-impl KeyEncoder for &str {
-    #[inline]
-    fn write_key<'a>(self, obj: &'a mut ObjectSer) -> ValueSer<'a> {
-        let obj = &mut *obj;
-        obj.entry_inner(|b| format_escaped_str(b, self))
-    }
-}
-
-impl KeyEncoder for fmt::Arguments<'_> {
-    #[inline]
-    fn write_key<'a>(self, obj: &'a mut ObjectSer) -> ValueSer<'a> {
-        if let Some(key) = self.as_str() {
-            obj.entry_inner(|b| format_escaped_str(b, key))
-        } else {
-            obj.entry_inner(|b| format_escaped_fmt(b, self))
-        }
-    }
-}
-
-/// Represents the JSON null value.
-pub struct Null;
-
-impl ValueEncoder for Null {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        v.write_raw_json(b"null");
-    }
-}
-
-impl<T: ValueEncoder> ValueEncoder for Vec<T> {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        value_as_list!(|v| {
-            for t in self {
-                v.entry().value(t);
-            }
-        });
-    }
-}
-
-impl<T: Copy + ValueEncoder> ValueEncoder for &[T] {
-    #[inline]
-    fn encode(self, v: ValueSer<'_>) {
-        value_as_list!(|v| {
-            for t in self {
-                v.entry().value(t);
-            }
-        });
-    }
-}
-
-impl<K: KeyEncoder, V: ValueEncoder, S> ValueEncoder for HashMap<K, V, S> {
-    #[inline]
-    fn encode(self, o: ValueSer<'_>) {
-        value_as_object!(|o| {
-            for (k, v) in self {
-                o.entry(k, v);
-            }
-        });
-    }
-}
-
-impl<K: KeyEncoder, V: ValueEncoder> ValueEncoder for BTreeMap<K, V> {
-    #[inline]
-    fn encode(self, o: ValueSer<'_>) {
-        value_as_object!(|o| {
-            for (k, v) in self {
-                o.entry(k, v);
-            }
-        });
-    }
-}

libs/proxy/postgres-protocol2/src/authentication/sasl.rs

@@ -52,7 +52,7 @@ pub(crate) async fn hi(str: &[u8], salt: &[u8], iterations: u32) -> [u8; 32] {
         }
         // yield every ~250us
         // hopefully reduces tail latencies
-        if i.is_multiple_of(1024) {
+        if i % 1024 == 0 {
             yield_now().await
         }
     }

libs/proxy/tokio-postgres2/src/client.rs

@@ -90,7 +90,7 @@ pub struct InnerClient {
 }
 
 impl InnerClient {
-    pub fn start(&mut self) -> Result<PartialQuery<'_>, Error> {
+    pub fn start(&mut self) -> Result<PartialQuery, Error> {
         self.responses.waiting += 1;
         Ok(PartialQuery(Some(self)))
     }
@@ -227,7 +227,7 @@ impl Client {
         &mut self,
         statement: &str,
         params: I,
-    ) -> Result<RowStream<'_>, Error>
+    ) -> Result<RowStream, Error>
     where
         S: AsRef<str>,
         I: IntoIterator<Item = Option<S>>,
@@ -262,7 +262,7 @@ impl Client {
     pub(crate) async fn simple_query_raw(
         &mut self,
         query: &str,
-    ) -> Result<SimpleQueryStream<'_>, Error> {
+    ) -> Result<SimpleQueryStream, Error> {
         simple_query::simple_query(self.inner_mut(), query).await
     }
 

libs/proxy/tokio-postgres2/src/generic_client.rs

@@ -12,11 +12,7 @@ mod private {
 /// This trait is "sealed", and cannot be implemented outside of this crate.
 pub trait GenericClient: private::Sealed {
     /// Like `Client::query_raw_txt`.
-    async fn query_raw_txt<S, I>(
-        &mut self,
-        statement: &str,
-        params: I,
-    ) -> Result<RowStream<'_>, Error>
+    async fn query_raw_txt<S, I>(&mut self, statement: &str, params: I) -> Result<RowStream, Error>
     where
         S: AsRef<str> + Sync + Send,
         I: IntoIterator<Item = Option<S>> + Sync + Send,
@@ -26,11 +22,7 @@ pub trait GenericClient: private::Sealed {
 impl private::Sealed for Client {}
 
 impl GenericClient for Client {
-    async fn query_raw_txt<S, I>(
-        &mut self,
-        statement: &str,
-        params: I,
-    ) -> Result<RowStream<'_>, Error>
+    async fn query_raw_txt<S, I>(&mut self, statement: &str, params: I) -> Result<RowStream, Error>
     where
         S: AsRef<str> + Sync + Send,
         I: IntoIterator<Item = Option<S>> + Sync + Send,
@@ -43,11 +35,7 @@ impl GenericClient for Client {
 impl private::Sealed for Transaction<'_> {}
 
 impl GenericClient for Transaction<'_> {
-    async fn query_raw_txt<S, I>(
-        &mut self,
-        statement: &str,
-        params: I,
-    ) -> Result<RowStream<'_>, Error>
+    async fn query_raw_txt<S, I>(&mut self, statement: &str, params: I) -> Result<RowStream, Error>
     where
         S: AsRef<str> + Sync + Send,
         I: IntoIterator<Item = Option<S>> + Sync + Send,

libs/proxy/tokio-postgres2/src/transaction.rs

@@ -47,7 +47,7 @@ impl<'a> Transaction<'a> {
         &mut self,
         statement: &str,
         params: I,
-    ) -> Result<RowStream<'_>, Error>
+    ) -> Result<RowStream, Error>
     where
         S: AsRef<str>,
         I: IntoIterator<Item = Option<S>>,

libs/remote_storage/Cargo.toml

@@ -13,7 +13,6 @@ aws-smithy-async.workspace = true
 aws-smithy-types.workspace = true
 aws-config.workspace = true
 aws-sdk-s3.workspace = true
-base64.workspace = true
 bytes.workspace = true
 camino = { workspace = true, features = ["serde1"] }
 humantime-serde.workspace = true
@@ -42,9 +41,6 @@ http-body-util.workspace = true
 itertools.workspace = true
 sync_wrapper = { workspace = true, features = ["futures"] }
 
-byteorder = "1.4"
-rand = "0.8.5"
-
 [dev-dependencies]
 camino-tempfile.workspace = true
 test-context.workspace = true

libs/remote_storage/src/azure_blob.rs

@@ -14,25 +14,17 @@ use anyhow::{Context, Result, anyhow};
 use azure_core::request_options::{IfMatchCondition, MaxResults, Metadata, Range};
 use azure_core::{Continuable, HttpClient, RetryOptions, TransportOptions};
 use azure_storage::StorageCredentials;
-use azure_storage_blobs::blob::BlobBlockType;
-use azure_storage_blobs::blob::BlockList;
+use azure_storage_blobs::blob::operations::GetBlobBuilder;
 use azure_storage_blobs::blob::{Blob, CopyStatus};
 use azure_storage_blobs::container::operations::ListBlobsBuilder;
-use azure_storage_blobs::prelude::ClientBuilder;
-use azure_storage_blobs::{blob::operations::GetBlobBuilder, prelude::ContainerClient};
-use base64::{Engine as _, engine::general_purpose::URL_SAFE};
-use byteorder::{BigEndian, ByteOrder};
+use azure_storage_blobs::prelude::{ClientBuilder, ContainerClient};
 use bytes::Bytes;
-use camino::Utf8Path;
 use futures::FutureExt;
 use futures::future::Either;
 use futures::stream::Stream;
 use futures_util::{StreamExt, TryStreamExt};
 use http_types::{StatusCode, Url};
 use scopeguard::ScopeGuard;
-use tokio::fs::File;
-use tokio::io::AsyncReadExt;
-use tokio::io::AsyncSeekExt;
 use tokio_util::sync::CancellationToken;
 use tracing::debug;
 use utils::backoff;
@@ -59,9 +51,6 @@ pub struct AzureBlobStorage {
 
     // Alternative timeout used for metadata objects which are expected to be small
     pub small_timeout: Duration,
-    /* BEGIN_HADRON */
-    pub put_block_size_mb: Option<usize>,
-    /* END_HADRON */
 }
 
 impl AzureBlobStorage {
@@ -118,9 +107,6 @@ impl AzureBlobStorage {
             concurrency_limiter: ConcurrencyLimiter::new(azure_config.concurrency_limit.get()),
             timeout,
             small_timeout,
-            /* BEGIN_HADRON */
-            put_block_size_mb: azure_config.put_block_size_mb,
-            /* END_HADRON */
         })
     }
 
@@ -597,137 +583,31 @@ impl RemoteStorage for AzureBlobStorage {
 
         let started_at = start_measuring_requests(kind);
 
-        let mut metadata_map = metadata.unwrap_or([].into());
-        let timeline_file_path = metadata_map.0.remove("databricks_azure_put_block");
-
-        /* BEGIN_HADRON */
-        let op = async move {
+        let op = async {
             let blob_client = self.client.blob_client(self.relative_path_to_name(to));
-            let put_block_size = self.put_block_size_mb.unwrap_or(0) * 1024 * 1024;
-            if timeline_file_path.is_none() || put_block_size == 0 {
-                // Use put_block_blob directly.
-                let from: Pin<
-                    Box<dyn Stream<Item = std::io::Result<Bytes>> + Send + Sync + 'static>,
-                > = Box::pin(from);
-                let from = NonSeekableStream::new(from, data_size_bytes);
-                let body = azure_core::Body::SeekableStream(Box::new(from));
 
-                let mut builder = blob_client.put_block_blob(body);
-                if !metadata_map.0.is_empty() {
-                    builder = builder.metadata(to_azure_metadata(metadata_map));
-                }
-                let fut = builder.into_future();
-                let fut = tokio::time::timeout(self.timeout, fut);
-                let result = fut.await;
-                match result {
-                    Ok(Ok(_response)) => return Ok(()),
-                    Ok(Err(azure)) => return Err(azure.into()),
-                    Err(_timeout) => return Err(TimeoutOrCancel::Timeout.into()),
-                };
-            }
-            // Upload chunks concurrently using Put Block.
-            // Each PutBlock uploads put_block_size bytes of the file.
-            let mut upload_futures: Vec<tokio::task::JoinHandle<Result<(), azure_core::Error>>> =
-                vec![];
-            let mut block_list = BlockList::default();
-            let mut start_bytes = 0u64;
-            let mut remaining_bytes = data_size_bytes;
-            let mut block_list_count = 0;
+            let from: Pin<Box<dyn Stream<Item = std::io::Result<Bytes>> + Send + Sync + 'static>> =
+                Box::pin(from);
 
-            while remaining_bytes > 0 {
-                let block_size = std::cmp::min(remaining_bytes, put_block_size);
-                let end_bytes = start_bytes + block_size as u64;
-                let block_id = block_list_count;
-                let timeout = self.timeout;
-                let blob_client = blob_client.clone();
-                let timeline_file = timeline_file_path.clone().unwrap().clone();
+            let from = NonSeekableStream::new(from, data_size_bytes);
 
-                let mut encoded_block_id = [0u8; 8];
-                BigEndian::write_u64(&mut encoded_block_id, block_id);
-                URL_SAFE.encode(encoded_block_id);
+            let body = azure_core::Body::SeekableStream(Box::new(from));
 
-                // Put one block.
-                let part_fut = async move {
-                    let mut file = File::open(Utf8Path::new(&timeline_file.clone())).await?;
-                    file.seek(io::SeekFrom::Start(start_bytes)).await?;
-                    let limited_reader = file.take(block_size as u64);
-                    let file_chunk_stream =
-                        tokio_util::io::ReaderStream::with_capacity(limited_reader, 1024 * 1024);
-                    let file_chunk_stream_pin: Pin<
-                        Box<dyn Stream<Item = std::io::Result<Bytes>> + Send + Sync + 'static>,
-                    > = Box::pin(file_chunk_stream);
-                    let stream_wrapper = NonSeekableStream::new(file_chunk_stream_pin, block_size);
-                    let body = azure_core::Body::SeekableStream(Box::new(stream_wrapper));
-                    // Azure put block takes URL-encoded block ids and all blocks must have the same byte length.
-                    // https://learn.microsoft.com/en-us/rest/api/storageservices/put-block?tabs=microsoft-entra-id#uri-parameters
-                    let builder = blob_client.put_block(encoded_block_id.to_vec(), body);
-                    let fut = builder.into_future();
-                    let fut = tokio::time::timeout(timeout, fut);
-                    let result = fut.await;
-                    tracing::debug!(
-                        "azure put block id-{} size {} start {} end {} file {} response {:#?}",
-                        block_id,
-                        block_size,
-                        start_bytes,
-                        end_bytes,
-                        timeline_file,
-                        result
-                    );
-                    match result {
-                        Ok(Ok(_response)) => Ok(()),
-                        Ok(Err(azure)) => Err(azure),
-                        Err(_timeout) => Err(azure_core::Error::new(
-                            azure_core::error::ErrorKind::Io,
-                            std::io::Error::new(
-                                std::io::ErrorKind::TimedOut,
-                                "Operation timed out",
-                            ),
-                        )),
-                    }
-                };
-                upload_futures.push(tokio::spawn(part_fut));
+            let mut builder = blob_client.put_block_blob(body);
 
-                block_list_count += 1;
-                remaining_bytes -= block_size;
-                start_bytes += block_size as u64;
-
-                block_list
-                    .blocks
-                    .push(BlobBlockType::Uncommitted(encoded_block_id.to_vec().into()));
+            if let Some(metadata) = metadata {
+                builder = builder.metadata(to_azure_metadata(metadata));
             }
 
-            tracing::debug!(
-                "azure put blocks {} total MB: {} chunk size MB: {}",
-                block_list_count,
-                data_size_bytes / 1024 / 1024,
-                put_block_size / 1024 / 1024
-            );
-            // Wait for all blocks to be uploaded.
-            let upload_results = futures::future::try_join_all(upload_futures).await;
-            if upload_results.is_err() {
-                return Err(anyhow::anyhow!(format!(
-                    "Failed to upload all blocks {:#?}",
-                    upload_results.unwrap_err()
-                )));
-            }
-
-            // Commit the blocks.
-            let mut builder = blob_client.put_block_list(block_list);
-            if !metadata_map.0.is_empty() {
-                builder = builder.metadata(to_azure_metadata(metadata_map));
-            }
             let fut = builder.into_future();
             let fut = tokio::time::timeout(self.timeout, fut);
-            let result = fut.await;
-            tracing::debug!("azure put block list response {:#?}", result);
 
-            match result {
+            match fut.await {
                 Ok(Ok(_response)) => Ok(()),
                 Ok(Err(azure)) => Err(azure.into()),
                 Err(_timeout) => Err(TimeoutOrCancel::Timeout.into()),
             }
         };
-        /* END_HADRON */
 
         let res = tokio::select! {
             res = op => res,
@@ -742,6 +622,7 @@ impl RemoteStorage for AzureBlobStorage {
         crate::metrics::BUCKET_METRICS
             .req_seconds
             .observe_elapsed(kind, outcome, started_at);
+
         res
     }
 

libs/remote_storage/src/config.rs

@@ -195,19 +195,8 @@ pub struct AzureConfig {
     pub max_keys_per_list_response: Option<i32>,
     #[serde(default = "default_azure_conn_pool_size")]
     pub conn_pool_size: usize,
-    /* BEGIN_HADRON */
-    #[serde(default = "default_azure_put_block_size_mb")]
-    pub put_block_size_mb: Option<usize>,
-    /* END_HADRON */
 }
 
-/* BEGIN_HADRON */
-fn default_azure_put_block_size_mb() -> Option<usize> {
-    // Disable parallel upload by default.
-    Some(0)
-}
-/* END_HADRON */
-
 fn default_remote_storage_azure_concurrency_limit() -> NonZeroUsize {
     NonZeroUsize::new(DEFAULT_REMOTE_STORAGE_AZURE_CONCURRENCY_LIMIT).unwrap()
 }
@@ -224,9 +213,6 @@ impl Debug for AzureConfig {
                 "max_keys_per_list_response",
                 &self.max_keys_per_list_response,
             )
-            /* BEGIN_HADRON */
-            .field("put_block_size_mb", &self.put_block_size_mb)
-            /* END_HADRON */
             .finish()
     }
 }
@@ -366,7 +352,6 @@ timeout = '5s'";
     upload_storage_class = 'INTELLIGENT_TIERING'
     timeout = '7s'
     conn_pool_size = 8
-    put_block_size_mb = 1024
     ";
 
         let config = parse(toml).unwrap();
@@ -382,9 +367,6 @@ timeout = '5s'";
                     concurrency_limit: default_remote_storage_azure_concurrency_limit(),
                     max_keys_per_list_response: DEFAULT_MAX_KEYS_PER_LIST_RESPONSE,
                     conn_pool_size: 8,
-                    /* BEGIN_HADRON */
-                    put_block_size_mb: Some(1024),
-                    /* END_HADRON */
                 }),
                 timeout: Duration::from_secs(7),
                 small_timeout: RemoteStorageConfig::DEFAULT_SMALL_TIMEOUT

libs/remote_storage/src/lib.rs

@@ -732,15 +732,9 @@ impl GenericRemoteStorage {
         })
     }
 
-    /* BEGIN_HADRON */
-    pub fn unreliable_wrapper(s: Self, fail_first: u64, fail_probability: u64) -> Self {
-        Self::Unreliable(Arc::new(UnreliableWrapper::new(
-            s,
-            fail_first,
-            fail_probability,
-        )))
+    pub fn unreliable_wrapper(s: Self, fail_first: u64) -> Self {
+        Self::Unreliable(Arc::new(UnreliableWrapper::new(s, fail_first)))
     }
-    /* END_HADRON */
 
     /// See [`RemoteStorage::upload`], which this method calls with `None` as metadata.
     pub async fn upload_storage_object(

libs/remote_storage/src/simulate_failures.rs

@@ -1,8 +1,6 @@
 //! This module provides a wrapper around a real RemoteStorage implementation that
 //! causes the first N attempts at each upload or download operatio to fail. For
 //! testing purposes.
-use rand::Rng;
-use std::cmp;
 use std::collections::HashMap;
 use std::collections::hash_map::Entry;
 use std::num::NonZeroU32;
@@ -27,13 +25,6 @@ pub struct UnreliableWrapper {
 
     // Tracks how many failed attempts of each operation has been made.
     attempts: Mutex<HashMap<RemoteOp, u64>>,
-
-    /* BEGIN_HADRON */
-    // This the probability of failure for each operation, ranged from [0, 100].
-    // The probability is default to 100, which means that all operations will fail.
-    // Storage will fail by probability up to attempts_to_fail times.
-    attempt_failure_probability: u64,
-    /* END_HADRON */
 }
 
 /// Used to identify retries of different unique operation.
@@ -49,11 +40,7 @@ enum RemoteOp {
 }
 
 impl UnreliableWrapper {
-    pub fn new(
-        inner: crate::GenericRemoteStorage,
-        attempts_to_fail: u64,
-        attempt_failure_probability: u64,
-    ) -> Self {
+    pub fn new(inner: crate::GenericRemoteStorage, attempts_to_fail: u64) -> Self {
         assert!(attempts_to_fail > 0);
         let inner = match inner {
             GenericRemoteStorage::AwsS3(s) => GenericRemoteStorage::AwsS3(s),
@@ -64,11 +51,9 @@ impl UnreliableWrapper {
                 panic!("Can't wrap unreliable wrapper unreliably")
             }
         };
-        let actual_attempt_failure_probability = cmp::min(attempt_failure_probability, 100);
         UnreliableWrapper {
             inner,
             attempts_to_fail,
-            attempt_failure_probability: actual_attempt_failure_probability,
             attempts: Mutex::new(HashMap::new()),
         }
     }
@@ -81,7 +66,6 @@ impl UnreliableWrapper {
     ///
     fn attempt(&self, op: RemoteOp) -> anyhow::Result<u64> {
         let mut attempts = self.attempts.lock().unwrap();
-        let mut rng = rand::thread_rng();
 
         match attempts.entry(op) {
             Entry::Occupied(mut e) => {
@@ -91,19 +75,15 @@ impl UnreliableWrapper {
                     *p
                 };
 
-                /* BEGIN_HADRON */
-                // If there are more attempts to fail, fail the request by probability.
-                if (attempts_before_this < self.attempts_to_fail)
-                    && (rng.gen_range(0..=100) < self.attempt_failure_probability)
-                {
+                if attempts_before_this >= self.attempts_to_fail {
+                    // let it succeed
+                    e.remove();
+                    Ok(attempts_before_this)
+                } else {
                     let error =
                         anyhow::anyhow!("simulated failure of remote operation {:?}", e.key());
                     Err(error)
-                } else {
-                    e.remove();
-                    Ok(attempts_before_this)
                 }
-                /* END_HADRON */
             }
             Entry::Vacant(e) => {
                 let error = anyhow::anyhow!("simulated failure of remote operation {:?}", e.key());

libs/remote_storage/tests/common/mod.rs

@@ -165,42 +165,10 @@ pub(crate) async fn upload_remote_data(
 
             let (data, data_len) =
                 upload_stream(format!("remote blob data {i}").into_bytes().into());
-
-            /* BEGIN_HADRON */
-            let mut metadata = None;
-            if matches!(&*task_client, GenericRemoteStorage::AzureBlob(_)) {
-                let file_path = "/tmp/dbx_upload_tmp_file.txt";
-                {
-                    // Open the file in append mode
-                    let mut file = std::fs::OpenOptions::new()
-                        .append(true)
-                        .create(true) // Create the file if it doesn't exist
-                        .open(file_path)?;
-                    // Append some bytes to the file
-                    std::io::Write::write_all(
-                        &mut file,
-                        &format!("remote blob data {i}").into_bytes(),
-                    )?;
-                    file.sync_all()?;
-                }
-                metadata = Some(remote_storage::StorageMetadata::from([(
-                    "databricks_azure_put_block",
-                    file_path,
-                )]));
-            }
-            /* END_HADRON */
-
             task_client
-                .upload(data, data_len, &blob_path, metadata, &cancel)
+                .upload(data, data_len, &blob_path, None, &cancel)
                 .await?;
 
-            // TODO: Check upload is using the put_block upload.
-            // We cannot consume data here since data is moved inside the upload.
-            // let total_bytes = data.fold(0, |acc, chunk| async move {
-            //     acc + chunk.map(|bytes| bytes.len()).unwrap_or(0)
-            // }).await;
-            // assert_eq!(total_bytes, data_len);
-
             Ok::<_, anyhow::Error>((blob_prefix, blob_path))
         });
     }

libs/remote_storage/tests/test_real_azure.rs

@@ -219,9 +219,6 @@ async fn create_azure_client(
             concurrency_limit: NonZeroUsize::new(100).unwrap(),
             max_keys_per_list_response,
             conn_pool_size: 8,
-            /* BEGIN_HADRON */
-            put_block_size_mb: Some(1),
-            /* END_HADRON */
         }),
         timeout: RemoteStorageConfig::DEFAULT_TIMEOUT,
         small_timeout: RemoteStorageConfig::DEFAULT_SMALL_TIMEOUT,

libs/safekeeper_api/src/models.rs

@@ -11,7 +11,7 @@ use utils::id::{NodeId, TenantId, TenantTimelineId, TimelineId};
 use utils::lsn::Lsn;
 use utils::pageserver_feedback::PageserverFeedback;
 
-use crate::membership::{Configuration, SafekeeperGeneration};
+use crate::membership::Configuration;
 use crate::{ServerInfo, Term};
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -221,7 +221,7 @@ pub struct TimelineMembershipSwitchRequest {
 pub struct TimelineMembershipSwitchResponse {
     pub previous_conf: Configuration,
     pub current_conf: Configuration,
-    pub last_log_term: Term,
+    pub term: Term,
     pub flush_lsn: Lsn,
 }
 
@@ -311,12 +311,3 @@ pub struct PullTimelineResponse {
     pub safekeeper_host: Option<String>,
     // TODO: add more fields?
 }
-
-/// Response to a timeline locate request.
-/// Storcon-only API.
-#[derive(Serialize, Deserialize, Clone, Debug)]
-pub struct TimelineLocateResponse {
-    pub generation: SafekeeperGeneration,
-    pub sk_set: Vec<NodeId>,
-    pub new_sk_set: Option<Vec<NodeId>>,
-}

libs/utils/src/env.rs

@@ -44,63 +44,3 @@ where
         }
     }
 }
-
-/* BEGIN_HADRON */
-pub enum DeploymentMode {
-    Local,
-    Dev,
-    Staging,
-    Prod,
-}
-
-pub fn get_deployment_mode() -> Option<DeploymentMode> {
-    match std::env::var("DEPLOYMENT_MODE") {
-        Ok(env) => match env.as_str() {
-            "development" => Some(DeploymentMode::Dev),
-            "staging" => Some(DeploymentMode::Staging),
-            "production" => Some(DeploymentMode::Prod),
-            _ => {
-                tracing::error!("Unexpected DEPLOYMENT_MODE: {}", env);
-                None
-            }
-        },
-        Err(_) => {
-            // tracing::error!("DEPLOYMENT_MODE not set");
-            None
-        }
-    }
-}
-
-pub fn is_dev_or_staging() -> bool {
-    matches!(
-        get_deployment_mode(),
-        Some(DeploymentMode::Dev) | Some(DeploymentMode::Staging)
-    )
-}
-
-pub enum TestingMode {
-    Chaos,
-    Stress,
-}
-
-pub fn get_test_mode() -> Option<TestingMode> {
-    match std::env::var("HADRON_TEST_MODE") {
-        Ok(env) => match env.as_str() {
-            "chaos" => Some(TestingMode::Chaos),
-            "stress" => Some(TestingMode::Stress),
-            _ => {
-                tracing::error!("Unexpected HADRON_TEST_MODE: {}", env);
-                None
-            }
-        },
-        Err(_) => {
-            tracing::error!("HADRON_TEST_MODE not set");
-            None
-        }
-    }
-}
-
-pub fn is_chaos_testing() -> bool {
-    matches!(get_test_mode(), Some(TestingMode::Chaos))
-}
-/* END_HADRON */

libs/utils/src/ip_address.rs

@@ -1,73 +0,0 @@
-use std::env::{VarError, var};
-use std::error::Error;
-use std::net::IpAddr;
-use std::str::FromStr;
-
-/// Name of the environment variable containing the reachable IP address of the node. If set, the IP address contained in this
-/// environment variable is used as the reachable IP address of the pageserver or safekeeper node during node registration.
-/// In a Kubernetes environment, this environment variable should be set by Kubernetes to the Pod IP (specified in the Pod
-/// template).
-pub const HADRON_NODE_IP_ADDRESS: &str = "HADRON_NODE_IP_ADDRESS";
-
-/// Read the reachable IP address of this page server from env var HADRON_NODE_IP_ADDRESS.
-/// In Kubernetes this environment variable is set to the Pod IP (specified in the Pod template).
-pub fn read_node_ip_addr_from_env() -> Result<Option<IpAddr>, Box<dyn Error>> {
-    match var(HADRON_NODE_IP_ADDRESS) {
-        Ok(v) => {
-            if let Ok(addr) = IpAddr::from_str(&v) {
-                Ok(Some(addr))
-            } else {
-                Err(format!("Invalid IP address string: {v}. Cannot be parsed as either an IPv4 or an IPv6 address.").into())
-            }
-        }
-        Err(VarError::NotPresent) => Ok(None),
-        Err(e) => Err(e.into()),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::env;
-    use std::net::{Ipv4Addr, Ipv6Addr};
-
-    #[test]
-    fn test_read_node_ip_addr_from_env() {
-        // SAFETY: test code
-        unsafe {
-            // Test with a valid IPv4 address
-            env::set_var(HADRON_NODE_IP_ADDRESS, "192.168.1.1");
-            let result = read_node_ip_addr_from_env().unwrap();
-            assert_eq!(result, Some(IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))));
-
-            // Test with a valid IPv6 address
-            env::set_var(
-                HADRON_NODE_IP_ADDRESS,
-                "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
-            );
-        }
-        let result = read_node_ip_addr_from_env().unwrap();
-        assert_eq!(
-            result,
-            Some(IpAddr::V6(
-                Ipv6Addr::from_str("2001:0db8:85a3:0000:0000:8a2e:0370:7334").unwrap()
-            ))
-        );
-
-        // Test with an invalid IP address
-        // SAFETY: test code
-        unsafe {
-            env::set_var(HADRON_NODE_IP_ADDRESS, "invalid_ip");
-        }
-        let result = read_node_ip_addr_from_env();
-        assert!(result.is_err());
-
-        // Test with no environment variable set
-        // SAFETY: test code
-        unsafe {
-            env::remove_var(HADRON_NODE_IP_ADDRESS);
-        }
-        let result = read_node_ip_addr_from_env().unwrap();
-        assert_eq!(result, None);
-    }
-}

libs/utils/src/lib.rs

@@ -26,9 +26,6 @@ pub mod auth;
 // utility functions and helper traits for unified unique id generation/serialization etc.
 pub mod id;
 
-// utility functions to obtain reachable IP addresses in PS/SK nodes.
-pub mod ip_address;
-
 pub mod shard;
 
 mod hex;
@@ -102,8 +99,6 @@ pub mod elapsed_accum;
 #[cfg(target_os = "linux")]
 pub mod linux_socket_ioctl;
 
-pub mod metrics_collector;
-
 // Re-export used in macro. Avoids adding git-version as dep in target crates.
 #[doc(hidden)]
 pub use git_version;

libs/utils/src/logging.rs

@@ -1,5 +1,4 @@
 use std::future::Future;
-use std::pin::Pin;
 use std::str::FromStr;
 use std::time::Duration;
 
@@ -8,7 +7,7 @@ use metrics::{IntCounter, IntCounterVec};
 use once_cell::sync::Lazy;
 use strum_macros::{EnumString, VariantNames};
 use tokio::time::Instant;
-use tracing::{info, warn};
+use tracing::info;
 
 /// Logs a critical error, similarly to `tracing::error!`. This will:
 ///
@@ -25,28 +24,12 @@ macro_rules! critical {
         if cfg!(debug_assertions) {
             panic!($($arg)*);
         }
-        // Increment both metrics
         $crate::logging::TRACING_EVENT_COUNT_METRIC.inc_critical();
         let backtrace = std::backtrace::Backtrace::capture();
         tracing::error!("CRITICAL: {}\n{backtrace}", format!($($arg)*));
     }};
 }
 
-#[macro_export]
-macro_rules! critical_timeline {
-    ($tenant_shard_id:expr, $timeline_id:expr, $($arg:tt)*) => {{
-        if cfg!(debug_assertions) {
-            panic!($($arg)*);
-        }
-        // Increment both metrics
-        $crate::logging::TRACING_EVENT_COUNT_METRIC.inc_critical();
-        $crate::logging::HADRON_CRITICAL_STORAGE_EVENT_COUNT_METRIC.inc(&$tenant_shard_id.to_string(), &$timeline_id.to_string());
-        let backtrace = std::backtrace::Backtrace::capture();
-        tracing::error!("CRITICAL: [tenant_shard_id: {}, timeline_id: {}] {}\n{backtrace}",
-                       $tenant_shard_id, $timeline_id, format!($($arg)*));
-    }};
-}
-
 #[derive(EnumString, strum_macros::Display, VariantNames, Eq, PartialEq, Debug, Clone, Copy)]
 #[strum(serialize_all = "snake_case")]
 pub enum LogFormat {
@@ -78,36 +61,6 @@ pub struct TracingEventCountMetric {
     trace: IntCounter,
 }
 
-// Begin Hadron: Add a HadronCriticalStorageEventCountMetric metric that is sliced by tenant_id and timeline_id
-pub struct HadronCriticalStorageEventCountMetric {
-    critical: IntCounterVec,
-}
-
-pub static HADRON_CRITICAL_STORAGE_EVENT_COUNT_METRIC: Lazy<HadronCriticalStorageEventCountMetric> =
-    Lazy::new(|| {
-        let vec = metrics::register_int_counter_vec!(
-            "hadron_critical_storage_event_count",
-            "Number of critical storage events, by tenant_id and timeline_id",
-            &["tenant_shard_id", "timeline_id"]
-        )
-        .expect("failed to define metric");
-        HadronCriticalStorageEventCountMetric::new(vec)
-    });
-
-impl HadronCriticalStorageEventCountMetric {
-    fn new(vec: IntCounterVec) -> Self {
-        Self { critical: vec }
-    }
-
-    // Allow public access from `critical!` macro.
-    pub fn inc(&self, tenant_shard_id: &str, timeline_id: &str) {
-        self.critical
-            .with_label_values(&[tenant_shard_id, timeline_id])
-            .inc();
-    }
-}
-// End Hadron
-
 pub static TRACING_EVENT_COUNT_METRIC: Lazy<TracingEventCountMetric> = Lazy::new(|| {
     let vec = metrics::register_int_counter_vec!(
         "libmetrics_tracing_event_count",
@@ -378,11 +331,10 @@ impl std::fmt::Debug for SecretString {
 ///
 /// TODO: consider upgrading this to a warning, but currently it fires too often.
 #[inline]
-pub async fn log_slow<O>(
-    name: &str,
-    threshold: Duration,
-    f: Pin<&mut impl Future<Output = O>>,
-) -> O {
+pub async fn log_slow<F, O>(name: &str, threshold: Duration, f: std::pin::Pin<&mut F>) -> O
+where
+    F: Future<Output = O>,
+{
     monitor_slow_future(
         threshold,
         threshold, // period = threshold
@@ -396,42 +348,16 @@ pub async fn log_slow<O>(
             if !is_slow {
                 return;
             }
-            let elapsed = elapsed_total.as_secs_f64();
             if ready {
-                info!("slow {name} completed after {elapsed:.3}s");
+                info!(
+                    "slow {name} completed after {:.3}s",
+                    elapsed_total.as_secs_f64()
+                );
             } else {
-                info!("slow {name} still running after {elapsed:.3}s");
-            }
-        },
-    )
-    .await
-}
-
-/// Logs a periodic warning if a future is slow to complete.
-#[inline]
-pub async fn warn_slow<O>(
-    name: &str,
-    threshold: Duration,
-    f: Pin<&mut impl Future<Output = O>>,
-) -> O {
-    monitor_slow_future(
-        threshold,
-        threshold, // period = threshold
-        f,
-        |MonitorSlowFutureCallback {
-             ready,
-             is_slow,
-             elapsed_total,
-             elapsed_since_last_callback: _,
-         }| {
-            if !is_slow {
-                return;
-            }
-            let elapsed = elapsed_total.as_secs_f64();
-            if ready {
-                warn!("slow {name} completed after {elapsed:.3}s");
-            } else {
-                warn!("slow {name} still running after {elapsed:.3}s");
+                info!(
+                    "slow {name} still running after {:.3}s",
+                    elapsed_total.as_secs_f64()
+                );
             }
         },
     )
@@ -444,7 +370,7 @@ pub async fn warn_slow<O>(
 pub async fn monitor_slow_future<F, O>(
     threshold: Duration,
     period: Duration,
-    mut fut: Pin<&mut F>,
+    mut fut: std::pin::Pin<&mut F>,
     mut cb: impl FnMut(MonitorSlowFutureCallback),
 ) -> O
 where

libs/utils/src/metrics_collector.rs

@@ -1,75 +0,0 @@
-use std::{
-    sync::{Arc, RwLock},
-    time::{Duration, Instant},
-};
-
-use metrics::{IntGauge, proto::MetricFamily, register_int_gauge};
-use once_cell::sync::Lazy;
-
-pub static METRICS_STALE_MILLIS: Lazy<IntGauge> = Lazy::new(|| {
-    register_int_gauge!(
-        "metrics_metrics_stale_milliseconds",
-        "The current metrics stale time in milliseconds"
-    )
-    .expect("failed to define a metric")
-});
-
-#[derive(Debug)]
-pub struct CollectedMetrics {
-    pub metrics: Vec<MetricFamily>,
-    pub collected_at: Instant,
-}
-
-impl CollectedMetrics {
-    fn new(metrics: Vec<MetricFamily>) -> Self {
-        Self {
-            metrics,
-            collected_at: Instant::now(),
-        }
-    }
-}
-
-#[derive(Debug)]
-pub struct MetricsCollector {
-    last_collected: RwLock<Arc<CollectedMetrics>>,
-}
-
-impl MetricsCollector {
-    pub fn new() -> Self {
-        Self {
-            last_collected: RwLock::new(Arc::new(CollectedMetrics::new(vec![]))),
-        }
-    }
-
-    #[tracing::instrument(name = "metrics_collector", skip_all)]
-    pub fn run_once(&self, cache_metrics: bool) -> Arc<CollectedMetrics> {
-        let started = Instant::now();
-        let metrics = metrics::gather();
-        let collected = Arc::new(CollectedMetrics::new(metrics));
-        if cache_metrics {
-            let mut guard = self.last_collected.write().unwrap();
-            *guard = collected.clone();
-        }
-        tracing::info!(
-            "Collected {} metric families in {} ms",
-            collected.metrics.len(),
-            started.elapsed().as_millis()
-        );
-        collected
-    }
-
-    pub fn last_collected(&self) -> Arc<CollectedMetrics> {
-        self.last_collected.read().unwrap().clone()
-    }
-}
-
-impl Default for MetricsCollector {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-// Interval for metrics collection. Currently hard-coded to be the same as the metrics scape interval from the obs agent
-pub static METRICS_COLLECTION_INTERVAL: Duration = Duration::from_secs(30);
-
-pub static METRICS_COLLECTOR: Lazy<MetricsCollector> = Lazy::new(MetricsCollector::default);

libs/utils/src/shard.rs

@@ -171,12 +171,6 @@ impl std::fmt::Display for ShardNumber {
     }
 }
 
-impl std::fmt::Display for ShardCount {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
 impl std::fmt::Display for ShardSlug<'_> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         write!(

libs/walproposer/src/api_bindings.rs

@@ -428,12 +428,6 @@ pub fn empty_shmem() -> crate::bindings::WalproposerShmemState {
         shard_number: 0,
     };
 
-    let empty_wal_rate_limiter = crate::bindings::WalRateLimiter {
-        should_limit: crate::bindings::pg_atomic_uint32 { value: 0 },
-        sent_bytes: 0,
-        last_recorded_time_us: 0,
-    };
-
     crate::bindings::WalproposerShmemState {
         propEpochStartLsn: crate::bindings::pg_atomic_uint64 { value: 0 },
         donor_name: [0; 64],
@@ -447,7 +441,6 @@ pub fn empty_shmem() -> crate::bindings::WalproposerShmemState {
         num_shards: 0,
         replica_promote: false,
         min_ps_feedback: empty_feedback,
-        wal_rate_limiter: empty_wal_rate_limiter,
     }
 }
 

pageserver/Cargo.toml

@@ -112,7 +112,6 @@ twox-hash.workspace = true
 procfs.workspace = true
 
 [dev-dependencies]
-base64.workspace = true
 criterion.workspace = true
 hex-literal.workspace = true
 tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }

pageserver/client/src/mgmt_api.rs

@@ -1,4 +1,4 @@
-use std::collections::{BTreeMap, HashMap};
+use std::collections::HashMap;
 use std::error::Error as _;
 use std::time::Duration;
 
@@ -251,70 +251,6 @@ impl Client {
         Ok(())
     }
 
-    pub async fn tenant_timeline_compact(
-        &self,
-        tenant_shard_id: TenantShardId,
-        timeline_id: TimelineId,
-        force_image_layer_creation: bool,
-        must_force_image_layer_creation: bool,
-        scheduled: bool,
-        wait_until_done: bool,
-    ) -> Result<()> {
-        let mut path = reqwest::Url::parse(&format!(
-            "{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}/compact",
-            self.mgmt_api_endpoint
-        ))
-        .expect("Cannot build URL");
-
-        if force_image_layer_creation {
-            path.query_pairs_mut()
-                .append_pair("force_image_layer_creation", "true");
-        }
-
-        if must_force_image_layer_creation {
-            path.query_pairs_mut()
-                .append_pair("must_force_image_layer_creation", "true");
-        }
-
-        if scheduled {
-            path.query_pairs_mut().append_pair("scheduled", "true");
-        }
-        if wait_until_done {
-            path.query_pairs_mut()
-                .append_pair("wait_until_scheduled_compaction_done", "true");
-            path.query_pairs_mut()
-                .append_pair("wait_until_uploaded", "true");
-        }
-        self.request(Method::PUT, path, ()).await?;
-        Ok(())
-    }
-
-    /* BEGIN_HADRON */
-    pub async fn tenant_timeline_describe(
-        &self,
-        tenant_shard_id: &TenantShardId,
-        timeline_id: &TimelineId,
-    ) -> Result<TimelineInfo> {
-        let mut path = reqwest::Url::parse(&format!(
-            "{}/v1/tenant/{tenant_shard_id}/timeline/{timeline_id}",
-            self.mgmt_api_endpoint
-        ))
-        .expect("Cannot build URL");
-        path.query_pairs_mut()
-            .append_pair("include-image-consistent-lsn", "true");
-
-        let response: reqwest::Response = self.request(Method::GET, path, ()).await?;
-        let body = response.json().await.map_err(Error::ReceiveBody)?;
-        Ok(body)
-    }
-
-    pub async fn list_tenant_visible_size(&self) -> Result<BTreeMap<TenantShardId, u64>> {
-        let uri = format!("{}/v1/list_tenant_visible_size", self.mgmt_api_endpoint);
-        let resp = self.get(&uri).await?;
-        resp.json().await.map_err(Error::ReceiveBody)
-    }
-    /* END_HADRON */
-
     pub async fn tenant_scan_remote_storage(
         &self,
         tenant_id: TenantId,

pageserver/client_grpc/Cargo.toml

@@ -1,24 +0,0 @@
-[package]
-name = "pageserver_client_grpc"
-version = "0.1.0"
-edition.workspace = true
-license.workspace = true
-
-[features]
-testing = ["pageserver_api/testing"]
-
-[dependencies]
-anyhow.workspace = true
-arc-swap.workspace = true
-bytes.workspace = true
-compute_api.workspace = true
-futures.workspace = true
-pageserver_api.workspace = true
-pageserver_page_api.workspace = true
-tokio.workspace = true
-tokio-stream.workspace = true
-tokio-util.workspace = true
-tonic.workspace = true
-tracing.workspace = true
-utils.workspace = true
-workspace_hack.workspace = true

pageserver/client_grpc/src/client.rs

@@ -1,604 +0,0 @@
-use std::collections::HashMap;
-use std::num::NonZero;
-use std::pin::pin;
-use std::sync::Arc;
-use std::time::{Duration, Instant};
-
-use anyhow::anyhow;
-use arc_swap::ArcSwap;
-use futures::stream::FuturesUnordered;
-use futures::{FutureExt as _, StreamExt as _};
-use tonic::codec::CompressionEncoding;
-use tracing::{debug, instrument};
-use utils::logging::warn_slow;
-
-use crate::pool::{ChannelPool, ClientGuard, ClientPool, StreamGuard, StreamPool};
-use crate::retry::Retry;
-use crate::split::GetPageSplitter;
-use compute_api::spec::PageserverProtocol;
-use pageserver_api::shard::ShardStripeSize;
-use pageserver_page_api as page_api;
-use utils::id::{TenantId, TimelineId};
-use utils::shard::{ShardCount, ShardIndex, ShardNumber};
-
-/// Max number of concurrent clients per channel (i.e. TCP connection). New channels will be spun up
-/// when full.
-///
-/// Normal requests are small, and we don't pipeline them, so we can afford a large number of
-/// streams per connection.
-///
-/// TODO: tune all of these constants, and consider making them configurable.
-const MAX_CLIENTS_PER_CHANNEL: NonZero<usize> = NonZero::new(64).unwrap();
-
-/// Max number of concurrent bulk GetPage streams per channel (i.e. TCP connection). These use a
-/// dedicated channel pool with a lower client limit, to avoid TCP-level head-of-line blocking and
-/// transmission delays. This also concentrates large window sizes on a smaller set of
-/// streams/connections, presumably reducing memory use.
-const MAX_BULK_CLIENTS_PER_CHANNEL: NonZero<usize> = NonZero::new(16).unwrap();
-
-/// The batch size threshold at which a GetPage request will use the bulk stream pool.
-///
-/// The gRPC initial window size is 64 KB. Each page is 8 KB, so let's avoid increasing the window
-/// size for the normal stream pool, and route requests for >= 5 pages (>32 KB) to the bulk pool.
-const BULK_THRESHOLD_BATCH_SIZE: usize = 5;
-
-/// The overall request call timeout, including retries and pool acquisition.
-/// TODO: should we retry forever? Should the caller decide?
-const CALL_TIMEOUT: Duration = Duration::from_secs(60);
-
-/// The per-request (retry attempt) timeout, including any lazy connection establishment.
-const REQUEST_TIMEOUT: Duration = Duration::from_secs(10);
-
-/// The initial request retry backoff duration. The first retry does not back off.
-/// TODO: use a different backoff for ResourceExhausted (rate limiting)? Needs server support.
-const BASE_BACKOFF: Duration = Duration::from_millis(5);
-
-/// The maximum request retry backoff duration.
-const MAX_BACKOFF: Duration = Duration::from_secs(5);
-
-/// Threshold and interval for warning about slow operation.
-const SLOW_THRESHOLD: Duration = Duration::from_secs(3);
-
-/// A rich Pageserver gRPC client for a single tenant timeline. This client is more capable than the
-/// basic `page_api::Client` gRPC client, and supports:
-///
-/// * Sharded tenants across multiple Pageservers.
-/// * Pooling of connections, clients, and streams for efficient resource use.
-/// * Concurrent use by many callers.
-/// * Internal handling of GetPage bidirectional streams.
-/// * Automatic retries.
-/// * Observability.
-///
-/// The client has dedicated connection/client/stream pools per shard, for resource reuse. These
-/// pools are unbounded: we allow scaling out as many concurrent streams as needed to serve all
-/// concurrent callers, which mostly eliminates head-of-line blocking. Idle streams are fairly
-/// cheap: the server task currently uses 26 KB of memory, so we can comfortably fit 100,000
-/// concurrent idle streams (2.5 GB memory). The worst case degenerates to the old libpq case with
-/// one stream per backend, but without the TCP connection overhead. In the common case we expect
-/// significantly lower stream counts due to stream sharing, driven e.g. by idle backends, LFC hits,
-/// read coalescing, sharding (backends typically only talk to one shard at a time), etc.
-///
-/// TODO: this client does not support base backups or LSN leases, as these are only used by
-/// compute_ctl. Consider adding this, but LSN leases need concurrent requests on all shards.
-pub struct PageserverClient {
-    /// The tenant ID.
-    tenant_id: TenantId,
-    /// The timeline ID.
-    timeline_id: TimelineId,
-    /// The JWT auth token for this tenant, if any.
-    auth_token: Option<String>,
-    /// The compression to use, if any.
-    compression: Option<CompressionEncoding>,
-    /// The shards for this tenant.
-    shards: ArcSwap<Shards>,
-}
-
-impl PageserverClient {
-    /// Creates a new Pageserver client for a given tenant and timeline. Uses the Pageservers given
-    /// in the shard spec, which must be complete and must use gRPC URLs.
-    pub fn new(
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        shard_spec: ShardSpec,
-        auth_token: Option<String>,
-        compression: Option<CompressionEncoding>,
-    ) -> anyhow::Result<Self> {
-        let shards = Shards::new(
-            tenant_id,
-            timeline_id,
-            shard_spec,
-            auth_token.clone(),
-            compression,
-        )?;
-        Ok(Self {
-            tenant_id,
-            timeline_id,
-            auth_token,
-            compression,
-            shards: ArcSwap::new(Arc::new(shards)),
-        })
-    }
-
-    /// Updates the shards from the given shard spec. In-flight requests will complete using the
-    /// existing shards, but may retry with the new shards if they fail.
-    ///
-    /// TODO: verify that in-flight requests are allowed to complete, and that the old pools are
-    /// properly spun down and dropped afterwards.
-    pub fn update_shards(&self, shard_spec: ShardSpec) -> anyhow::Result<()> {
-        // Validate the shard spec. We should really use `ArcSwap::rcu` for this, to avoid races
-        // with concurrent updates, but that involves creating a new `Shards` on every attempt,
-        // which spins up a bunch of Tokio tasks and such. These should already be checked elsewhere
-        // in the stack, and if they're violated then we already have problems elsewhere, so a
-        // best-effort but possibly-racy check is okay here.
-        let old = self.shards.load_full();
-        if shard_spec.count < old.count {
-            return Err(anyhow!(
-                "can't reduce shard count from {} to {}",
-                old.count,
-                shard_spec.count
-            ));
-        }
-        if !old.count.is_unsharded() && shard_spec.stripe_size != old.stripe_size {
-            return Err(anyhow!(
-                "can't change stripe size from {} to {}",
-                old.stripe_size,
-                shard_spec.stripe_size
-            ));
-        }
-
-        let shards = Shards::new(
-            self.tenant_id,
-            self.timeline_id,
-            shard_spec,
-            self.auth_token.clone(),
-            self.compression,
-        )?;
-        self.shards.store(Arc::new(shards));
-        Ok(())
-    }
-
-    /// Returns whether a relation exists.
-    #[instrument(skip_all, fields(rel=%req.rel, lsn=%req.read_lsn))]
-    pub async fn check_rel_exists(
-        &self,
-        req: page_api::CheckRelExistsRequest,
-    ) -> tonic::Result<page_api::CheckRelExistsResponse> {
-        debug!("sending request: {req:?}");
-        let resp = Self::with_retries(CALL_TIMEOUT, async |_| {
-            // Relation metadata is only available on shard 0.
-            let mut client = self.shards.load_full().get_zero().client().await?;
-            Self::with_timeout(REQUEST_TIMEOUT, client.check_rel_exists(req)).await
-        })
-        .await?;
-        debug!("received response: {resp:?}");
-        Ok(resp)
-    }
-
-    /// Returns the total size of a database, as # of bytes.
-    #[instrument(skip_all, fields(db_oid=%req.db_oid, lsn=%req.read_lsn))]
-    pub async fn get_db_size(
-        &self,
-        req: page_api::GetDbSizeRequest,
-    ) -> tonic::Result<page_api::GetDbSizeResponse> {
-        debug!("sending request: {req:?}");
-        let resp = Self::with_retries(CALL_TIMEOUT, async |_| {
-            // Relation metadata is only available on shard 0.
-            let mut client = self.shards.load_full().get_zero().client().await?;
-            Self::with_timeout(REQUEST_TIMEOUT, client.get_db_size(req)).await
-        })
-        .await?;
-        debug!("received response: {resp:?}");
-        Ok(resp)
-    }
-
-    /// Fetches pages. The `request_id` must be unique across all in-flight requests, and the
-    /// `attempt` must be 0 (incremented on retry). Automatically splits requests that straddle
-    /// shard boundaries, and assembles the responses.
-    ///
-    /// Unlike `page_api::Client`, this automatically converts `status_code` into `tonic::Status`
-    /// errors. All responses will have `GetPageStatusCode::Ok`.
-    #[instrument(skip_all, fields(
-        req_id = %req.request_id,
-        class = %req.request_class,
-        rel = %req.rel,
-        blkno = %req.block_numbers[0],
-        blks = %req.block_numbers.len(),
-        lsn = %req.read_lsn,
-    ))]
-    pub async fn get_page(
-        &self,
-        req: page_api::GetPageRequest,
-    ) -> tonic::Result<page_api::GetPageResponse> {
-        // Make sure we have at least one page.
-        if req.block_numbers.is_empty() {
-            return Err(tonic::Status::invalid_argument("no block number"));
-        }
-        // The request attempt must be 0. The client will increment it internally.
-        if req.request_id.attempt != 0 {
-            return Err(tonic::Status::invalid_argument("request attempt must be 0"));
-        }
-
-        debug!("sending request: {req:?}");
-
-        // The shards may change while we're fetching pages. We execute the request using a stable
-        // view of the shards (especially important for requests that span shards), but retry the
-        // top-level (pre-split) request to pick up shard changes. This can lead to unnecessary
-        // retries and re-splits in some cases where requests span shards, but these are expected to
-        // be rare.
-        //
-        // TODO: the gRPC server and client doesn't yet properly support shard splits. Revisit this
-        // once we figure out how to handle these.
-        let resp = Self::with_retries(CALL_TIMEOUT, async |attempt| {
-            let mut req = req.clone();
-            req.request_id.attempt = attempt as u32;
-            let shards = self.shards.load_full();
-            Self::with_timeout(REQUEST_TIMEOUT, Self::get_page_with_shards(req, &shards)).await
-        })
-        .await?;
-
-        debug!("received response: {resp:?}");
-        Ok(resp)
-    }
-
-    /// Fetches pages using the given shards. This uses a stable view of the shards, regardless of
-    /// concurrent shard updates. Does not retry internally, but is retried by `get_page()`.
-    async fn get_page_with_shards(
-        req: page_api::GetPageRequest,
-        shards: &Shards,
-    ) -> tonic::Result<page_api::GetPageResponse> {
-        // Fast path: request is for a single shard.
-        if let Some(shard_id) =
-            GetPageSplitter::for_single_shard(&req, shards.count, shards.stripe_size)
-        {
-            return Self::get_page_with_shard(req, shards.get(shard_id)?).await;
-        }
-
-        // Request spans multiple shards. Split it, dispatch concurrent per-shard requests, and
-        // reassemble the responses.
-        let mut splitter = GetPageSplitter::split(req, shards.count, shards.stripe_size);
-
-        let mut shard_requests = FuturesUnordered::new();
-        for (shard_id, shard_req) in splitter.drain_requests() {
-            let future = Self::get_page_with_shard(shard_req, shards.get(shard_id)?)
-                .map(move |result| result.map(|resp| (shard_id, resp)));
-            shard_requests.push(future);
-        }
-
-        while let Some((shard_id, shard_response)) = shard_requests.next().await.transpose()? {
-            splitter.add_response(shard_id, shard_response)?;
-        }
-
-        splitter.get_response()
-    }
-
-    /// Fetches pages on the given shard. Does not retry internally.
-    async fn get_page_with_shard(
-        req: page_api::GetPageRequest,
-        shard: &Shard,
-    ) -> tonic::Result<page_api::GetPageResponse> {
-        let mut stream = shard.stream(Self::is_bulk(&req)).await?;
-        let resp = stream.send(req.clone()).await?;
-
-        // Convert per-request errors into a tonic::Status.
-        if resp.status_code != page_api::GetPageStatusCode::Ok {
-            return Err(tonic::Status::new(
-                resp.status_code.into(),
-                resp.reason.unwrap_or_else(|| String::from("unknown error")),
-            ));
-        }
-
-        // Check that we received the expected pages.
-        if req.rel != resp.rel {
-            return Err(tonic::Status::internal(format!(
-                "shard {} returned wrong relation, expected {} got {}",
-                shard.id, req.rel, resp.rel
-            )));
-        }
-        if !req
-            .block_numbers
-            .iter()
-            .copied()
-            .eq(resp.pages.iter().map(|p| p.block_number))
-        {
-            return Err(tonic::Status::internal(format!(
-                "shard {} returned wrong pages, expected {:?} got {:?}",
-                shard.id,
-                req.block_numbers,
-                resp.pages
-                    .iter()
-                    .map(|page| page.block_number)
-                    .collect::<Vec<_>>()
-            )));
-        }
-
-        Ok(resp)
-    }
-
-    /// Returns the size of a relation, as # of blocks.
-    #[instrument(skip_all, fields(rel=%req.rel, lsn=%req.read_lsn))]
-    pub async fn get_rel_size(
-        &self,
-        req: page_api::GetRelSizeRequest,
-    ) -> tonic::Result<page_api::GetRelSizeResponse> {
-        debug!("sending request: {req:?}");
-        let resp = Self::with_retries(CALL_TIMEOUT, async |_| {
-            // Relation metadata is only available on shard 0.
-            let mut client = self.shards.load_full().get_zero().client().await?;
-            Self::with_timeout(REQUEST_TIMEOUT, client.get_rel_size(req)).await
-        })
-        .await?;
-        debug!("received response: {resp:?}");
-        Ok(resp)
-    }
-
-    /// Fetches an SLRU segment.
-    #[instrument(skip_all, fields(kind=%req.kind, segno=%req.segno, lsn=%req.read_lsn))]
-    pub async fn get_slru_segment(
-        &self,
-        req: page_api::GetSlruSegmentRequest,
-    ) -> tonic::Result<page_api::GetSlruSegmentResponse> {
-        debug!("sending request: {req:?}");
-        let resp = Self::with_retries(CALL_TIMEOUT, async |_| {
-            // SLRU segments are only available on shard 0.
-            let mut client = self.shards.load_full().get_zero().client().await?;
-            Self::with_timeout(REQUEST_TIMEOUT, client.get_slru_segment(req)).await
-        })
-        .await?;
-        debug!("received response: {resp:?}");
-        Ok(resp)
-    }
-
-    /// Runs the given async closure with retries up to the given timeout. Only certain gRPC status
-    /// codes are retried, see [`Retry::should_retry`]. Returns `DeadlineExceeded` on timeout.
-    async fn with_retries<T, F, O>(timeout: Duration, f: F) -> tonic::Result<T>
-    where
-        F: FnMut(usize) -> O, // pass attempt number, starting at 0
-        O: Future<Output = tonic::Result<T>>,
-    {
-        Retry {
-            timeout: Some(timeout),
-            base_backoff: BASE_BACKOFF,
-            max_backoff: MAX_BACKOFF,
-        }
-        .with(f)
-        .await
-    }
-
-    /// Runs the given future with a timeout. Returns `DeadlineExceeded` on timeout.
-    async fn with_timeout<T>(
-        timeout: Duration,
-        f: impl Future<Output = tonic::Result<T>>,
-    ) -> tonic::Result<T> {
-        let started = Instant::now();
-        tokio::time::timeout(timeout, f).await.map_err(|_| {
-            tonic::Status::deadline_exceeded(format!(
-                "request timed out after {:.3}s",
-                started.elapsed().as_secs_f64()
-            ))
-        })?
-    }
-
-    /// Returns true if the request is considered a bulk request and should use the bulk pool.
-    fn is_bulk(req: &page_api::GetPageRequest) -> bool {
-        req.block_numbers.len() >= BULK_THRESHOLD_BATCH_SIZE
-    }
-}
-
-/// Shard specification for a PageserverClient.
-pub struct ShardSpec {
-    /// Maps shard indices to gRPC URLs.
-    ///
-    /// INVARIANT: every shard 0..count is present, and shard 0 is always present.
-    /// INVARIANT: every URL is valid and uses grpc:// scheme.
-    urls: HashMap<ShardIndex, String>,
-    /// The shard count.
-    ///
-    /// NB: this is 0 for unsharded tenants, following `ShardIndex::unsharded()` convention.
-    count: ShardCount,
-    /// The stripe size for these shards.
-    stripe_size: ShardStripeSize,
-}
-
-impl ShardSpec {
-    /// Creates a new shard spec with the given URLs and stripe size. All shards must be given.
-    /// The stripe size may be omitted for unsharded tenants.
-    pub fn new(
-        urls: HashMap<ShardIndex, String>,
-        stripe_size: Option<ShardStripeSize>,
-    ) -> anyhow::Result<Self> {
-        // Compute the shard count.
-        let count = match urls.len() {
-            0 => return Err(anyhow!("no shards provided")),
-            1 => ShardCount::new(0), // NB: unsharded tenants use 0, like `ShardIndex::unsharded()`
-            n if n > u8::MAX as usize => return Err(anyhow!("too many shards: {n}")),
-            n => ShardCount::new(n as u8),
-        };
-
-        // Determine the stripe size. It doesn't matter for unsharded tenants.
-        if stripe_size.is_none() && !count.is_unsharded() {
-            return Err(anyhow!("stripe size must be given for sharded tenants"));
-        }
-        let stripe_size = stripe_size.unwrap_or_default();
-
-        // Validate the shard spec.
-        for (shard_id, url) in &urls {
-            // The shard index must match the computed shard count, even for unsharded tenants.
-            if shard_id.shard_count != count {
-                return Err(anyhow!("invalid shard index {shard_id}, expected {count}"));
-            }
-            // The shard index' number and count must be consistent.
-            if !shard_id.is_unsharded() && shard_id.shard_number.0 >= shard_id.shard_count.0 {
-                return Err(anyhow!("invalid shard index {shard_id}"));
-            }
-            // The above conditions guarantee that we have all shards 0..count: len() matches count,
-            // shard number < count, and numbers are unique (via hashmap).
-
-            // Validate the URL.
-            if PageserverProtocol::from_connstring(url)? != PageserverProtocol::Grpc {
-                return Err(anyhow!("invalid shard URL {url}: must use gRPC"));
-            }
-        }
-
-        Ok(Self {
-            urls,
-            count,
-            stripe_size,
-        })
-    }
-}
-
-/// Tracks the tenant's shards.
-struct Shards {
-    /// Shards by shard index.
-    ///
-    /// INVARIANT: every shard 0..count is present.
-    /// INVARIANT: shard 0 is always present.
-    by_index: HashMap<ShardIndex, Shard>,
-    /// The shard count.
-    ///
-    /// NB: this is 0 for unsharded tenants, following `ShardIndex::unsharded()` convention.
-    count: ShardCount,
-    /// The stripe size. Only used for sharded tenants.
-    stripe_size: ShardStripeSize,
-}
-
-impl Shards {
-    /// Creates a new set of shards based on a shard spec.
-    fn new(
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        shard_spec: ShardSpec,
-        auth_token: Option<String>,
-        compression: Option<CompressionEncoding>,
-    ) -> anyhow::Result<Self> {
-        // NB: the shard spec has already been validated when constructed.
-        let mut shards = HashMap::with_capacity(shard_spec.urls.len());
-        for (shard_id, url) in shard_spec.urls {
-            shards.insert(
-                shard_id,
-                Shard::new(
-                    url,
-                    tenant_id,
-                    timeline_id,
-                    shard_id,
-                    auth_token.clone(),
-                    compression,
-                )?,
-            );
-        }
-
-        Ok(Self {
-            by_index: shards,
-            count: shard_spec.count,
-            stripe_size: shard_spec.stripe_size,
-        })
-    }
-
-    /// Looks up the given shard.
-    #[allow(clippy::result_large_err)] // TODO: check perf impact
-    fn get(&self, shard_id: ShardIndex) -> tonic::Result<&Shard> {
-        self.by_index
-            .get(&shard_id)
-            .ok_or_else(|| tonic::Status::not_found(format!("unknown shard {shard_id}")))
-    }
-
-    /// Returns shard 0.
-    fn get_zero(&self) -> &Shard {
-        self.get(ShardIndex::new(ShardNumber(0), self.count))
-            .expect("always present")
-    }
-}
-
-/// A single shard. Has dedicated resource pools with the following structure:
-///
-/// * Channel pool: MAX_CLIENTS_PER_CHANNEL.
-///   * Client pool: unbounded.
-///     * Stream pool: unbounded.
-/// * Bulk channel pool: MAX_BULK_CLIENTS_PER_CHANNEL.
-///   * Bulk client pool: unbounded.
-///     * Bulk stream pool: unbounded.
-///
-/// We use a separate bulk channel pool with a lower concurrency limit for large batch requests.
-/// This avoids TCP-level head-of-line blocking, and also concentrates large window sizes on a
-/// smaller set of streams/connections, which presumably reduces memory use. Neither of these pools
-/// are bounded, nor do they pipeline requests, so the latency characteristics should be mostly
-/// similar (except for TCP transmission time).
-///
-/// TODO: since we never use bounded pools, we could consider removing the pool limiters. However,
-/// the code is fairly trivial, so we may as well keep them around for now in case we need them.
-struct Shard {
-    /// The shard ID.
-    id: ShardIndex,
-    /// Unary gRPC client pool.
-    client_pool: Arc<ClientPool>,
-    /// GetPage stream pool.
-    stream_pool: Arc<StreamPool>,
-    /// GetPage stream pool for bulk requests.
-    bulk_stream_pool: Arc<StreamPool>,
-}
-
-impl Shard {
-    /// Creates a new shard. It has its own dedicated resource pools.
-    fn new(
-        url: String,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        shard_id: ShardIndex,
-        auth_token: Option<String>,
-        compression: Option<CompressionEncoding>,
-    ) -> anyhow::Result<Self> {
-        // Shard pools for unary requests and non-bulk GetPage requests.
-        let client_pool = ClientPool::new(
-            ChannelPool::new(url.clone(), MAX_CLIENTS_PER_CHANNEL)?,
-            tenant_id,
-            timeline_id,
-            shard_id,
-            auth_token.clone(),
-            compression,
-            None, // unbounded
-        );
-        let stream_pool = StreamPool::new(client_pool.clone(), None); // unbounded
-
-        // Bulk GetPage stream pool for large batches (prefetches, sequential scans, vacuum, etc.).
-        let bulk_stream_pool = StreamPool::new(
-            ClientPool::new(
-                ChannelPool::new(url, MAX_BULK_CLIENTS_PER_CHANNEL)?,
-                tenant_id,
-                timeline_id,
-                shard_id,
-                auth_token,
-                compression,
-                None, // unbounded,
-            ),
-            None, // unbounded
-        );
-
-        Ok(Self {
-            id: shard_id,
-            client_pool,
-            stream_pool,
-            bulk_stream_pool,
-        })
-    }
-
-    /// Returns a pooled client for this shard.
-    #[instrument(skip_all)]
-    async fn client(&self) -> tonic::Result<ClientGuard> {
-        warn_slow(
-            "client pool acquisition",
-            SLOW_THRESHOLD,
-            pin!(self.client_pool.get()),
-        )
-        .await
-    }
-
-    /// Returns a pooled stream for this shard. If `bulk` is `true`, uses the dedicated bulk pool.
-    #[instrument(skip_all, fields(bulk))]
-    async fn stream(&self, bulk: bool) -> tonic::Result<StreamGuard> {
-        let pool = match bulk {
-            false => &self.stream_pool,
-            true => &self.bulk_stream_pool,
-        };
-        warn_slow("stream pool acquisition", SLOW_THRESHOLD, pin!(pool.get())).await
-    }
-}

pageserver/client_grpc/src/lib.rs

@@ -1,6 +0,0 @@
-mod client;
-mod pool;
-mod retry;
-mod split;
-
-pub use client::{PageserverClient, ShardSpec};

pageserver/client_grpc/src/pool.rs

@@ -1,662 +0,0 @@
-//! This module provides various Pageserver gRPC client resource pools.
-//!
-//! These pools are designed to reuse gRPC resources (connections, clients, and streams) across
-//! multiple concurrent callers (i.e. Postgres backends). This avoids the resource cost and latency
-//! of creating dedicated TCP connections and server tasks for every Postgres backend.
-//!
-//! Each resource has its own, nested pool. The pools are custom-built for the properties of each
-//! resource -- they are different enough that a generic pool isn't suitable.
-//!
-//! * ChannelPool: manages gRPC channels (TCP connections) to a single Pageserver. Multiple clients
-//!   can acquire and use the same channel concurrently (via HTTP/2 stream multiplexing), up to a
-//!   per-channel client limit. Channels are closed immediately when empty, and indirectly rely on
-//!   client/stream idle timeouts.
-//!
-//! * ClientPool: manages gRPC clients for a single tenant shard. Each client acquires a (shared)
-//!   channel from the ChannelPool for the client's lifetime. A client can only be acquired by a
-//!   single caller at a time, and is returned to the pool when dropped. Idle clients are removed
-//!   from the pool after a while to free up resources.
-//!
-//! * StreamPool: manages bidirectional gRPC GetPage streams. Each stream acquires a client from the
-//!   ClientPool for the stream's lifetime. A stream can only be acquired by a single caller at a
-//!   time, and is returned to the pool when dropped. Idle streams are removed from the pool after
-//!   a while to free up resources.
-//!
-//!   The stream only supports sending a single, synchronous request at a time, and does not support
-//!   pipelining multiple requests from different callers onto the same stream -- instead, we scale
-//!   out concurrent streams to improve throughput. There are many reasons for this design choice:
-//!
-//!     * It (mostly) eliminates head-of-line blocking. A single stream is processed sequentially by
-//!       a single server task, which may block e.g. on layer downloads, LSN waits, etc.
-//!
-//!     * Cancellation becomes trivial, by closing the stream. Otherwise, if a caller goes away
-//!       (e.g. because of a timeout), the request would still be processed by the server and block
-//!       requests behind it in the stream. It might even block its own timeout retry.
-//!
-//!     * Stream scheduling becomes significantly simpler and cheaper.
-//!
-//!     * Individual callers can still use client-side batching for pipelining.
-//!
-//!     * Idle streams are cheap. Benchmarks show that an idle GetPage stream takes up about 26 KB
-//!       per stream (2.5 GB for 100,000 streams), so we can afford to scale out.
-//!
-//! Each channel corresponds to one TCP connection. Each client unary request and each stream
-//! corresponds to one HTTP/2 stream and server task.
-//!
-//! TODO: error handling (including custom error types).
-//! TODO: observability.
-
-use std::collections::BTreeMap;
-use std::num::NonZero;
-use std::ops::{Deref, DerefMut};
-use std::pin::Pin;
-use std::sync::atomic::{AtomicUsize, Ordering};
-use std::sync::{Arc, Mutex, Weak};
-use std::time::{Duration, Instant};
-
-use futures::{Stream, StreamExt as _};
-use tokio::sync::{OwnedSemaphorePermit, Semaphore, watch};
-use tokio_stream::wrappers::WatchStream;
-use tokio_util::sync::CancellationToken;
-use tonic::codec::CompressionEncoding;
-use tonic::transport::{Channel, Endpoint};
-
-use pageserver_page_api as page_api;
-use utils::id::{TenantId, TimelineId};
-use utils::shard::ShardIndex;
-
-/// Reap clients/streams that have been idle for this long. Channels are reaped immediately when
-/// empty, and indirectly rely on the client/stream idle timeouts.
-///
-/// A stream's client will be reaped after 2x the idle threshold (first stream the client), but
-/// that's okay -- if the stream closes abruptly (e.g. due to timeout or cancellation), we want to
-/// keep its client around in the pool for a while.
-const REAP_IDLE_THRESHOLD: Duration = match cfg!(any(test, feature = "testing")) {
-    false => Duration::from_secs(180),
-    true => Duration::from_secs(1), // exercise reaping in tests
-};
-
-/// Reap idle resources with this interval.
-const REAP_IDLE_INTERVAL: Duration = match cfg!(any(test, feature = "testing")) {
-    false => Duration::from_secs(10),
-    true => Duration::from_secs(1), // exercise reaping in tests
-};
-
-/// A gRPC channel pool, for a single Pageserver. A channel is shared by many clients (via HTTP/2
-/// stream multiplexing), up to `clients_per_channel` -- a new channel will be spun up beyond this.
-/// The pool does not limit the number of channels, and instead relies on `ClientPool` or
-/// `StreamPool` to limit the number of concurrent clients.
-///
-/// The pool is always wrapped in an outer `Arc`, to allow long-lived guards across tasks/threads.
-///
-/// TODO: consider prewarming a set of channels, to avoid initial connection latency.
-/// TODO: consider adding a circuit breaker for errors and fail fast.
-pub struct ChannelPool {
-    /// Pageserver endpoint to connect to.
-    endpoint: Endpoint,
-    /// Max number of clients per channel. Beyond this, a new channel will be created.
-    max_clients_per_channel: NonZero<usize>,
-    /// Open channels.
-    channels: Mutex<BTreeMap<ChannelID, ChannelEntry>>,
-    /// Channel ID generator.
-    next_channel_id: AtomicUsize,
-}
-
-type ChannelID = usize;
-
-struct ChannelEntry {
-    /// The gRPC channel (i.e. TCP connection). Shared by multiple clients.
-    channel: Channel,
-    /// Number of clients using this channel.
-    clients: usize,
-}
-
-impl ChannelPool {
-    /// Creates a new channel pool for the given Pageserver endpoint.
-    pub fn new<E>(endpoint: E, max_clients_per_channel: NonZero<usize>) -> anyhow::Result<Arc<Self>>
-    where
-        E: TryInto<Endpoint> + Send + Sync + 'static,
-        <E as TryInto<Endpoint>>::Error: std::error::Error + Send + Sync,
-    {
-        Ok(Arc::new(Self {
-            endpoint: endpoint.try_into()?,
-            max_clients_per_channel,
-            channels: Mutex::default(),
-            next_channel_id: AtomicUsize::default(),
-        }))
-    }
-
-    /// Acquires a gRPC channel for a client. Multiple clients may acquire the same channel.
-    ///
-    /// This never blocks (except for mutex acquisition). The channel is connected lazily on first
-    /// use, and the `ChannelPool` does not have a channel limit. Channels will be re-established
-    /// automatically on failure (TODO: verify).
-    ///
-    /// Callers should not clone the returned channel, and must hold onto the returned guard as long
-    /// as the channel is in use. It is unfortunately not possible to enforce this: the Protobuf
-    /// client requires an owned `Channel` and we don't have access to the channel's internal
-    /// refcount.
-    ///
-    /// This is not performance-sensitive. It is only called when creating a new client, and clients
-    /// are pooled and reused by `ClientPool`. The total number of channels will also be small. O(n)
-    /// performance is therefore okay.
-    pub fn get(self: &Arc<Self>) -> ChannelGuard {
-        let mut channels = self.channels.lock().unwrap();
-
-        // Try to find an existing channel with available capacity. We check entries in BTreeMap
-        // order, to fill up the lower-ordered channels first. The client/stream pools also prefer
-        // clients with lower-ordered channel IDs first. This will cluster clients in lower-ordered
-        // channels, and free up higher-ordered channels such that they can be reaped.
-        for (&id, entry) in channels.iter_mut() {
-            assert!(
-                entry.clients <= self.max_clients_per_channel.get(),
-                "channel overflow"
-            );
-            assert_ne!(entry.clients, 0, "empty channel not reaped");
-            if entry.clients < self.max_clients_per_channel.get() {
-                entry.clients += 1;
-                return ChannelGuard {
-                    pool: Arc::downgrade(self),
-                    id,
-                    channel: Some(entry.channel.clone()),
-                };
-            }
-        }
-
-        // Create a new channel. We connect lazily on first use, such that we don't block here and
-        // other clients can join onto the same channel while it's connecting.
-        let channel = self.endpoint.connect_lazy();
-
-        let id = self.next_channel_id.fetch_add(1, Ordering::Relaxed);
-        let entry = ChannelEntry {
-            channel: channel.clone(),
-            clients: 1, // account for the guard below
-        };
-        channels.insert(id, entry);
-
-        ChannelGuard {
-            pool: Arc::downgrade(self),
-            id,
-            channel: Some(channel),
-        }
-    }
-}
-
-/// Tracks a channel acquired from the pool. The owned inner channel can be obtained with `take()`,
-/// since the gRPC client requires an owned `Channel`.
-pub struct ChannelGuard {
-    pool: Weak<ChannelPool>,
-    id: ChannelID,
-    channel: Option<Channel>,
-}
-
-impl ChannelGuard {
-    /// Returns the inner owned channel. Panics if called more than once. The caller must hold onto
-    /// the guard as long as the channel is in use, and should not clone it.
-    pub fn take(&mut self) -> Channel {
-        self.channel.take().expect("channel already taken")
-    }
-}
-
-/// Returns the channel to the pool. The channel is closed when empty.
-impl Drop for ChannelGuard {
-    fn drop(&mut self) {
-        let Some(pool) = self.pool.upgrade() else {
-            return; // pool was dropped
-        };
-
-        let mut channels = pool.channels.lock().unwrap();
-        let entry = channels.get_mut(&self.id).expect("unknown channel");
-        assert!(entry.clients > 0, "channel underflow");
-        entry.clients -= 1;
-
-        // Reap empty channels immediately.
-        if entry.clients == 0 {
-            channels.remove(&self.id);
-        }
-    }
-}
-
-/// A pool of gRPC clients for a single tenant shard. Each client acquires a channel from the inner
-/// `ChannelPool`. A client is only given out to single caller at a time. The pool limits the total
-/// number of concurrent clients to `max_clients` via semaphore.
-///
-/// The pool is always wrapped in an outer `Arc`, to allow long-lived guards across tasks/threads.
-pub struct ClientPool {
-    /// Tenant ID.
-    tenant_id: TenantId,
-    /// Timeline ID.
-    timeline_id: TimelineId,
-    /// Shard ID.
-    shard_id: ShardIndex,
-    /// Authentication token, if any.
-    auth_token: Option<String>,
-    /// Compression to use.
-    compression: Option<CompressionEncoding>,
-    /// Channel pool to acquire channels from.
-    channel_pool: Arc<ChannelPool>,
-    /// Limits the max number of concurrent clients for this pool. None if the pool is unbounded.
-    limiter: Option<Arc<Semaphore>>,
-    /// Idle pooled clients. Acquired clients are removed from here and returned on drop.
-    ///
-    /// The first client in the map will be acquired next. The map is sorted by client ID, which in
-    /// turn is sorted by its channel ID, such that we prefer acquiring idle clients from
-    /// lower-ordered channels. This allows us to free up and reap higher-ordered channels.
-    idle: Mutex<BTreeMap<ClientID, ClientEntry>>,
-    /// Reaps idle clients.
-    idle_reaper: Reaper,
-    /// Unique client ID generator.
-    next_client_id: AtomicUsize,
-}
-
-type ClientID = (ChannelID, usize);
-
-struct ClientEntry {
-    /// The pooled gRPC client.
-    client: page_api::Client,
-    /// The channel guard for the channel used by the client.
-    channel_guard: ChannelGuard,
-    /// The client has been idle since this time. All clients in `ClientPool::idle` are idle by
-    /// definition, so this is the time when it was added back to the pool.
-    idle_since: Instant,
-}
-
-impl ClientPool {
-    /// Creates a new client pool for the given tenant shard. Channels are acquired from the given
-    /// `ChannelPool`, which must point to a Pageserver that hosts the tenant shard. Allows up to
-    /// `max_clients` concurrent clients, or unbounded if None.
-    pub fn new(
-        channel_pool: Arc<ChannelPool>,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        shard_id: ShardIndex,
-        auth_token: Option<String>,
-        compression: Option<CompressionEncoding>,
-        max_clients: Option<NonZero<usize>>,
-    ) -> Arc<Self> {
-        let pool = Arc::new(Self {
-            tenant_id,
-            timeline_id,
-            shard_id,
-            auth_token,
-            compression,
-            channel_pool,
-            idle: Mutex::default(),
-            idle_reaper: Reaper::new(REAP_IDLE_THRESHOLD, REAP_IDLE_INTERVAL),
-            limiter: max_clients.map(|max| Arc::new(Semaphore::new(max.get()))),
-            next_client_id: AtomicUsize::default(),
-        });
-        pool.idle_reaper.spawn(&pool);
-        pool
-    }
-
-    /// Gets a client from the pool, or creates a new one if necessary. Connections are established
-    /// lazily and do not block, but this call can block if the pool is at `max_clients`. The client
-    /// is returned to the pool when the guard is dropped.
-    ///
-    /// This is moderately performance-sensitive. It is called for every unary request, but these
-    /// establish a new gRPC stream per request so they're already expensive. GetPage requests use
-    /// the `StreamPool` instead.
-    pub async fn get(self: &Arc<Self>) -> tonic::Result<ClientGuard> {
-        // Acquire a permit if the pool is bounded.
-        let mut permit = None;
-        if let Some(limiter) = self.limiter.clone() {
-            permit = Some(limiter.acquire_owned().await.expect("never closed"));
-        }
-
-        // Fast path: acquire an idle client from the pool.
-        if let Some((id, entry)) = self.idle.lock().unwrap().pop_first() {
-            return Ok(ClientGuard {
-                pool: Arc::downgrade(self),
-                id,
-                client: Some(entry.client),
-                channel_guard: Some(entry.channel_guard),
-                permit,
-            });
-        }
-
-        // Construct a new client.
-        let mut channel_guard = self.channel_pool.get();
-        let client = page_api::Client::new(
-            channel_guard.take(),
-            self.tenant_id,
-            self.timeline_id,
-            self.shard_id,
-            self.auth_token.clone(),
-            self.compression,
-        )
-        .map_err(|err| tonic::Status::internal(format!("failed to create client: {err}")))?;
-
-        Ok(ClientGuard {
-            pool: Arc::downgrade(self),
-            id: (
-                channel_guard.id,
-                self.next_client_id.fetch_add(1, Ordering::Relaxed),
-            ),
-            client: Some(client),
-            channel_guard: Some(channel_guard),
-            permit,
-        })
-    }
-}
-
-impl Reapable for ClientPool {
-    /// Reaps clients that have been idle since before the cutoff.
-    fn reap_idle(&self, cutoff: Instant) {
-        self.idle
-            .lock()
-            .unwrap()
-            .retain(|_, entry| entry.idle_since >= cutoff)
-    }
-}
-
-/// A client acquired from the pool. The inner client can be accessed via Deref. The client is
-/// returned to the pool when dropped.
-pub struct ClientGuard {
-    pool: Weak<ClientPool>,
-    id: ClientID,
-    client: Option<page_api::Client>,     // Some until dropped
-    channel_guard: Option<ChannelGuard>,  // Some until dropped
-    permit: Option<OwnedSemaphorePermit>, // None if pool is unbounded
-}
-
-impl Deref for ClientGuard {
-    type Target = page_api::Client;
-
-    fn deref(&self) -> &Self::Target {
-        self.client.as_ref().expect("not dropped")
-    }
-}
-
-impl DerefMut for ClientGuard {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        self.client.as_mut().expect("not dropped")
-    }
-}
-
-/// Returns the client to the pool.
-impl Drop for ClientGuard {
-    fn drop(&mut self) {
-        let Some(pool) = self.pool.upgrade() else {
-            return; // pool was dropped
-        };
-
-        let entry = ClientEntry {
-            client: self.client.take().expect("dropped once"),
-            channel_guard: self.channel_guard.take().expect("dropped once"),
-            idle_since: Instant::now(),
-        };
-        pool.idle.lock().unwrap().insert(self.id, entry);
-
-        _ = self.permit; // returned on drop, referenced for visibility
-    }
-}
-
-/// A pool of bidirectional gRPC streams. Currently only used for GetPage streams. Each stream
-/// acquires a client from the inner `ClientPool` for the stream's lifetime.
-///
-/// Individual streams only send a single request at a time, and do not pipeline multiple callers
-/// onto the same stream. Instead, we scale out the number of concurrent streams. This is primarily
-/// to eliminate head-of-line blocking. See the module documentation for more details.
-///
-/// TODO: consider making this generic over request and response types; not currently needed.
-pub struct StreamPool {
-    /// The client pool to acquire clients from. Must be unbounded.
-    client_pool: Arc<ClientPool>,
-    /// Idle pooled streams. Acquired streams are removed from here and returned on drop.
-    ///
-    /// The first stream in the map will be acquired next. The map is sorted by stream ID, which is
-    /// equivalent to the client ID and in turn sorted by its channel ID. This way we prefer
-    /// acquiring idle streams from lower-ordered channels, which allows us to free up and reap
-    /// higher-ordered channels.
-    idle: Mutex<BTreeMap<StreamID, StreamEntry>>,
-    /// Limits the max number of concurrent streams. None if the pool is unbounded.
-    limiter: Option<Arc<Semaphore>>,
-    /// Reaps idle streams.
-    idle_reaper: Reaper,
-}
-
-/// The stream ID. Reuses the inner client ID.
-type StreamID = ClientID;
-
-/// A pooled stream.
-struct StreamEntry {
-    /// The bidirectional stream.
-    stream: BiStream,
-    /// The time when this stream was last used, i.e. when it was put back into `StreamPool::idle`.
-    idle_since: Instant,
-}
-
-/// A bidirectional GetPage stream and its client. Can send requests and receive responses.
-struct BiStream {
-    /// The owning client. Holds onto the channel slot while the stream is alive.
-    client: ClientGuard,
-    /// Stream for sending requests. Uses a watch channel, so it can only send a single request at a
-    /// time, and the caller must await the response before sending another request. This is
-    /// enforced by `StreamGuard::send`.
-    sender: watch::Sender<page_api::GetPageRequest>,
-    /// Stream for receiving responses.
-    receiver: Pin<Box<dyn Stream<Item = tonic::Result<page_api::GetPageResponse>> + Send>>,
-}
-
-impl StreamPool {
-    /// Creates a new stream pool, using the given client pool. It will use up to `max_streams`
-    /// concurrent streams.
-    ///
-    /// The client pool must be unbounded. The stream pool will enforce its own limits, and because
-    /// streams are long-lived they can cause persistent starvation if they exhaust the client pool.
-    /// The stream pool should generally have its own dedicated client pool (but it can share a
-    /// channel pool with others since these are always unbounded).
-    pub fn new(client_pool: Arc<ClientPool>, max_streams: Option<NonZero<usize>>) -> Arc<Self> {
-        assert!(client_pool.limiter.is_none(), "bounded client pool");
-        let pool = Arc::new(Self {
-            client_pool,
-            idle: Mutex::default(),
-            limiter: max_streams.map(|max_streams| Arc::new(Semaphore::new(max_streams.get()))),
-            idle_reaper: Reaper::new(REAP_IDLE_THRESHOLD, REAP_IDLE_INTERVAL),
-        });
-        pool.idle_reaper.spawn(&pool);
-        pool
-    }
-
-    /// Acquires an available stream from the pool, or spins up a new stream if all streams are
-    /// full. Returns a guard that can be used to send requests and await the responses. Blocks if
-    /// the pool is full.
-    ///
-    /// This is very performance-sensitive, as it is on the GetPage hot path.
-    ///
-    /// TODO: is a `Mutex<BTreeMap>` performant enough? Will it become too contended? We can't
-    /// trivially use e.g. DashMap or sharding, because we want to pop lower-ordered streams first
-    /// to free up higher-ordered channels.
-    pub async fn get(self: &Arc<Self>) -> tonic::Result<StreamGuard> {
-        // Acquire a permit if the pool is bounded.
-        let mut permit = None;
-        if let Some(limiter) = self.limiter.clone() {
-            permit = Some(limiter.acquire_owned().await.expect("never closed"));
-        }
-
-        // Fast path: acquire an idle stream from the pool.
-        if let Some((_, entry)) = self.idle.lock().unwrap().pop_first() {
-            return Ok(StreamGuard {
-                pool: Arc::downgrade(self),
-                stream: Some(entry.stream),
-                can_reuse: true,
-                permit,
-            });
-        }
-
-        // Spin up a new stream. Uses a watch channel to send a single request at a time, since
-        // `StreamGuard::send` enforces this anyway and it avoids unnecessary channel overhead.
-        let mut client = self.client_pool.get().await?;
-
-        let (req_tx, req_rx) = watch::channel(page_api::GetPageRequest::default());
-        let req_stream = WatchStream::from_changes(req_rx);
-        let resp_stream = client.get_pages(req_stream).await?;
-
-        Ok(StreamGuard {
-            pool: Arc::downgrade(self),
-            stream: Some(BiStream {
-                client,
-                sender: req_tx,
-                receiver: Box::pin(resp_stream),
-            }),
-            can_reuse: true,
-            permit,
-        })
-    }
-}
-
-impl Reapable for StreamPool {
-    /// Reaps streams that have been idle since before the cutoff.
-    fn reap_idle(&self, cutoff: Instant) {
-        self.idle
-            .lock()
-            .unwrap()
-            .retain(|_, entry| entry.idle_since >= cutoff);
-    }
-}
-
-/// A stream acquired from the pool. Returned to the pool when dropped, unless there are still
-/// in-flight requests on the stream, or the stream failed.
-pub struct StreamGuard {
-    pool: Weak<StreamPool>,
-    stream: Option<BiStream>,             // Some until dropped
-    can_reuse: bool,                      // returned to pool if true
-    permit: Option<OwnedSemaphorePermit>, // None if pool is unbounded
-}
-
-impl StreamGuard {
-    /// Sends a request on the stream and awaits the response. If the future is dropped before it
-    /// resolves (e.g. due to a timeout or cancellation), the stream will be closed to cancel the
-    /// request and is not returned to the pool. The same is true if the stream errors, in which
-    /// case the caller can't send further requests on the stream.
-    ///
-    /// We only support sending a single request at a time, to eliminate head-of-line blocking. See
-    /// module documentation for details.
-    ///
-    /// NB: errors are often returned as `GetPageResponse::status_code` instead of `tonic::Status`
-    /// to avoid tearing down the stream for per-request errors. Callers must check this.
-    pub async fn send(
-        &mut self,
-        req: page_api::GetPageRequest,
-    ) -> tonic::Result<page_api::GetPageResponse> {
-        let req_id = req.request_id;
-        let stream = self.stream.as_mut().expect("not dropped");
-
-        // Mark the stream as not reusable while the request is in flight. We can't return the
-        // stream to the pool until we receive the response, to avoid head-of-line blocking and
-        // stale responses. Failed streams can't be reused either.
-        if !self.can_reuse {
-            return Err(tonic::Status::internal("stream can't be reused"));
-        }
-        self.can_reuse = false;
-
-        // Send the request and receive the response.
-        //
-        // NB: this uses a watch channel, so it's unsafe to change this code to pipeline requests.
-        stream
-            .sender
-            .send(req)
-            .map_err(|_| tonic::Status::unavailable("stream closed"))?;
-
-        let resp = stream
-            .receiver
-            .next()
-            .await
-            .ok_or_else(|| tonic::Status::unavailable("stream closed"))??;
-
-        if resp.request_id != req_id {
-            return Err(tonic::Status::internal(format!(
-                "response ID {} does not match request ID {}",
-                resp.request_id, req_id
-            )));
-        }
-
-        // Success, mark the stream as reusable.
-        self.can_reuse = true;
-
-        Ok(resp)
-    }
-}
-
-impl Drop for StreamGuard {
-    fn drop(&mut self) {
-        let Some(pool) = self.pool.upgrade() else {
-            return; // pool was dropped
-        };
-
-        // If the stream isn't reusable, it can't be returned to the pool.
-        if !self.can_reuse {
-            return;
-        }
-
-        // Place the idle stream back into the pool.
-        let entry = StreamEntry {
-            stream: self.stream.take().expect("dropped once"),
-            idle_since: Instant::now(),
-        };
-        pool.idle
-            .lock()
-            .unwrap()
-            .insert(entry.stream.client.id, entry);
-
-        _ = self.permit; // returned on drop, referenced for visibility
-    }
-}
-
-/// Periodically reaps idle resources from a pool.
-struct Reaper {
-    /// The task check interval.
-    interval: Duration,
-    /// The threshold for reaping idle resources.
-    threshold: Duration,
-    /// Cancels the reaper task. Cancelled when the reaper is dropped.
-    cancel: CancellationToken,
-}
-
-impl Reaper {
-    /// Creates a new reaper.
-    pub fn new(threshold: Duration, interval: Duration) -> Self {
-        Self {
-            cancel: CancellationToken::new(),
-            threshold,
-            interval,
-        }
-    }
-
-    /// Spawns a task to periodically reap idle resources from the given task pool. The task is
-    /// cancelled when the reaper is dropped.
-    pub fn spawn(&self, pool: &Arc<impl Reapable>) {
-        // NB: hold a weak pool reference, otherwise the task will prevent dropping the pool.
-        let pool = Arc::downgrade(pool);
-        let cancel = self.cancel.clone();
-        let (interval, threshold) = (self.interval, self.threshold);
-
-        tokio::spawn(async move {
-            loop {
-                tokio::select! {
-                    _ = tokio::time::sleep(interval) => {
-                        let Some(pool) = pool.upgrade() else {
-                            return; // pool was dropped
-                        };
-                        pool.reap_idle(Instant::now() - threshold);
-                    }
-
-                    _ = cancel.cancelled() => return,
-                }
-            }
-        });
-    }
-}
-
-impl Drop for Reaper {
-    fn drop(&mut self) {
-        self.cancel.cancel(); // cancel reaper task
-    }
-}
-
-/// A reapable resource pool.
-trait Reapable: Send + Sync + 'static {
-    /// Reaps resources that have been idle since before the given cutoff.
-    fn reap_idle(&self, cutoff: Instant);
-}