Notes on cutover

rename
docs: add hot secondaries rfc
2026-07-07 14:10:43 +00:00 · 2025-03-14 19:19:28 +00:00 · 2025-03-14 15:33:23 +00:00 · 2025-03-13 17:22:56 +00:00
92 changed files with 1041 additions and 2256 deletions
--- a/.github/workflows/benchmarking.yml
+++ b/.github/workflows/benchmarking.yml
@@ -141,8 +141,6 @@ jobs:
          --ignore test_runner/performance/test_physical_replication.py
          --ignore test_runner/performance/test_perf_ingest_using_pgcopydb.py
          --ignore test_runner/performance/test_cumulative_statistics_persistence.py
-          --ignore test_runner/performance/test_perf_many_relations.py
-          --ignore test_runner/performance/test_perf_oltp_large_tenant.py
      env:
        BENCHMARK_CONNSTR: ${{ steps.create-neon-project.outputs.dsn }}
        VIP_VAP_ACCESS_TOKEN: "${{ secrets.VIP_VAP_ACCESS_TOKEN }}"
--- a/.github/workflows/build_and_test.yml
+++ b/.github/workflows/build_and_test.yml
@@ -1061,7 +1061,7 @@ jobs:
          exit 1

  deploy:
-    needs: [ check-permissions, push-neon-image-dev, push-compute-image-dev, push-neon-image-prod, push-compute-image-prod, meta, build-and-test-locally, trigger-custom-extensions-build-and-wait ]
+    needs: [ check-permissions, push-neon-image-prod, push-compute-image-prod, meta, build-and-test-locally, trigger-custom-extensions-build-and-wait ]
    # `!failure() && !cancelled()` is required because the workflow depends on the job that can be skipped: `push-neon-image-prod` and `push-compute-image-prod`
    if: ${{ contains(fromJSON('["push-main", "storage-release", "proxy-release", "compute-release"]'), needs.meta.outputs.run-kind) && !failure() && !cancelled() }}
    permissions:
--- a/.github/workflows/periodic_pagebench.yml
+++ b/.github/workflows/periodic_pagebench.yml
@@ -3,12 +3,12 @@ name: Periodic pagebench performance test on dedicated EC2 machine in eu-central
 on:
  schedule:
    # * is a special character in YAML so you have to quote this string
-    #        ┌───────────── minute (0 - 59)
-    #        │   ┌───────────── hour (0 - 23)
-    #        │   │ ┌───────────── day of the month (1 - 31)
-    #        │   │ │ ┌───────────── month (1 - 12 or JAN-DEC)
-    #        │   │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
-    - cron: '0 */3 * * *' # Runs every 3 hours
+    #          ┌───────────── minute (0 - 59)
+    #          │ ┌───────────── hour (0 - 23)
+    #          │ │ ┌───────────── day of the month (1 - 31)
+    #          │ │ │ ┌───────────── month (1 - 12 or JAN-DEC)
+    #          │ │ │ │ ┌───────────── day of the week (0 - 6 or SUN-SAT)
+    - cron:  '0 18 * * *' # Runs at 6 PM UTC every day
  workflow_dispatch: # Allows manual triggering of the workflow
    inputs:
      commit_hash:
@@ -78,10 +78,8 @@ jobs:
      run: |
        if [ -z "$INPUT_COMMIT_HASH" ]; then
          echo "COMMIT_HASH=$(curl -s https://api.github.com/repos/neondatabase/neon/commits/main | jq -r '.sha')" >> $GITHUB_ENV
-          echo "COMMIT_HASH_TYPE=latest" >> $GITHUB_ENV
        else
          echo "COMMIT_HASH=$INPUT_COMMIT_HASH" >> $GITHUB_ENV
-          echo "COMMIT_HASH_TYPE=manual" >> $GITHUB_ENV
        fi

    - name: Start Bench with run_id
@@ -91,7 +89,7 @@ jobs:
        -H 'accept: application/json' \
        -H 'Content-Type: application/json' \
        -H "Authorization: Bearer $API_KEY" \
-        -d "{\"neonRepoCommitHash\": \"${COMMIT_HASH}\", \"neonRepoCommitHashType\": \"${COMMIT_HASH_TYPE}\"}"
+        -d "{\"neonRepoCommitHash\": \"${COMMIT_HASH}\"}"

    - name: Poll Test Status
      id: poll_step
--- a/5
+++ b/5
@@ -1,9 +1,8 @@
 # Autoscaling
 /libs/vm_monitor/ @neondatabase/autoscaling

-# DevProd & PerfCorr
-/.github/ @neondatabase/developer-productivity @neondatabase/performance-correctness
-/test_runner/	@neondatabase/performance-correctness
+# DevProd
+/.github/ @neondatabase/developer-productivity

 # Compute
 /pgxn/ @neondatabase/compute
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -783,28 +783,6 @@ dependencies = [
 "tracing",
 ]

-[[package]]
-name = "axum-extra"
-version = "0.10.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "460fc6f625a1f7705c6cf62d0d070794e94668988b1c38111baeec177c715f7b"
-dependencies = [
- "axum",
- "axum-core",
- "bytes",
- "futures-util",
- "headers",
- "http 1.1.0",
- "http-body 1.0.0",
- "http-body-util",
- "mime",
- "pin-project-lite",
- "serde",
- "tower 0.5.2",
- "tower-layer",
- "tower-service",
-]
-
 [[package]]
 name = "azure_core"
 version = "0.21.0"
@@ -947,9 +925,9 @@ checksum = "0ea22880d78093b0cbe17c89f64a7d457941e65759157ec6cb31a31d652b05e5"

 [[package]]
 name = "base64"
-version = "0.21.7"
+version = "0.21.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+checksum = "3f1e31e207a6b8fb791a38ea3105e6cb541f55e4d029902d3039a4ad07cc4105"

 [[package]]
 name = "base64"
@@ -1127,9 +1105,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"

 [[package]]
 name = "cc"
-version = "1.2.16"
+version = "1.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
+checksum = "b16803a61b81d9eabb7eae2588776c4c1e584b738ede45fdbb4c972cec1e9945"
 dependencies = [
 "jobserver",
 "libc",
@@ -1327,7 +1305,6 @@ dependencies = [
 "aws-sdk-s3",
 "aws-smithy-types",
 "axum",
- "axum-extra",
 "base64 0.13.1",
 "bytes",
 "camino",
@@ -1339,7 +1316,6 @@ dependencies = [
 "flate2",
 "futures",
 "http 1.1.0",
- "jsonwebtoken",
 "metrics",
 "nix 0.27.1",
 "notify",
@@ -2321,7 +2297,7 @@ name = "framed-websockets"
 version = "0.1.0"
 source = "git+https://github.com/neondatabase/framed-websockets#34eff3d6f8cfccbc5f35e4f65314ff7328621127"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.1",
 "bytemuck",
 "bytes",
 "futures-core",
@@ -2677,7 +2653,7 @@ version = "7.5.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.1",
 "byteorder",
 "crossbeam-channel",
 "flate2",
@@ -2685,30 +2661,6 @@ dependencies = [
 "num-traits",
 ]

-[[package]]
-name = "headers"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "322106e6bd0cba2d5ead589ddb8150a13d7c4217cf80d7c4f682ca994ccc6aa9"
-dependencies = [
- "base64 0.21.7",
- "bytes",
- "headers-core",
- "http 1.1.0",
- "httpdate",
- "mime",
- "sha1",
-]
-
-[[package]]
-name = "headers-core"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4"
-dependencies = [
- "http 1.1.0",
-]
-
 [[package]]
 name = "heck"
 version = "0.5.0"
@@ -2846,9 +2798,12 @@ name = "http-utils"
 version = "0.1.0"
 dependencies = [
 "anyhow",
+ "backtrace",
 "bytes",
 "fail",
+ "flate2",
 "hyper 0.14.30",
+ "inferno 0.12.0",
 "itertools 0.10.5",
 "jemalloc_pprof",
 "metrics",
@@ -3347,9 +3302,9 @@ checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"

 [[package]]
 name = "jemalloc_pprof"
-version = "0.7.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5622af6d21ff86ed7797ef98e11b8f302da25ec69a7db9f6cde8e2e1c8df9992"
+checksum = "1a883828bd6a4b957cd9f618886ff19e5f3ebd34e06ba0e855849e049fef32fb"
 dependencies = [
 "anyhow",
 "libc",
@@ -3433,7 +3388,7 @@ version = "9.2.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.1",
 "js-sys",
 "pem",
 "ring",
@@ -3548,9 +3503,9 @@ dependencies = [

 [[package]]
 name = "mappings"
-version = "0.7.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e434981a332777c2b3062652d16a55f8e74fa78e6b1882633f0d77399c84fc2a"
+checksum = "ce9229c438fbf1c333926e2053c4c091feabbd40a1b590ec62710fea2384af9e"
 dependencies = [
 "anyhow",
 "libc",
@@ -4303,7 +4258,6 @@ dependencies = [
 "tracing",
 "url",
 "utils",
- "uuid",
 "wal_decoder",
 "walkdir",
 "workspace_hack",
@@ -4386,9 +4340,9 @@ dependencies = [

 [[package]]
 name = "papaya"
-version = "0.2.0"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aab21828b6b5952fdadd6c377728ffae53ec3a21b2febc47319ab65741f7e2fd"
+checksum = "dc7c76487f7eaa00a0fc1d7f88dc6b295aec478d11b0fc79f857b62c2874124c"
 dependencies = [
 "equivalent",
 "seize",
@@ -4516,7 +4470,7 @@ version = "3.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.1",
 "serde",
 ]

@@ -4840,14 +4794,12 @@ dependencies = [

 [[package]]
 name = "pprof_util"
-version = "0.7.0"
+version = "0.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9fa015c78eed2130951e22c58d2095849391e73817ab2e74f71b0b9f63dd8416"
+checksum = "65c568b3f8c1c37886ae07459b1946249e725c315306b03be5632f84c239f781"
 dependencies = [
 "anyhow",
- "backtrace",
 "flate2",
- "inferno 0.12.0",
 "num",
 "paste",
 "prost",
@@ -5627,16 +5579,16 @@ dependencies = [

 [[package]]
 name = "ring"
-version = "0.17.13"
+version = "0.17.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee"
+checksum = "684d5e6e18f669ccebf64a92236bb7db9a34f07be010e3627368182027180866"
 dependencies = [
 "cc",
- "cfg-if",
 "getrandom 0.2.11",
 "libc",
+ "spin",
 "untrusted",
- "windows-sys 0.52.0",
+ "windows-sys 0.48.0",
 ]

 [[package]]
@@ -5863,7 +5815,7 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.1",
 ]

 [[package]]
@@ -5872,7 +5824,7 @@ version = "2.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f48172685e6ff52a556baa527774f61fcaa884f59daf3375c62a3f1cd2549dab"
 dependencies = [
- "base64 0.21.7",
+ "base64 0.21.1",
 "rustls-pki-types",
 ]

@@ -6111,9 +6063,9 @@ dependencies = [

 [[package]]
 name = "seize"
-version = "0.5.0"
+version = "0.4.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4b8d813387d566f627f3ea1b914c068aac94c40ae27ec43f5f33bde65abefe7"
+checksum = "d84b0c858bdd30cb56f5597f8b3bf702ec23829e652cc636a1e5a7b9de46ae93"
 dependencies = [
 "libc",
 "windows-sys 0.52.0",
@@ -7406,12 +7358,10 @@ version = "0.6.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "403fa3b783d4b626a8ad51d766ab03cb6d2dbfc46b1c5d4448395e6628dc9697"
 dependencies = [
- "base64 0.22.1",
 "bitflags 2.8.0",
 "bytes",
 "http 1.1.0",
 "http-body 1.0.0",
- "mime",
 "pin-project-lite",
 "tower-layer",
 "tower-service",
@@ -7765,6 +7715,7 @@ dependencies = [
 "anyhow",
 "arc-swap",
 "async-compression",
+ "backtrace",
 "bincode",
 "byteorder",
 "bytes",
@@ -8318,7 +8269,7 @@ dependencies = [
 "ahash",
 "anyhow",
 "base64 0.13.1",
- "base64 0.21.7",
+ "base64 0.21.1",
 "base64ct",
 "bytes",
 "camino",
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -53,6 +53,7 @@ anyhow = { version = "1.0", features = ["backtrace"] }
 arc-swap = "1.6"
 async-compression = { version = "0.4.0", features = ["tokio", "gzip", "zstd"] }
 atomic-take = "1.1.0"
+backtrace = "0.3.74"
 flate2 = "1.0.26"
 assert-json-diff = "2"
 async-stream = "0.3"
@@ -67,7 +68,6 @@ aws-credential-types = "1.2.0"
 aws-sigv4 = { version = "1.2", features = ["sign-http"] }
 aws-types = "1.3"
 axum = { version = "0.8.1", features = ["ws"] }
-axum-extra = { version = "0.10.0", features = ["typed-header"] }
 base64 = "0.13.0"
 bincode = "1.3"
 bindgen = "0.71"
@@ -114,10 +114,11 @@ hyper-util = "0.1"
 tokio-tungstenite = "0.21.0"
 indexmap = "2"
 indoc = "2"
+inferno = "0.12.0"
 ipnet = "2.10.0"
 itertools = "0.10"
 itoa = "1.0.11"
-jemalloc_pprof = { version = "0.7", features = ["symbolize", "flamegraph"] }
+jemalloc_pprof = "0.6"
 jsonwebtoken = "9"
 lasso = "0.7"
 libc = "0.2"
@@ -192,7 +193,7 @@ toml = "0.8"
 toml_edit = "0.22"
 tonic = {version = "0.12.3", default-features = false, features = ["channel", "tls", "tls-roots"]}
 tower = { version = "0.5.2", default-features = false }
-tower-http = { version = "0.6.2", features = ["auth", "request-id", "trace"] }
+tower-http = { version = "0.6.2", features = ["request-id", "trace"] }

 # This revision uses opentelemetry 0.27. There's no tag for it.
 tower-otel = { git = "https://github.com/mattiapenati/tower-otel", rev = "56a7321053bcb72443888257b622ba0d43a11fcd" }
--- a/compute/compute-node.Dockerfile
+++ b/compute/compute-node.Dockerfile
@@ -1484,7 +1484,7 @@ WORKDIR /ext-src
 COPY compute/patches/pg_duckdb_v031.patch .
 COPY compute/patches/duckdb_v120.patch .
 # pg_duckdb build requires source dir to be a git repo to get submodules
-# allow neon_superuser to execute some functions that in pg_duckdb are available to superuser only:
+# allow neon_superuser to execute some functions that in pg_duckdb are available to superuser only: 
 # - extension management function duckdb.install_extension()
 # - access to duckdb.extensions table and its sequence
 RUN git clone --depth 1 --branch v0.3.1 https://github.com/duckdb/pg_duckdb.git pg_duckdb-src && \
@@ -1499,8 +1499,8 @@ ARG PG_VERSION
 COPY --from=pg_duckdb-src /ext-src/ /ext-src/
 WORKDIR /ext-src/pg_duckdb-src
 RUN make install -j $(getconf _NPROCESSORS_ONLN) && \
-    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control
-
+    echo 'trusted = true' >> /usr/local/pgsql/share/extension/pg_duckdb.control 
+        
 #########################################################################################
 #
 # Layer "pg_repack"
@@ -1758,15 +1758,15 @@ ARG TARGETARCH
 # test_runner/regress/test_compute_metrics.py
 # See comment on the top of the file regading `echo`, `-e` and `\n`
 RUN if [ "$TARGETARCH" = "amd64" ]; then\
-        postgres_exporter_sha256='59aa4a7bb0f7d361f5e05732f5ed8c03cc08f78449cef5856eadec33a627694b';\
+        postgres_exporter_sha256='027e75dda7af621237ff8f5ac66b78a40b0093595f06768612b92b1374bd3105';\
        pgbouncer_exporter_sha256='c9f7cf8dcff44f0472057e9bf52613d93f3ffbc381ad7547a959daa63c5e84ac';\
        sql_exporter_sha256='38e439732bbf6e28ca4a94d7bc3686d3fa1abdb0050773d5617a9efdb9e64d08';\
    else\
-        postgres_exporter_sha256='d1dedea97f56c6d965837bfd1fbb3e35a3b4a4556f8cccee8bd513d8ee086124';\
+        postgres_exporter_sha256='131a376d25778ff9701a4c81f703f179e0b58db5c2c496e66fa43f8179484786';\
        pgbouncer_exporter_sha256='217c4afd7e6492ae904055bc14fe603552cf9bac458c063407e991d68c519da3';\
        sql_exporter_sha256='11918b00be6e2c3a67564adfdb2414fdcbb15a5db76ea17d1d1a944237a893c6';\
    fi\
-    && curl -sL https://github.com/prometheus-community/postgres_exporter/releases/download/v0.17.1/postgres_exporter-0.17.1.linux-${TARGETARCH}.tar.gz\
+    && curl -sL https://github.com/prometheus-community/postgres_exporter/releases/download/v0.16.0/postgres_exporter-0.16.0.linux-${TARGETARCH}.tar.gz\
     | tar xzf - --strip-components=1 -C.\
    && curl -sL https://github.com/prometheus-community/pgbouncer_exporter/releases/download/v0.10.2/pgbouncer_exporter-0.10.2.linux-${TARGETARCH}.tar.gz\
     | tar xzf - --strip-components=1 -C.\
@@ -1933,7 +1933,6 @@ RUN apt update && \
        locales \
        procps \
        ca-certificates \
-        rsyslog \
        $VERSION_INSTALLS && \
    apt clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \
    localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8
@@ -1979,13 +1978,6 @@ COPY --from=sql_exporter_preprocessor --chmod=0644 /home/nonroot/compute/etc/neo
 # Make the libraries we built available
 RUN echo '/usr/local/lib' >> /etc/ld.so.conf && /sbin/ldconfig

-# rsyslog config permissions
-# directory for rsyslogd pid file
-RUN mkdir /var/run/rsyslogd && \
-    chown -R postgres:postgres /var/run/rsyslogd && \
-    chown -R postgres:postgres /etc/rsyslog.d/
-
-
 ENV LANG=en_US.utf8
 USER postgres
 ENTRYPOINT ["/usr/local/bin/compute_ctl"]
--- a/compute/etc/neon_collector.jsonnet
+++ b/compute/etc/neon_collector.jsonnet
@@ -29,7 +29,6 @@
    import 'sql_exporter/lfc_approximate_working_set_size.libsonnet',
    import 'sql_exporter/lfc_approximate_working_set_size_windows.libsonnet',
    import 'sql_exporter/lfc_cache_size_limit.libsonnet',
-    import 'sql_exporter/lfc_chunk_size.libsonnet',
    import 'sql_exporter/lfc_hits.libsonnet',
    import 'sql_exporter/lfc_misses.libsonnet',
    import 'sql_exporter/lfc_used.libsonnet',
--- a/compute/etc/sql_exporter/db_total_size.sql
+++ b/compute/etc/sql_exporter/db_total_size.sql
@@ -1,5 +1 @@
-SELECT sum(pg_database_size(datname)) AS total
-FROM pg_database
-- Ignore invalid databases, as we will likely have problems with
-- getting their size from the Pageserver.
-WHERE datconnlimit != -2;
+SELECT sum(pg_database_size(datname)) AS total FROM pg_database;
--- a/compute/etc/sql_exporter/lfc_chunk_size.libsonnet
+++ b/compute/etc/sql_exporter/lfc_chunk_size.libsonnet
@@ -1,10 +0,0 @@
-{
-  metric_name: 'lfc_chunk_size',
-  type: 'gauge',
-  help: 'LFC chunk size, measured in 8KiB pages',
-  key_labels: null,
-  values: [
-    'lfc_chunk_size_pages',
-  ],
-  query: importstr 'sql_exporter/lfc_chunk_size.sql',
-}
--- a/compute/etc/sql_exporter/lfc_chunk_size.sql
+++ b/compute/etc/sql_exporter/lfc_chunk_size.sql
@@ -1 +0,0 @@
-SELECT lfc_value AS lfc_chunk_size_pages FROM neon.neon_lfc_stats WHERE lfc_key = 'file_cache_chunk_size_pages';
--- a/compute/etc/sql_exporter/pg_stats_userdb.sql
+++ b/compute/etc/sql_exporter/pg_stats_userdb.sql
@@ -1,20 +1,10 @@
 -- We export stats for 10 non-system databases. Without this limit it is too
 -- easy to abuse the system by creating lots of databases.

-SELECT pg_database_size(datname) AS db_size,
-  deadlocks,
-  tup_inserted AS inserted,
-  tup_updated AS updated,
-  tup_deleted AS deleted,
-  datname
+SELECT pg_database_size(datname) AS db_size, deadlocks, tup_inserted AS inserted,
+  tup_updated AS updated, tup_deleted AS deleted, datname
 FROM pg_stat_database
 WHERE datname IN (
  SELECT datname FROM pg_database
-  -- Ignore invalid databases, as we will likely have problems with
-  -- getting their size from the Pageserver.
-  WHERE datconnlimit != -2
-    AND datname <> 'postgres'
-    AND NOT datistemplate
-  ORDER BY oid
-  LIMIT 10
+  WHERE datname <> 'postgres' AND NOT datistemplate ORDER BY oid LIMIT 10
 );
--- a/compute/vm-image-spec-bookworm.yaml
+++ b/compute/vm-image-spec-bookworm.yaml
@@ -39,10 +39,6 @@ commands:
    user: nobody
    sysvInitAction: respawn
    shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499'
-  - name: rsyslogd
-    user: postgres
-    sysvInitAction: respawn
-    shell: '/usr/sbin/rsyslogd -n -i /var/run/rsyslogd/rsyslogd.pid -f /etc/compute_rsyslog.conf'
 shutdownHook: |
  su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10'
 files:
@@ -58,7 +54,7 @@ files:
      # regardless of hostname (ALL)
      #
      # Also allow it to shut down the VM. The fast_import job does that when it's finished.
-      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd
+      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff
  - filename: cgconfig.conf
    content: |
      # Configuration for cgroups in VM compute nodes
@@ -73,12 +69,6 @@ files:
          }
          memory {}
      }
-# Create dummy rsyslog config, because it refuses to start without at least one action configured.
-# compute_ctl will rewrite this file with the actual configuration, if needed.
-  - filename: compute_rsyslog.conf
-    content: |
-      *.*    /dev/null
-      $IncludeConfig /etc/rsyslog.d/*.conf
 build: |
  # Build cgroup-tools
  #
@@ -142,12 +132,6 @@ merge: |
  RUN set -e \
      && chmod 0644 /etc/cgconfig.conf

-
-  COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
-  RUN chmod 0666 /etc/compute_rsyslog.conf
-  RUN chmod 0666 /var/log/
-
-
  COPY --from=libcgroup-builder /libcgroup-install/bin/*  /usr/bin/
  COPY --from=libcgroup-builder /libcgroup-install/lib/*  /usr/lib/
  COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/
--- a/compute/vm-image-spec-bullseye.yaml
+++ b/compute/vm-image-spec-bullseye.yaml
@@ -39,10 +39,6 @@ commands:
    user: nobody
    sysvInitAction: respawn
    shell: '/bin/sql_exporter -config.file=/etc/sql_exporter_autoscaling.yml -web.listen-address=:9499'
-  - name: rsyslogd
-    user: postgres
-    sysvInitAction: respawn
-    shell: '/usr/sbin/rsyslogd -n -i /var/run/rsyslogd/rsyslogd.pid -f /etc/compute_rsyslog.conf'
 shutdownHook: |
  su -p postgres --session-command '/usr/local/bin/pg_ctl stop -D /var/db/postgres/compute/pgdata -m fast --wait -t 10'
 files:
@@ -58,7 +54,7 @@ files:
      # regardless of hostname (ALL)
      #
      # Also allow it to shut down the VM. The fast_import job does that when it's finished.
-      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff, /usr/sbin/rsyslogd
+      postgres ALL=(root) NOPASSWD: /neonvm/bin/resize-swap, /neonvm/bin/set-disk-quota, /neonvm/bin/poweroff
  - filename: cgconfig.conf
    content: |
      # Configuration for cgroups in VM compute nodes
@@ -73,12 +69,6 @@ files:
          }
          memory {}
      }
-# Create dummy rsyslog config, because it refuses to start without at least one action configured.
-# compute_ctl will rewrite this file with the actual configuration, if needed.
-  - filename: compute_rsyslog.conf
-    content: |
-      *.*    /dev/null
-      $IncludeConfig /etc/rsyslog.d/*.conf
 build: |
  # Build cgroup-tools
  #
@@ -138,11 +128,6 @@ merge: |
  RUN set -e \
      && chmod 0644 /etc/cgconfig.conf

-  COPY compute_rsyslog.conf /etc/compute_rsyslog.conf
-  RUN chmod 0666 /etc/compute_rsyslog.conf
-  RUN chmod 0666 /var/log/
-
-
  COPY --from=libcgroup-builder /libcgroup-install/bin/*  /usr/bin/
  COPY --from=libcgroup-builder /libcgroup-install/lib/*  /usr/lib/
  COPY --from=libcgroup-builder /libcgroup-install/sbin/* /usr/sbin/
--- a/compute_tools/Cargo.toml
+++ b/compute_tools/Cargo.toml
@@ -17,7 +17,6 @@ aws-sdk-kms.workspace = true
 aws-smithy-types.workspace = true
 anyhow.workspace = true
 axum = { workspace = true, features = [] }
-axum-extra.workspace = true
 camino.workspace = true
 chrono.workspace = true
 cfg-if.workspace = true
@@ -26,7 +25,6 @@ fail.workspace = true
 flate2.workspace = true
 futures.workspace = true
 http.workspace = true
-jsonwebtoken.workspace = true
 metrics.workspace = true
 nix.workspace = true
 notify.workspace = true
--- a/compute_tools/src/bin/compute_ctl.rs
+++ b/compute_tools/src/bin/compute_ctl.rs
@@ -179,7 +179,6 @@ fn main() -> Result<()> {
            live_config_allowed: cli_spec.live_config_allowed,
        },
        cli_spec.spec,
-        cli_spec.compute_ctl_config,
    )?;

    let exit_code = compute_node.run()?;
--- a/compute_tools/src/compute.rs
+++ b/compute_tools/src/compute.rs
@@ -11,10 +11,8 @@ use std::{env, fs};
 use anyhow::{Context, Result};
 use chrono::{DateTime, Utc};
 use compute_api::privilege::Privilege;
-use compute_api::responses::{ComputeCtlConfig, ComputeMetrics, ComputeStatus};
-use compute_api::spec::{
-    ComputeAudit, ComputeFeature, ComputeMode, ComputeSpec, ExtVersion, PgIdent,
-};
+use compute_api::responses::{ComputeMetrics, ComputeStatus};
+use compute_api::spec::{ComputeFeature, ComputeMode, ComputeSpec, ExtVersion, PgIdent};
 use futures::StreamExt;
 use futures::future::join_all;
 use futures::stream::FuturesUnordered;
@@ -37,7 +35,6 @@ use crate::logger::startup_context_from_env;
 use crate::lsn_lease::launch_lsn_lease_bg_task_for_static;
 use crate::monitor::launch_monitor;
 use crate::pg_helpers::*;
-use crate::rsyslog::configure_audit_rsyslog;
 use crate::spec::*;
 use crate::swap::resize_swap;
 use crate::sync_sk::{check_if_synced, ping_safekeeper};
@@ -135,8 +132,6 @@ pub struct ComputeState {
    /// passed by the control plane with a /configure HTTP request.
    pub pspec: Option<ParsedSpec>,

-    pub compute_ctl_config: ComputeCtlConfig,
-
    /// If the spec is passed by a /configure request, 'startup_span' is the
    /// /configure request's tracing span. The main thread enters it when it
    /// processes the compute startup, so that the compute startup is considered
@@ -160,7 +155,6 @@ impl ComputeState {
            last_active: None,
            error: None,
            pspec: None,
-            compute_ctl_config: ComputeCtlConfig::default(),
            startup_span: None,
            metrics: ComputeMetrics::default(),
        }
@@ -297,12 +291,81 @@ struct StartVmMonitorResult {
    vm_monitor: Option<tokio::task::JoinHandle<Result<()>>>,
 }

+pub(crate) fn construct_superuser_query(spec: &ComputeSpec) -> String {
+    let roles = spec
+        .cluster
+        .roles
+        .iter()
+        .map(|r| escape_literal(&r.name))
+        .collect::<Vec<_>>();
+
+    let dbs = spec
+        .cluster
+        .databases
+        .iter()
+        .map(|db| escape_literal(&db.name))
+        .collect::<Vec<_>>();
+
+    let roles_decl = if roles.is_empty() {
+        String::from("roles text[] := NULL;")
+    } else {
+        format!(
+            r#"
+               roles text[] := ARRAY(SELECT rolname
+                                     FROM pg_catalog.pg_roles
+                                     WHERE rolname IN ({}));"#,
+            roles.join(", ")
+        )
+    };
+
+    let database_decl = if dbs.is_empty() {
+        String::from("dbs text[] := NULL;")
+    } else {
+        format!(
+            r#"
+               dbs text[] := ARRAY(SELECT datname
+                                   FROM pg_catalog.pg_database
+                                   WHERE datname IN ({}));"#,
+            dbs.join(", ")
+        )
+    };
+
+    // ALL PRIVILEGES grants CREATE, CONNECT, and TEMPORARY on all databases
+    // (see https://www.postgresql.org/docs/current/ddl-priv.html)
+    let query = format!(
+        r#"
+            DO $$
+                DECLARE
+                    r text;
+                    {}
+                    {}
+                BEGIN
+                    IF NOT EXISTS (
+                        SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser')
+                    THEN
+                        CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN REPLICATION BYPASSRLS IN ROLE pg_read_all_data, pg_write_all_data;
+                        IF array_length(roles, 1) IS NOT NULL THEN
+                            EXECUTE format('GRANT neon_superuser TO %s',
+                                           array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(roles) as x), ', '));
+                            FOREACH r IN ARRAY roles LOOP
+                                EXECUTE format('ALTER ROLE %s CREATEROLE CREATEDB', quote_ident(r));
+                            END LOOP;
+                        END IF;
+                        IF array_length(dbs, 1) IS NOT NULL THEN
+                            EXECUTE format('GRANT ALL PRIVILEGES ON DATABASE %s TO neon_superuser',
+                                           array_to_string(ARRAY(SELECT quote_ident(x) FROM unnest(dbs) as x), ', '));
+                        END IF;
+                    END IF;
+                END
+            $$;"#,
+        roles_decl, database_decl,
+    );
+
+    query
+}
+
 impl ComputeNode {
-    pub fn new(
-        params: ComputeNodeParams,
-        cli_spec: Option<ComputeSpec>,
-        compute_ctl_config: ComputeCtlConfig,
-    ) -> Result<Self> {
+    pub fn new(params: ComputeNodeParams, cli_spec: Option<ComputeSpec>) -> Result<Self> {
        let connstr = params.connstr.as_str();
        let conn_conf = postgres::config::Config::from_str(connstr)
            .context("cannot build postgres config from connstr")?;
@@ -314,7 +377,6 @@ impl ComputeNode {
            let pspec = ParsedSpec::try_from(cli_spec).map_err(|msg| anyhow::anyhow!(msg))?;
            new_state.pspec = Some(pspec);
        }
-        new_state.compute_ctl_config = compute_ctl_config;

        Ok(ComputeNode {
            params,
@@ -343,19 +405,11 @@ impl ComputeNode {

        // Launch the external HTTP server first, so that we can serve control plane
        // requests while configuration is still in progress.
-        crate::http::server::Server::External {
-            port: this.params.external_http_port,
-            jwks: this.state.lock().unwrap().compute_ctl_config.jwks.clone(),
-            compute_id: this.params.compute_id.clone(),
-        }
-        .launch(&this);
+        crate::http::server::Server::External(this.params.external_http_port).launch(&this);

        // The internal HTTP server could be launched later, but there isn't much
        // sense in waiting.
-        crate::http::server::Server::Internal {
-            port: this.params.internal_http_port,
-        }
-        .launch(&this);
+        crate::http::server::Server::Internal(this.params.internal_http_port).launch(&this);

        // If we got a spec from the CLI already, use that. Otherwise wait for the
        // control plane to pass it to us with a /configure HTTP request
@@ -414,8 +468,6 @@ impl ComputeNode {
                        // Kills the actual task running the monitor
                        handle.abort();
                    }
-                } else {
-                    _ = vm_monitor; // appease unused lint on macOS
                }
            }
        }
@@ -606,23 +658,6 @@ impl ComputeNode {
            });
        }

-        // Configure and start rsyslog if necessary
-        if let ComputeAudit::Hipaa = pspec.spec.audit_log_level {
-            let remote_endpoint = std::env::var("AUDIT_LOGGING_ENDPOINT").unwrap_or("".to_string());
-            if remote_endpoint.is_empty() {
-                anyhow::bail!("AUDIT_LOGGING_ENDPOINT is empty");
-            }
-
-            let log_directory_path = Path::new(&self.params.pgdata).join("log");
-            // TODO: make this more robust
-            // now rsyslog starts once and there is no monitoring or restart if it fails
-            configure_audit_rsyslog(
-                log_directory_path.to_str().unwrap(),
-                "hipaa",
-                &remote_endpoint,
-            )?;
-        }
-
        // Launch remaining service threads
        let _monitor_handle = launch_monitor(self);
        let _configurator_handle = launch_configurator(self);
@@ -756,7 +791,6 @@ impl ComputeNode {
                };
                StartVmMonitorResult { token, vm_monitor }
            } else {
-                _ = disable_lfc_resizing; // appease unused lint on macOS
                StartVmMonitorResult { }
            }
        }
--- a/compute_tools/src/config.rs
+++ b/compute_tools/src/config.rs
@@ -1,4 +1,3 @@
-use anyhow::Result;
 use std::fmt::Write as FmtWrite;
 use std::fs::{File, OpenOptions};
 use std::io;
@@ -6,11 +5,10 @@ use std::io::Write;
 use std::io::prelude::*;
 use std::path::Path;

-use compute_api::spec::{ComputeAudit, ComputeMode, ComputeSpec, GenericOption};
+use anyhow::Result;
+use compute_api::spec::{ComputeMode, ComputeSpec, GenericOption};

-use crate::pg_helpers::{
-    GenericOptionExt, GenericOptionsSearch, PgOptionsSerialize, escape_conf_value,
-};
+use crate::pg_helpers::{GenericOptionExt, PgOptionsSerialize, escape_conf_value};

 /// Check that `line` is inside a text file and put it there if it is not.
 /// Create file if it doesn't exist.
@@ -140,54 +138,6 @@ pub fn write_postgres_conf(
        writeln!(file, "# Managed by compute_ctl: end")?;
    }

-    // If audit logging is enabled, configure pgaudit.
-    //
-    // Note, that this is called after the settings from spec are written.
-    // This way we always override the settings from the spec
-    // and don't allow the user or the control plane admin to change them.
-    if let ComputeAudit::Hipaa = spec.audit_log_level {
-        writeln!(file, "# Managed by compute_ctl audit settings: begin")?;
-        // This log level is very verbose
-        // but this is necessary for HIPAA compliance.
-        writeln!(file, "pgaudit.log='all'")?;
-        writeln!(file, "pgaudit.log_parameter=on")?;
-        // Disable logging of catalog queries
-        // The catalog doesn't contain sensitive data, so we don't need to audit it.
-        writeln!(file, "pgaudit.log_catalog=off")?;
-        // Set log rotation to 5 minutes
-        // TODO: tune this after performance testing
-        writeln!(file, "pgaudit.log_rotation_age=5")?;
-
-        // Add audit shared_preload_libraries, if they are not present.
-        //
-        // The caller who sets the flag is responsible for ensuring that the necessary
-        // shared_preload_libraries are present in the compute image,
-        // otherwise the compute start will fail.
-        if let Some(libs) = spec.cluster.settings.find("shared_preload_libraries") {
-            let mut extra_shared_preload_libraries = String::new();
-            if !libs.contains("pgaudit") {
-                extra_shared_preload_libraries.push_str(",pgaudit");
-            }
-            if !libs.contains("pgauditlogtofile") {
-                extra_shared_preload_libraries.push_str(",pgauditlogtofile");
-            }
-            writeln!(
-                file,
-                "shared_preload_libraries='{}{}'",
-                libs, extra_shared_preload_libraries
-            )?;
-        } else {
-            // Typically, this should be unreacheable,
-            // because we always set at least some shared_preload_libraries in the spec
-            // but let's handle it explicitly anyway.
-            writeln!(
-                file,
-                "shared_preload_libraries='neon,pgaudit,pgauditlogtofile'"
-            )?;
-        }
-        writeln!(file, "# Managed by compute_ctl audit settings: end")?;
-    }
-
    writeln!(file, "neon.extension_server_port={}", extension_server_port)?;

    if spec.drop_subscriptions_before_start {
--- a/compute_tools/src/config_template/compute_audit_rsyslog_template.conf
+++ b/compute_tools/src/config_template/compute_audit_rsyslog_template.conf
@@ -1,10 +0,0 @@
-# Load imfile module to read log files
-module(load="imfile")
-
-# Input configuration for log files in the specified directory
-# Replace {log_directory} with the directory containing the log files
-input(type="imfile" File="{log_directory}/*.log" Tag="{tag}" Severity="info" Facility="local0")
-global(workDirectory="/var/log")
-
-# Forward logs to remote syslog server
-*.* @@{remote_endpoint}
--- a/compute_tools/src/extension_server.rs
+++ b/compute_tools/src/extension_server.rs
@@ -253,31 +253,27 @@ pub fn create_control_files(remote_extensions: &RemoteExtSpec, pgbin: &str) {
    }
 }

-// Do request to extension storage proxy, e.g.,
+// Do request to extension storage proxy, i.e.
 // curl http://pg-ext-s3-gateway/latest/v15/extensions/anon.tar.zst
-// using HTTP GET and return the response body as bytes.
+// using HHTP GET
+// and return the response body as bytes
+//
 async fn download_extension_tar(ext_remote_storage: &str, ext_path: &str) -> Result<Bytes> {
    let uri = format!("{}/{}", ext_remote_storage, ext_path);
-    let filename = Path::new(ext_path)
-        .file_name()
-        .unwrap_or_else(|| std::ffi::OsStr::new("unknown"))
-        .to_str()
-        .unwrap_or("unknown")
-        .to_string();

-    info!("Downloading extension file '{}' from uri {}", filename, uri);
+    info!("Download extension {} from uri {}", ext_path, uri);

    match do_extension_server_request(&uri).await {
        Ok(resp) => {
            info!("Successfully downloaded remote extension data {}", ext_path);
            REMOTE_EXT_REQUESTS_TOTAL
-                .with_label_values(&[&StatusCode::OK.to_string(), &filename])
+                .with_label_values(&[&StatusCode::OK.to_string()])
                .inc();
            Ok(resp)
        }
        Err((msg, status)) => {
            REMOTE_EXT_REQUESTS_TOTAL
-                .with_label_values(&[&status, &filename])
+                .with_label_values(&[&status])
                .inc();
            bail!(msg);
        }
--- a/compute_tools/src/http/extract/mod.rs
+++ b/compute_tools/src/http/extract/mod.rs
@@ -1,9 +1,7 @@
 pub(crate) mod json;
 pub(crate) mod path;
 pub(crate) mod query;
-pub(crate) mod request_id;

 pub(crate) use json::Json;
 pub(crate) use path::Path;
 pub(crate) use query::Query;
-pub(crate) use request_id::RequestId;
--- a/compute_tools/src/http/extract/request_id.rs
+++ b/compute_tools/src/http/extract/request_id.rs
@@ -1,86 +0,0 @@
-use std::{
-    fmt::Display,
-    ops::{Deref, DerefMut},
-};
-
-use axum::{extract::FromRequestParts, response::IntoResponse};
-use http::{StatusCode, request::Parts};
-
-use crate::http::{JsonResponse, headers::X_REQUEST_ID};
-
-/// Extract the request ID from the `X-Request-Id` header.
-#[derive(Debug, Clone, Default)]
-pub(crate) struct RequestId(pub String);
-
-#[derive(Debug)]
-/// Rejection used for [`RequestId`].
-///
-/// Contains one variant for each way the [`RequestId`] extractor can
-/// fail.
-pub(crate) enum RequestIdRejection {
-    /// The request is missing the header.
-    MissingRequestId,
-
-    /// The value of the header is invalid UTF-8.
-    InvalidUtf8,
-}
-
-impl RequestIdRejection {
-    pub fn status(&self) -> StatusCode {
-        match self {
-            RequestIdRejection::MissingRequestId => StatusCode::INTERNAL_SERVER_ERROR,
-            RequestIdRejection::InvalidUtf8 => StatusCode::BAD_REQUEST,
-        }
-    }
-
-    pub fn message(&self) -> String {
-        match self {
-            RequestIdRejection::MissingRequestId => "request ID is missing",
-            RequestIdRejection::InvalidUtf8 => "request ID is invalid UTF-8",
-        }
-        .to_string()
-    }
-}
-
-impl IntoResponse for RequestIdRejection {
-    fn into_response(self) -> axum::response::Response {
-        JsonResponse::error(self.status(), self.message())
-    }
-}
-
-impl<S> FromRequestParts<S> for RequestId
-where
-    S: Send + Sync,
-{
-    type Rejection = RequestIdRejection;
-
-    async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result<Self, Self::Rejection> {
-        match parts.headers.get(X_REQUEST_ID) {
-            Some(value) => match value.to_str() {
-                Ok(request_id) => Ok(Self(request_id.to_string())),
-                Err(_) => Err(RequestIdRejection::InvalidUtf8),
-            },
-            None => Err(RequestIdRejection::MissingRequestId),
-        }
-    }
-}
-
-impl Deref for RequestId {
-    type Target = String;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl DerefMut for RequestId {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.0
-    }
-}
-
-impl Display for RequestId {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.write_str(&self.0)
-    }
-}
--- a/compute_tools/src/http/headers.rs
+++ b/compute_tools/src/http/headers.rs
@@ -1,2 +0,0 @@
-/// Constant for `X-Request-Id` header.
-pub const X_REQUEST_ID: &str = "x-request-id";
--- a/compute_tools/src/http/middleware/authorize.rs
+++ b/compute_tools/src/http/middleware/authorize.rs
@@ -1,145 +0,0 @@
-use std::{collections::HashSet, net::SocketAddr};
-
-use anyhow::{Result, anyhow};
-use axum::{RequestExt, body::Body, extract::ConnectInfo};
-use axum_extra::{
-    TypedHeader,
-    headers::{Authorization, authorization::Bearer},
-};
-use futures::future::BoxFuture;
-use http::{Request, Response, StatusCode};
-use jsonwebtoken::{Algorithm, DecodingKey, TokenData, Validation, jwk::JwkSet};
-use serde::Deserialize;
-use tower_http::auth::AsyncAuthorizeRequest;
-use tracing::warn;
-
-use crate::http::{JsonResponse, extract::RequestId};
-
-#[derive(Clone, Debug, Deserialize)]
-pub(in crate::http) struct Claims {
-    compute_id: String,
-}
-
-#[derive(Clone, Debug)]
-pub(in crate::http) struct Authorize {
-    compute_id: String,
-    jwks: JwkSet,
-    validation: Validation,
-}
-
-impl Authorize {
-    pub fn new(compute_id: String, jwks: JwkSet) -> Self {
-        let mut validation = Validation::new(Algorithm::EdDSA);
-        // Nothing is currently required
-        validation.required_spec_claims = HashSet::new();
-        validation.validate_exp = true;
-        // Unused by the control plane
-        validation.validate_aud = false;
-        // Unused by the control plane
-        validation.validate_nbf = false;
-
-        Self {
-            compute_id,
-            jwks,
-            validation,
-        }
-    }
-}
-
-impl AsyncAuthorizeRequest<Body> for Authorize {
-    type RequestBody = Body;
-    type ResponseBody = Body;
-    type Future = BoxFuture<'static, Result<Request<Body>, Response<Self::ResponseBody>>>;
-
-    fn authorize(&mut self, mut request: Request<Body>) -> Self::Future {
-        let compute_id = self.compute_id.clone();
-        let jwks = self.jwks.clone();
-        let validation = self.validation.clone();
-
-        Box::pin(async move {
-            let request_id = request.extract_parts::<RequestId>().await.unwrap();
-
-            // TODO: Remove this check after a successful rollout
-            if jwks.keys.is_empty() {
-                warn!(%request_id, "Authorization has not been configured");
-
-                return Ok(request);
-            }
-
-            let connect_info = request
-                .extract_parts::<ConnectInfo<SocketAddr>>()
-                .await
-                .unwrap();
-
-            // In the event the request is coming from the loopback interface,
-            // allow all requests
-            if connect_info.ip().is_loopback() {
-                warn!(%request_id, "Bypassed authorization because request is coming from the loopback interface");
-
-                return Ok(request);
-            }
-
-            let TypedHeader(Authorization(bearer)) = request
-                .extract_parts::<TypedHeader<Authorization<Bearer>>>()
-                .await
-                .map_err(|_| {
-                    JsonResponse::error(StatusCode::BAD_REQUEST, "invalid authorization token")
-                })?;
-
-            let data = match Self::verify(&jwks, bearer.token(), &validation) {
-                Ok(claims) => claims,
-                Err(e) => return Err(JsonResponse::error(StatusCode::UNAUTHORIZED, e)),
-            };
-
-            if data.claims.compute_id != compute_id {
-                return Err(JsonResponse::error(
-                    StatusCode::UNAUTHORIZED,
-                    "invalid claims in authorization token",
-                ));
-            }
-
-            // Make claims available to any subsequent middleware or request
-            // handlers
-            request.extensions_mut().insert(data.claims);
-
-            Ok(request)
-        })
-    }
-}
-
-impl Authorize {
-    /// Verify the token using the JSON Web Key set and return the token data.
-    fn verify(jwks: &JwkSet, token: &str, validation: &Validation) -> Result<TokenData<Claims>> {
-        debug_assert!(!jwks.keys.is_empty());
-
-        for jwk in jwks.keys.iter() {
-            let decoding_key = match DecodingKey::from_jwk(jwk) {
-                Ok(key) => key,
-                Err(e) => {
-                    warn!(
-                        "Failed to construct decoding key from {}: {}",
-                        jwk.common.key_id.as_ref().unwrap(),
-                        e
-                    );
-
-                    continue;
-                }
-            };
-
-            match jsonwebtoken::decode::<Claims>(token, &decoding_key, validation) {
-                Ok(data) => return Ok(data),
-                Err(e) => {
-                    warn!(
-                        "Failed to decode authorization token using {}: {}",
-                        jwk.common.key_id.as_ref().unwrap(),
-                        e
-                    );
-
-                    continue;
-                }
-            }
-        }
-
-        Err(anyhow!("Failed to verify authorization token"))
-    }
-}
--- a/compute_tools/src/http/middleware/mod.rs
+++ b/compute_tools/src/http/middleware/mod.rs
@@ -1 +0,0 @@
-pub(in crate::http) mod authorize;
--- a/compute_tools/src/http/mod.rs
+++ b/compute_tools/src/http/mod.rs
@@ -7,8 +7,6 @@ use serde::Serialize;
 use tracing::error;

 mod extract;
-mod headers;
-mod middleware;
 mod routes;
 pub mod server;

--- a/compute_tools/src/http/server.rs
+++ b/compute_tools/src/http/server.rs
@@ -10,58 +10,48 @@ use axum::middleware::{self, Next};
 use axum::response::{IntoResponse, Response};
 use axum::routing::{get, post};
 use http::StatusCode;
-use jsonwebtoken::jwk::JwkSet;
 use tokio::net::TcpListener;
 use tower::ServiceBuilder;
-use tower_http::{
-    auth::AsyncRequireAuthorizationLayer, request_id::PropagateRequestIdLayer, trace::TraceLayer,
-};
-use tracing::{Span, error, info};
+use tower_http::request_id::PropagateRequestIdLayer;
+use tower_http::trace::TraceLayer;
+use tracing::{Span, debug, error, info};
 use uuid::Uuid;

-use super::{
-    headers::X_REQUEST_ID,
-    middleware::authorize::Authorize,
-    routes::{
-        check_writability, configure, database_schema, dbs_and_roles, extension_server, extensions,
-        grants, insights, metrics, metrics_json, status, terminate,
-    },
+use super::routes::{
+    check_writability, configure, database_schema, dbs_and_roles, extension_server, extensions,
+    grants, insights, metrics, metrics_json, status, terminate,
 };
 use crate::compute::ComputeNode;

+const X_REQUEST_ID: &str = "x-request-id";
+
 /// `compute_ctl` has two servers: internal and external. The internal server
 /// binds to the loopback interface and handles communication from clients on
 /// the compute. The external server is what receives communication from the
 /// control plane, the metrics scraper, etc. We make the distinction because
 /// certain routes in `compute_ctl` only need to be exposed to local processes
 /// like Postgres via the neon extension and local_proxy.
-#[derive(Clone, Debug)]
+#[derive(Clone, Copy, Debug)]
 pub enum Server {
-    Internal {
-        port: u16,
-    },
-    External {
-        port: u16,
-        jwks: JwkSet,
-        compute_id: String,
-    },
+    Internal(u16),
+    External(u16),
 }

 impl Display for Server {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
-            Server::Internal { .. } => f.write_str("internal"),
-            Server::External { .. } => f.write_str("external"),
+            Server::Internal(_) => f.write_str("internal"),
+            Server::External(_) => f.write_str("external"),
        }
    }
 }

-impl From<&Server> for Router<Arc<ComputeNode>> {
-    fn from(server: &Server) -> Self {
+impl From<Server> for Router<Arc<ComputeNode>> {
+    fn from(server: Server) -> Self {
        let mut router = Router::<Arc<ComputeNode>>::new();

        router = match server {
-            Server::Internal { .. } => {
+            Server::Internal(_) => {
                router = router
                    .route(
                        "/extension_server/{*filename}",
@@ -79,71 +69,59 @@ impl From<&Server> for Router<Arc<ComputeNode>> {

                router
            }
-            Server::External {
-                jwks, compute_id, ..
-            } => {
-                let unauthenticated_router =
-                    Router::<Arc<ComputeNode>>::new().route("/metrics", get(metrics::get_metrics));
-
-                let authenticated_router = Router::<Arc<ComputeNode>>::new()
-                    .route("/check_writability", post(check_writability::is_writable))
-                    .route("/configure", post(configure::configure))
-                    .route("/database_schema", get(database_schema::get_schema_dump))
-                    .route("/dbs_and_roles", get(dbs_and_roles::get_catalog_objects))
-                    .route("/insights", get(insights::get_insights))
-                    .route("/metrics.json", get(metrics_json::get_metrics))
-                    .route("/status", get(status::get_status))
-                    .route("/terminate", post(terminate::terminate))
-                    .layer(AsyncRequireAuthorizationLayer::new(Authorize::new(
-                        compute_id.clone(),
-                        jwks.clone(),
-                    )));
-
-                router
-                    .merge(unauthenticated_router)
-                    .merge(authenticated_router)
-            }
+            Server::External(_) => router
+                .route("/check_writability", post(check_writability::is_writable))
+                .route("/configure", post(configure::configure))
+                .route("/database_schema", get(database_schema::get_schema_dump))
+                .route("/dbs_and_roles", get(dbs_and_roles::get_catalog_objects))
+                .route("/insights", get(insights::get_insights))
+                .route("/metrics", get(metrics::get_metrics))
+                .route("/metrics.json", get(metrics_json::get_metrics))
+                .route("/status", get(status::get_status))
+                .route("/terminate", post(terminate::terminate)),
        };

-        router
-            .fallback(Server::handle_404)
-            .method_not_allowed_fallback(Server::handle_405)
-            .layer(
-                ServiceBuilder::new()
-                    .layer(tower_otel::trace::HttpLayer::server(tracing::Level::INFO))
-                    // Add this middleware since we assume the request ID exists
-                    .layer(middleware::from_fn(maybe_add_request_id_header))
-                    .layer(
-                        TraceLayer::new_for_http()
-                            .on_request(|request: &http::Request<_>, _span: &Span| {
-                                let request_id = request
+        router.fallback(Server::handle_404).method_not_allowed_fallback(Server::handle_405).layer(
+            ServiceBuilder::new()
+                // Add this middleware since we assume the request ID exists
+                .layer(middleware::from_fn(maybe_add_request_id_header))
+                .layer(
+                    TraceLayer::new_for_http()
+                        .on_request(|request: &http::Request<_>, _span: &Span| {
+                            let request_id = request
+                                .headers()
+                                .get(X_REQUEST_ID)
+                                .unwrap()
+                                .to_str()
+                                .unwrap();
+
+                            match request.uri().path() {
+                                "/metrics" => {
+                                    debug!(%request_id, "{} {}", request.method(), request.uri())
+                                }
+                                _ => info!(%request_id, "{} {}", request.method(), request.uri()),
+                            };
+                        })
+                        .on_response(
+                            |response: &http::Response<_>, latency: Duration, _span: &Span| {
+                                let request_id = response
                                    .headers()
                                    .get(X_REQUEST_ID)
                                    .unwrap()
                                    .to_str()
                                    .unwrap();

-                                info!(%request_id, "{} {}", request.method(), request.uri());
-                            })
-                            .on_response(
-                                |response: &http::Response<_>, latency: Duration, _span: &Span| {
-                                    let request_id = response
-                                        .headers()
-                                        .get(X_REQUEST_ID)
-                                        .unwrap()
-                                        .to_str()
-                                        .unwrap();
-
-                                    info!(
-                                        %request_id,
-                                        code = response.status().as_u16(),
-                                        latency = latency.as_millis()
-                                    );
-                                },
-                            ),
-                    )
-                    .layer(PropagateRequestIdLayer::x_request_id()),
-            )
+                                info!(
+                                    %request_id,
+                                    code = response.status().as_u16(),
+                                    latency = latency.as_millis()
+                                )
+                            },
+                        ),
+                )
+                .layer(PropagateRequestIdLayer::x_request_id()),
+        )
+            .layer(tower_otel::trace::HttpLayer::server(tracing::Level::INFO))
    }
 }

@@ -167,15 +145,15 @@ impl Server {
        match self {
            // TODO: Change this to Ipv6Addr::LOCALHOST when the GitHub runners
            // allow binding to localhost
-            Server::Internal { .. } => IpAddr::from(Ipv6Addr::UNSPECIFIED),
-            Server::External { .. } => IpAddr::from(Ipv6Addr::UNSPECIFIED),
+            Server::Internal(_) => IpAddr::from(Ipv6Addr::UNSPECIFIED),
+            Server::External(_) => IpAddr::from(Ipv6Addr::UNSPECIFIED),
        }
    }

-    fn port(&self) -> u16 {
+    fn port(self) -> u16 {
        match self {
-            Server::Internal { port, .. } => *port,
-            Server::External { port, .. } => *port,
+            Server::Internal(port) => port,
+            Server::External(port) => port,
        }
    }

@@ -202,9 +180,7 @@ impl Server {
            );
        }

-        let router = Router::from(&self)
-            .with_state(compute)
-            .into_make_service_with_connect_info::<SocketAddr>();
+        let router = Router::from(self).with_state(compute);

        if let Err(e) = axum::serve(listener, router).await {
            error!("compute_ctl {} HTTP server error: {}", self, e);
--- a/compute_tools/src/lib.rs
+++ b/compute_tools/src/lib.rs
@@ -21,7 +21,6 @@ mod migration;
 pub mod monitor;
 pub mod params;
 pub mod pg_helpers;
-pub mod rsyslog;
 pub mod spec;
 mod spec_apply;
 pub mod swap;
--- a/compute_tools/src/metrics.rs
+++ b/compute_tools/src/metrics.rs
@@ -54,7 +54,9 @@ pub(crate) static REMOTE_EXT_REQUESTS_TOTAL: Lazy<IntCounterVec> = Lazy::new(||
    register_int_counter_vec!(
        "compute_ctl_remote_ext_requests_total",
        "Total number of requests made by compute_ctl to download extensions from S3 proxy by status",
-        &["http_status", "filename"]
+        // Do not use any labels like extension name yet.
+        // We can add them later if needed.
+        &["http_status"]
    )
    .expect("failed to define a metric")
 });
--- a/compute_tools/src/pg_helpers.rs
+++ b/compute_tools/src/pg_helpers.rs
@@ -186,40 +186,15 @@ impl DatabaseExt for Database {
 /// Postgres SQL queries and DATABASE_URL.
 pub trait Escaping {
    fn pg_quote(&self) -> String;
-    fn pg_quote_dollar(&self) -> (String, String);
 }

 impl Escaping for PgIdent {
    /// This is intended to mimic Postgres quote_ident(), but for simplicity it
    /// always quotes provided string with `""` and escapes every `"`.
    /// **Not idempotent**, i.e. if string is already escaped it will be escaped again.
-    /// N.B. it's not useful for escaping identifiers that are used inside WHERE
-    /// clause, use `escape_literal()` instead.
    fn pg_quote(&self) -> String {
-        format!("\"{}\"", self.replace('"', "\"\""))
-    }
-
-    /// This helper is intended to be used for dollar-escaping strings for usage
-    /// inside PL/pgSQL procedures. In addition to dollar-escaping the string,
-    /// it also returns a tag that is intended to be used inside the outer
-    /// PL/pgSQL procedure. If you do not need an outer tag, just discard it.
-    /// Here we somewhat mimic the logic of Postgres' `pg_get_functiondef()`,
-    /// <https://github.com/postgres/postgres/blob/8b49392b270b4ac0b9f5c210e2a503546841e832/src/backend/utils/adt/ruleutils.c#L2924>
-    fn pg_quote_dollar(&self) -> (String, String) {
-        let mut tag: String = "".to_string();
-        let mut outer_tag = "x".to_string();
-
-        // Find the first suitable tag that is not present in the string.
-        // Postgres' max role/DB name length is 63 bytes, so even in the
-        // worst case it won't take long.
-        while self.contains(&format!("${tag}$")) || self.contains(&format!("${outer_tag}$")) {
-            tag += "x";
-            outer_tag = tag.clone() + "x";
-        }
-
-        let escaped = format!("${tag}${self}${tag}$");
-
-        (escaped, outer_tag)
+        let result = format!("\"{}\"", self.replace('"', "\"\""));
+        result
    }
 }

@@ -251,13 +226,10 @@ pub async fn get_existing_dbs_async(
    // invalid state. See:
    //   https://github.com/postgres/postgres/commit/a4b4cc1d60f7e8ccfcc8ff8cb80c28ee411ad9a9
    let rowstream = client
-        // We use a subquery instead of a fancy `datdba::regrole::text AS owner`,
-        // because the latter automatically wraps the result in double quotes,
-        // if the role name contains special characters.
        .query_raw::<str, &String, &[String; 0]>(
            "SELECT
                datname AS name,
-                (SELECT rolname FROM pg_roles WHERE oid = datdba) AS owner,
+                datdba::regrole::text AS owner,
                NOT datallowconn AS restrict_conn,
                datconnlimit = - 2 AS invalid
            FROM
--- a/compute_tools/src/rsyslog.rs
+++ b/compute_tools/src/rsyslog.rs
@@ -1,77 +0,0 @@
-use std::process::Command;
-use std::{fs::OpenOptions, io::Write};
-
-use anyhow::{Context, Result};
-use tracing::info;
-
-fn get_rsyslog_pid() -> Option<String> {
-    let output = Command::new("pgrep")
-        .arg("rsyslogd")
-        .output()
-        .expect("Failed to execute pgrep");
-
-    if !output.stdout.is_empty() {
-        let pid = std::str::from_utf8(&output.stdout)
-            .expect("Invalid UTF-8 in process output")
-            .trim()
-            .to_string();
-        Some(pid)
-    } else {
-        None
-    }
-}
-
-// Restart rsyslogd to apply the new configuration.
-// This is necessary, because there is no other way to reload the rsyslog configuration.
-//
-// Rsyslogd shouldn't lose any messages, because of the restart,
-// because it tracks the last read position in the log files
-// and will continue reading from that position.
-// TODO: test it properly
-//
-fn restart_rsyslog() -> Result<()> {
-    let old_pid = get_rsyslog_pid().context("rsyslogd is not running")?;
-    info!("rsyslogd is running with pid: {}, restart it", old_pid);
-
-    // kill it to restart
-    let _ = Command::new("pkill")
-        .arg("rsyslogd")
-        .output()
-        .context("Failed to stop rsyslogd")?;
-
-    Ok(())
-}
-
-pub fn configure_audit_rsyslog(
-    log_directory: &str,
-    tag: &str,
-    remote_endpoint: &str,
-) -> Result<()> {
-    let config_content: String = format!(
-        include_str!("config_template/compute_audit_rsyslog_template.conf"),
-        log_directory = log_directory,
-        tag = tag,
-        remote_endpoint = remote_endpoint
-    );
-
-    info!("rsyslog config_content: {}", config_content);
-
-    let rsyslog_conf_path = "/etc/rsyslog.d/compute_audit_rsyslog.conf";
-    let mut file = OpenOptions::new()
-        .create(true)
-        .write(true)
-        .truncate(true)
-        .open(rsyslog_conf_path)?;
-
-    file.write_all(config_content.as_bytes())?;
-
-    info!(
-        "rsyslog configuration file {} added successfully. Starting rsyslogd",
-        rsyslog_conf_path
-    );
-
-    // start the service, using the configuration
-    restart_rsyslog()?;
-
-    Ok(())
-}
--- a/compute_tools/src/spec_apply.rs
+++ b/compute_tools/src/spec_apply.rs
@@ -6,22 +6,21 @@ use std::sync::Arc;

 use anyhow::{Context, Result};
 use compute_api::responses::ComputeStatus;
-use compute_api::spec::{ComputeAudit, ComputeFeature, ComputeSpec, Database, PgIdent, Role};
+use compute_api::spec::{ComputeFeature, ComputeSpec, Database, PgIdent, Role};
 use futures::future::join_all;
 use tokio::sync::RwLock;
 use tokio_postgres::Client;
 use tokio_postgres::error::SqlState;
 use tracing::{Instrument, debug, error, info, info_span, instrument, warn};

-use crate::compute::{ComputeNode, ComputeState};
+use crate::compute::{ComputeNode, ComputeState, construct_superuser_query};
 use crate::pg_helpers::{
-    DatabaseExt, Escaping, GenericOptionsSearch, RoleExt, get_existing_dbs_async,
+    DatabaseExt, Escaping, GenericOptionsSearch, RoleExt, escape_literal, get_existing_dbs_async,
    get_existing_roles_async,
 };
 use crate::spec_apply::ApplySpecPhase::{
-    CreateAndAlterDatabases, CreateAndAlterRoles, CreateAvailabilityCheck, CreateNeonSuperuser,
-    CreatePgauditExtension, CreatePgauditlogtofileExtension, CreateSchemaNeon,
-    DisablePostgresDBPgAudit, DropInvalidDatabases, DropRoles, FinalizeDropLogicalSubscriptions,
+    CreateAndAlterDatabases, CreateAndAlterRoles, CreateAvailabilityCheck, CreateSchemaNeon,
+    CreateSuperUser, DropInvalidDatabases, DropRoles, FinalizeDropLogicalSubscriptions,
    HandleNeonExtension, HandleOtherExtensions, RenameAndDeleteDatabases, RenameRoles,
    RunInEachDatabase,
 };
@@ -188,7 +187,7 @@ impl ComputeNode {
            }

            for phase in [
-                CreateNeonSuperuser,
+                CreateSuperUser,
                DropInvalidDatabases,
                RenameRoles,
                CreateAndAlterRoles,
@@ -278,19 +277,6 @@ impl ComputeNode {
                phases.push(FinalizeDropLogicalSubscriptions);
            }

-            // Keep DisablePostgresDBPgAudit phase at the end,
-            // so that all config operations are audit logged.
-            match spec.audit_log_level
-            {
-                ComputeAudit::Hipaa => {
-                    phases.push(CreatePgauditExtension);
-                    phases.push(CreatePgauditlogtofileExtension);
-                    phases.push(DisablePostgresDBPgAudit);
-                }
-                ComputeAudit::Log => { /* not implemented yet */ }
-                ComputeAudit::Disabled => {}
-            }
-
            for phase in phases {
                debug!("Applying phase {:?}", &phase);
                apply_operations(
@@ -469,7 +455,7 @@ pub enum PerDatabasePhase {

 #[derive(Clone, Debug)]
 pub enum ApplySpecPhase {
-    CreateNeonSuperuser,
+    CreateSuperUser,
    DropInvalidDatabases,
    RenameRoles,
    CreateAndAlterRoles,
@@ -477,9 +463,6 @@ pub enum ApplySpecPhase {
    CreateAndAlterDatabases,
    CreateSchemaNeon,
    RunInEachDatabase { db: DB, subphase: PerDatabasePhase },
-    CreatePgauditExtension,
-    CreatePgauditlogtofileExtension,
-    DisablePostgresDBPgAudit,
    HandleOtherExtensions,
    HandleNeonExtension,
    CreateAvailabilityCheck,
@@ -596,10 +579,14 @@ async fn get_operations<'a>(
    apply_spec_phase: &'a ApplySpecPhase,
 ) -> Result<Box<dyn Iterator<Item = Operation> + 'a + Send>> {
    match apply_spec_phase {
-        ApplySpecPhase::CreateNeonSuperuser => Ok(Box::new(once(Operation {
-            query: include_str!("sql/create_neon_superuser.sql").to_string(),
-            comment: None,
-        }))),
+        ApplySpecPhase::CreateSuperUser => {
+            let query = construct_superuser_query(spec);
+
+            Ok(Box::new(once(Operation {
+                query,
+                comment: None,
+            })))
+        }
        ApplySpecPhase::DropInvalidDatabases => {
            let mut ctx = ctx.write().await;
            let databases = &mut ctx.dbs;
@@ -733,15 +720,14 @@ async fn get_operations<'a>(
                        // We do not check whether the DB exists or not,
                        // Postgres will take care of it for us
                        "delete_db" => {
-                            let (db_name, outer_tag) = op.name.pg_quote_dollar();
                            // In Postgres we can't drop a database if it is a template.
                            // So we need to unset the template flag first, but it could
                            // be a retry, so we could've already dropped the database.
                            // Check that database exists first to make it idempotent.
                            let unset_template_query: String = format!(
                                include_str!("sql/unset_template_for_drop_dbs.sql"),
-                                datname = db_name,
-                                outer_tag = outer_tag,
+                                datname_str = escape_literal(&op.name),
+                                datname = &op.name.pg_quote()
                            );

                            // Use FORCE to drop database even if there are active connections.
@@ -848,8 +834,6 @@ async fn get_operations<'a>(
                                comment: None,
                            },
                            Operation {
-                                // ALL PRIVILEGES grants CREATE, CONNECT, and TEMPORARY on the database
-                                // (see https://www.postgresql.org/docs/current/ddl-priv.html)
                                query: format!(
                                    "GRANT ALL PRIVILEGES ON DATABASE {} TO neon_superuser",
                                    db.name.pg_quote()
@@ -909,11 +893,9 @@ async fn get_operations<'a>(
                PerDatabasePhase::DropLogicalSubscriptions => {
                    match &db {
                        DB::UserDB(db) => {
-                            let (db_name, outer_tag) = db.name.pg_quote_dollar();
                            let drop_subscription_query: String = format!(
                                include_str!("sql/drop_subscriptions.sql"),
-                                datname_str = db_name,
-                                outer_tag = outer_tag,
+                                datname_str = escape_literal(&db.name),
                            );

                            let operations = vec![Operation {
@@ -952,7 +934,6 @@ async fn get_operations<'a>(
                                    DB::SystemDB => PgIdent::from("cloud_admin").pg_quote(),
                                    DB::UserDB(db) => db.owner.pg_quote(),
                                };
-                                let (escaped_role, outer_tag) = op.name.pg_quote_dollar();

                                Some(vec![
                                    // This will reassign all dependent objects to the db owner
@@ -967,9 +948,7 @@ async fn get_operations<'a>(
                                    Operation {
                                        query: format!(
                                            include_str!("sql/pre_drop_role_revoke_privileges.sql"),
-                                            // N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`
-                                            role_name = escaped_role,
-                                            outer_tag = outer_tag,
+                                            role_name = quoted,
                                        ),
                                        comment: None,
                                    },
@@ -994,14 +973,12 @@ async fn get_operations<'a>(
                        DB::SystemDB => return Ok(Box::new(empty())),
                        DB::UserDB(db) => db,
                    };
-                    let (db_owner, outer_tag) = db.owner.pg_quote_dollar();

                    let operations = vec![
                        Operation {
                            query: format!(
                                include_str!("sql/set_public_schema_owner.sql"),
-                                db_owner = db_owner,
-                                outer_tag = outer_tag,
+                                db_owner = db.owner.pg_quote()
                            ),
                            comment: None,
                        },
@@ -1121,25 +1098,6 @@ async fn get_operations<'a>(
            }
            Ok(Box::new(empty()))
        }
-        ApplySpecPhase::CreatePgauditExtension => Ok(Box::new(once(Operation {
-            query: String::from("CREATE EXTENSION IF NOT EXISTS pgaudit"),
-            comment: Some(String::from("create pgaudit extensions")),
-        }))),
-        ApplySpecPhase::CreatePgauditlogtofileExtension => Ok(Box::new(once(Operation {
-            query: String::from("CREATE EXTENSION IF NOT EXISTS pgauditlogtofile"),
-            comment: Some(String::from("create pgauditlogtofile extensions")),
-        }))),
-        // Disable pgaudit logging for postgres database.
-        // Postgres is neon system database used by monitors
-        // and compute_ctl tuning functions and thus generates a lot of noise.
-        // We do not consider data stored in this database as sensitive.
-        ApplySpecPhase::DisablePostgresDBPgAudit => {
-            let query = "ALTER DATABASE postgres SET pgaudit.log to 'none'";
-            Ok(Box::new(once(Operation {
-                query: query.to_string(),
-                comment: Some(query.to_string()),
-            })))
-        }
        ApplySpecPhase::HandleNeonExtension => {
            let operations = vec![
                Operation {
--- a/compute_tools/src/sql/create_neon_superuser.sql
+++ b/compute_tools/src/sql/create_neon_superuser.sql
@@ -1,8 +0,0 @@
-DO $$
-    BEGIN
-        IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = 'neon_superuser')
-        THEN
-            CREATE ROLE neon_superuser CREATEDB CREATEROLE NOLOGIN REPLICATION BYPASSRLS IN ROLE pg_read_all_data, pg_write_all_data;
-        END IF;
-    END
-$$;
--- a/compute_tools/src/sql/drop_subscriptions.sql
+++ b/compute_tools/src/sql/drop_subscriptions.sql
@@ -1,4 +1,4 @@
-DO ${outer_tag}$
+DO $$
 DECLARE
    subname TEXT;
 BEGIN
@@ -9,4 +9,4 @@ BEGIN
        EXECUTE format('DROP SUBSCRIPTION %I;', subname);
    END LOOP;
 END;
-${outer_tag}$;
+$$;
--- a/compute_tools/src/sql/pre_drop_role_revoke_privileges.sql
+++ b/compute_tools/src/sql/pre_drop_role_revoke_privileges.sql
@@ -1,6 +1,6 @@
 SET SESSION ROLE neon_superuser;

-DO ${outer_tag}$
+DO $$
 DECLARE
    schema TEXT;
    revoke_query TEXT;
@@ -16,15 +16,13 @@ BEGIN
        WHERE schema_name IN ('public')
    LOOP
        revoke_query := format(
-            'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM %I GRANTED BY neon_superuser;',
-            schema,
-            -- N.B. this has to be properly dollar-escaped with `pg_quote_dollar()`
-            {role_name}
+            'REVOKE ALL PRIVILEGES ON ALL TABLES IN SCHEMA %I FROM {role_name} GRANTED BY neon_superuser;',
+            schema
        );

        EXECUTE revoke_query;
    END LOOP;
 END;
-${outer_tag}$;
+$$;

 RESET ROLE;
--- a/compute_tools/src/sql/set_public_schema_owner.sql
+++ b/compute_tools/src/sql/set_public_schema_owner.sql
@@ -1,4 +1,5 @@
-DO ${outer_tag}$
+DO
+$$
    DECLARE
        schema_owner TEXT;
    BEGIN
@@ -15,8 +16,8 @@ DO ${outer_tag}$

            IF schema_owner = 'cloud_admin' OR schema_owner = 'zenith_admin'
            THEN
-                EXECUTE format('ALTER SCHEMA public OWNER TO %I', {db_owner});
+                ALTER SCHEMA public OWNER TO {db_owner};
            END IF;
        END IF;
    END
-${outer_tag}$;
+$$;
--- a/compute_tools/src/sql/unset_template_for_drop_dbs.sql
+++ b/compute_tools/src/sql/unset_template_for_drop_dbs.sql
@@ -1,12 +1,12 @@
-DO ${outer_tag}$
+DO $$
    BEGIN
        IF EXISTS(
            SELECT 1
            FROM pg_catalog.pg_database
-            WHERE datname = {datname}
+            WHERE datname = {datname_str}
        )
        THEN
-            EXECUTE format('ALTER DATABASE %I is_template false', {datname});
+            ALTER DATABASE {datname} is_template false;
        END IF;
    END
-${outer_tag}$;
+$$;
--- a/compute_tools/tests/pg_helpers_tests.rs
+++ b/compute_tools/tests/pg_helpers_tests.rs
@@ -61,23 +61,6 @@ test.escaping = 'here''s a backslash \\ and a quote '' and a double-quote " hoor
        assert_eq!(ident.pg_quote(), "\"\"\"name\"\";\\n select 1;\"");
    }

-    #[test]
-    fn ident_pg_quote_dollar() {
-        let test_cases = vec![
-            ("name", ("$$name$$", "x")),
-            ("name$$", ("$x$name$$$x$", "xx")),
-            ("name$$$", ("$x$name$$$$x$", "xx")),
-            ("name$$$$", ("$x$name$$$$$x$", "xx")),
-            ("name$x$", ("$xx$name$x$$xx$", "xxx")),
-        ];
-
-        for (input, expected) in test_cases {
-            let (escaped, tag) = PgIdent::from(input).pg_quote_dollar();
-            assert_eq!(escaped, expected.0);
-            assert_eq!(tag, expected.1);
-        }
-    }
-
    #[test]
    fn generic_options_search() {
        let generic_options: GenericOptions = Some(vec![
--- a/control_plane/src/endpoint.rs
+++ b/control_plane/src/endpoint.rs
@@ -48,8 +48,7 @@ use anyhow::{Context, Result, anyhow, bail};
 use compute_api::requests::ConfigurationRequest;
 use compute_api::responses::{ComputeCtlConfig, ComputeStatus, ComputeStatusResponse};
 use compute_api::spec::{
-    Cluster, ComputeAudit, ComputeFeature, ComputeMode, ComputeSpec, Database, PgIdent,
-    RemoteExtSpec, Role,
+    Cluster, ComputeFeature, ComputeMode, ComputeSpec, Database, PgIdent, RemoteExtSpec, Role,
 };
 use nix::sys::signal::{Signal, kill};
 use pageserver_api::shard::ShardStripeSize;
@@ -669,7 +668,6 @@ impl Endpoint {
            local_proxy_config: None,
            reconfigure_concurrency: self.reconfigure_concurrency,
            drop_subscriptions_before_start: self.drop_subscriptions_before_start,
-            audit_log_level: ComputeAudit::Disabled,
        };

        // this strange code is needed to support respec() in tests
--- a/control_plane/src/local_env.rs
+++ b/control_plane/src/local_env.rs
@@ -165,6 +165,8 @@ pub struct NeonStorageControllerConf {

    #[serde(with = "humantime_serde")]
    pub long_reconcile_threshold: Option<Duration>,
+
+    pub load_safekeepers: bool,
 }

 impl NeonStorageControllerConf {
@@ -188,6 +190,7 @@ impl Default for NeonStorageControllerConf {
            max_secondary_lag_bytes: None,
            heartbeat_interval: Self::DEFAULT_HEARTBEAT_INTERVAL,
            long_reconcile_threshold: None,
+            load_safekeepers: true,
        }
    }
 }
--- a/control_plane/src/storage_controller.rs
+++ b/control_plane/src/storage_controller.rs
@@ -537,6 +537,10 @@ impl StorageController {
            args.push("--start-as-candidate".to_string());
        }

+        if self.config.load_safekeepers {
+            args.push("--load-safekeepers".to_string());
+        }
+
        if let Some(private_key) = &self.private_key {
            let claims = Claims::new(None, Scope::PageServerApi);
            let jwt_token =
--- a/docs/rfcs/041-rel-sparse-keyspace.md
+++ b/docs/rfcs/041-rel-sparse-keyspace.md
@@ -1,201 +0,0 @@
-# Sparse Keyspace for Relation Directories
-
-## Summary
-
-This is an RFC describing a new storage strategy for storing relation directories.
-
-## Motivation
-
-Postgres maintains a directory structure for databases and relations. In Neon, we store these information
-by serializing the directory data in a single key (see `pgdatadir_mapping.rs`).
-
-```rust
-// DbDir:
-// 00 00000000 00000000 00000000 00   00000000
-
-// RelDir:
-// 00 SPCNODE  DBNODE   00000000 00   00000001 (Postgres never uses relfilenode 0)
-```
-
-We have a dedicated structure on the ingestion path to serialize the relation directory into this single key.
-
-```rust
-#[derive(Debug, Serialize, Deserialize, Default)]
-pub(crate) struct RelDirectory {
-    // Set of relations that exist. (relfilenode, forknum)
-    //
-    // TODO: Store it as a btree or radix tree or something else that spans multiple
-    // key-value pairs, if you have a lot of relations
-    pub(crate) rels: HashSet<(Oid, u8)>,
-}
-```
-
-The current codebase has the following three access patterns for the relation directory.
-
-1. Check if a relation exists.
-2. List all relations.
-3. Create/drop a relation.
-
-For (1), we currently have to get the reldir key, deserialize it, and check whether the relation exists in the
-hash set. For (2), we get the reldir key and the hash set. For (3), we need first to get
-and deserialize the key, add the new relation record to the hash set, and then serialize it and write it back.
-
-If we have 100k relations in a database, we would have a 100k-large hash set. Then, every
-relation created and dropped would have deserialized and serialized this 100k-large hash set. This makes the
-relation create/drop process to be quadratic. When we check if a relation exists in the ingestion path,
-we would have to deserialize this super big 100k-large key before checking if a single relation exists.
-
-In this RFC, we will propose a new way to store the reldir data in the sparse keyspace and propose how
-to seamlessly migrate users to use the new keyspace.
-
-The PoC patch is implemented in [PR10316](https://github.com/neondatabase/neon/pull/10316).
-
-## Key Mapping
-
-We will use the recently introduced sparse keyspace to store actual data. Sparse keyspace was proposed in
-[038-aux-file-v2.md](038-aux-file-v2.md). The original reldir has one single value of `HashSet<(Oid, u8)>`
-for each of the databases (identified as `spcnode, dbnode`). We encode the `Oid` (`relnode, forknum`),
-into the key.
-
-```plain
-(REL_DIR_KEY_PREFIX, spcnode, dbnode, relnode, forknum, 1) -> deleted
-(REL_DIR_KEY_PREFIX, spcnode, dbnode, relnode, forknum, 1) -> exists
-```
-
-Assume all reldir data are stored in this new keyspace; the 3 reldir operations we mentioned before can be
-implemented as follows.
-
-1. Check if a relation exists: check if the key maps to "exists".
-2. List all relations: scan the sprase keyspace over the `rel_dir_key_prefix`. Extract relnode and forknum from the key.
-3. Create/drop a relation: write "exists" or "deleted" to the corresponding key of the relation. The delete tombstone will
-   be removed during image layer generation upon compaction.
-
-Note that "exists" and "deleted" will be encoded as a single byte as two variants of an enum.
-The mapping is implemented as `rel_tag_sparse_key` in the PoC patch.
-
-## Changes to Sparse Keyspace
-
-Previously, we only used sparse keyspaces for the aux files, which did not carry over when branching. The reldir
-information needs to be preserved from the parent branch to the child branch. Therefore, the read path needs
-to be updated accordingly to accommodate such "inherited sparse keys". This is done in
-[PR#10313](https://github.com/neondatabase/neon/pull/10313).
-
-## Coexistence of the Old and New Keyspaces
-
-Migrating to the new keyspace will be done gradually: when we flip a config item to enable the new reldir keyspace, the
-ingestion path will start to write to the new keyspace and the old reldir data will be kept in the old one. The read
-path needs to combine the data from both keyspaces.
-
-Theoretically, we could do a rewrite at the startup time that scans all relation directories and copies that data into the
-new keyspace. However, this could take a long time, especially if we have thousands of tenants doing the migration
-process simultaneously after the pageserver restarts. Therefore, we propose the coexistence strategy so that the
-migration can happen seamlessly and imposes no potential downtime for the user.
-
-With the coexistence assumption, the 3 reldir operations will be implemented as follows:
-
-1. Check if a relation exists
-   - Check the new keyspace if the key maps to any value. If it maps to "exists" or "deleted", directly
-    return it to the user.
-   - Otherwise, deserialize the old reldir key and get the result.
-2. List all relations: scan the sparse keyspace over the `rel_dir_key_prefix` and deserialize the old reldir key.
-   Combine them to obtain the final result.
-3. Create/drop a relation: write "exists" or "deleted" to the corresponding key of the relation into the new keyspace.
-   - We assume no overwrite of relations will happen (i.e., the user won't create a relation at the same Oid). This will be implemented as a runtime check.
-   - For relation creation, we add `sparse_reldir_tableX -> exists` to the keyspace.
-   - For relation drop, we first check if the relation is recorded in the old keyspace. If yes, we deserialize the old reldir key,
-    remove the relation, and then write it back. Otherwise, we put `sparse_reldir_tableX -> deleted` to the keyspace.
-   - The delete tombstone will be removed during image layer generation upon compaction.
-
-This process ensures that the transition will not introduce any downtime and all new updates are written to the new keyspace. The total
-amount of data in the storage would be `O(relations_modifications)` and we can guarantee `O(current_relations)` after compaction.
-There could be some relations that exist in the old reldir key for a long time. Refer to the "Full Migration" section on how to deal
-with them. Plus, for relation modifications, it will have `O(old_relations)` complexity until we do the full migration, which gives
-us `O(1)` complexity after fully opt-in the sparse keyspace.
-
-The process also implies that a relation will only exists either in the old reldir key or in the new sparse keyspace. It is not possible
-to have a table to be recorded in the old reldir key while later having a delete tombstone for it in the sparse keyspace at any LSN.
-
-We will introduce a config item and an index_part record to record the current status of the migration process.
-
- Config item `enable_reldir_v2`: controls whether the ingestion path writes the reldir info into the new keyspace.
- `index_part.json` field `reldir_v2_status`: whether the timeline has written any key into the new reldir keyspace.
-
-If `enable_reldir_v2` is set to `true` and the timeline ingests the first key into the new reldir keyspace, it will update
-`index_part.json` to set `reldir_v2_status` to `Status::Migrating`. Even if `enable_reldir_v2` gets flipped back to
-`false` (i.e., when the pageserver restarts and such config isn't persisted), the read/write path will still
-read/write to the new keyspace to avoid data inconsistency. This also indicates that the migration is one-way only:
-once v2 is enabled, the user cannot go back to v1.
-
-## Next Steps
-
-### Full Migration
-
-This won't be implemented in the project's first phase but might be implemented in the future. Having both v1 and
-v2 existing in the system would force us to keep the code to deserialize the old reldir key forever. To entirely deprecate this
-code path, we must ensure the timeline has no old reldir data.
-
-We can trigger a special image layer generation process at the gc-horizon. The generated image layers will cover several keyspaces:
-the old reldir key in each of the databases, and the new reldir sparse keyspace. It will remove the old reldir key while
-copying them into the corresponding keys in the sparse keyspace in the resulting image. This special process happens in
-the background during compaction. For example, assume this special process is triggered at LSN 0/180. The `create_image_layers`
-process discovers the following keys at this LSN.
-
-```plain
-db1/reldir_key -> (table 1, table 2, table 3)
-...db1 rel keys
-db2/reldir_key -> (table 4, table 5, table 6)
-...db2 rel keys
-sparse_reldir_db2_table7 -> exists
-sparse_reldir_db1_table8 -> deleted
-```
-
-It will generate the following keys:
-
-```plain
-db1/reldir_key -> () # we have to keep the key because it is part of `collect_keyspace`.
-...db1 rel keys
-db2/reldir_key -> ()
-...db2 rel keys
-
-- start image layer for the sparse keyspace at sparse_reldir_prefix at LSN 0/180
-sparse_reldir_db1_table1 -> exists
-sparse_reldir_db1_table2 -> exists
-sparse_reldir_db1_table3 -> exists
-sparse_reldir_db2_table4 -> exists
-sparse_reldir_db2_table5 -> exists
-sparse_reldir_db2_table6 -> exists
-sparse_reldir_db2_table7 -> exists
-- end image layer for the sparse keyspace at sparse_reldir_prefix+1
-
-# The `sparse_reldir_db1_table8` key gets dropped as part of the image layer generation code for the sparse keyspace.
-# Note that the read path will stop reading if a key is not found in the image layer covering the key range so there
-# are no correctness issue.
-```
-
-We must verify that no pending modifications to the old reldir exists in the delta/image layers above the gc-horizon before
-we start this process (We can do a vectored read to get the full key history of the old reldir key and ensure there are no more images
-above the gc-horizon). Otherwise, it will violate the property that "a relation will only exists either in the old reldir key or
-in the new sparse keyspace". After we run this migration process, we can mark `reldir_v2_status` in the `index_part.json` to
-`Status::Migrated`, and the read path won't need to read from the old reldir anymore. Once the status is set to `Migrated`, we
-don't need to add the key into `collect_keyspace` and therefore all of them will be removed from all future image layers.
-
-The migration process can be proactively triggered across all attached/detached tenants to help us fully remove the old reldir code.
-
-### Consolidate Relation Size Keys
-
-We have relsize at the end of all relation nodes.
-
-```plain
-// RelSize:
-// 00 SPCNODE  DBNODE   RELNODE  FORK FFFFFFFF
-```
-
-This means that computing logical size requires us to do several single-key gets across the keyspace,
-potentially requiring downloading many layer files. We could consolidate them into a single
-keyspace, improving logical size calculation performance.
-
-### Migrate DBDir Keys
-
-We assume the number of databases created by the users will be small, and therefore, the current way
-of storing the database directory would be acceptable. In the future, we could also migrate DBDir keys into
-the sparse keyspace to support large amount of databases.
--- a/docs/rfcs/2025-03-15-hot-secondaries.md
+++ b/docs/rfcs/2025-03-15-hot-secondaries.md
@@ -0,0 +1,164 @@
+
+# Pageserver Hot Secondaries
+
+## Summary
+
+It is proposed to add a new mode for pageserver tenant shard locations,
+called "hot secondary", which is able to serve page_service requests but
+does not do all the same housekeeping as an attached location, and does
+not store any additional data in S3.
+
+There is a stark tradeoff between resource cost and complexity: a very simple solution would be to have multiple full attached locations doing independent I/O, but this RFC proposes some additional complexity to
+reduce cost.
+
+## Background
+
+In the [pageserver migration RFC](028-pageserver-migration.md), we introduced the concept of "warm secondaries".  These are pageserver locations that poll remote storage for a _heatmap_ describing which layers they should hold, and then download those layers from S3.  This enables them to rapidly transition into a usable attached location with a warm cache.
+
+Combined with the storage controller's detect of pageserver failures, warm
+secondaries enabled high availability of pageservers with a recovery time
+objective (RTO) measured in seconds (depends on configured heartbeat frequency) -- occasional cloud instance failures are typically recovered
+in well under a minute, without human intervention.
+
+## Purpose
+
+We aim to provide a sub-second RTO for pageserver failures, for mission
+critical workloads.  To do this, we should enable the postgres client
+to make its own decision about cutting over to a secondary, rather than
+waiting for the controller to detect a failure and instruct it to
+use a different pageserver.  These secondaries should be maintained
+in a continuously readable state rather than requiring explicit activation.
+
+Because low-RTO failover is intrinsically vulnerable to "flapping"/false
+positives, reads from such a hot secondary will not "promote" the secondary: we don't want to flap back and forth at millisecond timescales.  Rather, reads will be served by hot secondaries at any time, 
+but their transition to an attached (primary) location will still be
+managed by the storage controller.
+
+## Design of Hot Location Mode
+
+At a high level, hot locations are basically the same `Tenant` and `Timeline` types as an attached location, but with some behavioral tweaks.  This RFC won't get into code structure details: these changes
+may be expressed as different types (more robust) or as different modes
+for existing types (less code churn, more complexity).
+
+### Load and ingest
+
+Initially, we may start in the same way as a normal attached location:
+by discovering the latest metadata in remote storage and constructing
+a LayerMap.
+
+We should also do ingest as normal: subscribing to safekeeper and streaming
+writes into ephemeral layer files that are then frozen into L0s.  However,
+we do not want to wastefully upload these to S3 (they duplicate what the
+attached location is already writing).
+
+### "Virtual" compaction
+
+Clearly ingesting but never uploading or compacting will generate an unbounded stack of L0 layers, unless we do something about it.
+
+To solve this, we may add a special type of compaction that re-reads
+from remote storage, updates the layer map to contain all L1
+and image layers from the remote metadata, and triggers download of these.
+
+We do not download remote L0s during virtual compaction, because the hot secondary has also been ingesting and generating these, so it would be wasteful.  We just trim any local L0s which are now covered by the L1 high watermark of the remote metadata, and retain any that are still needed to serve reads.
+
+Note that this process is expected to generate some overlaps in LSN space: we might have an L0 that we generated locally which overlaps with an L1 from remote storage.  getpage@lsn logic must handle this, and avoid assuming non-overlapping layers (i.e. having read some deltas from L0, we must not read the same deltas again in an L1, we must remember what LSN we already passed).
+
+The average total network download bandwidth of the hot secondary is equal to the rate at which the attached location generates L1 and image layers, plus the rate at which WAL is generated.
+
+The average total disk write bandwidth is the sum of WAL generation rate plus L1/image generation rate: this is about the same as a normal attached location.  The average disk _read_ bandwidth of a hot secondary is far lower than an attached location because it is not reading back layers to compact them -- layers are only read in periods where the attached location was unavailable, so computes started reading from a hot secondary.
+
+The trigger for virtual compaction can be similar to the existing trigger
+for L1 compaction on attached locations: once we build up a deep stack of L0s, then we do virtual compaction to trim it.  This assumes that the attached location has kept up with compaction.  The hot secondary can be
+more tolerant of a deeper L0 stack because it is less often serving
+reads: for example it might make sense to trigger normal L1 compaction at 10 L0 layers, and trigger shallow compaction at 15 L0 layers, giving a good chance that by the time the hot secondary does compaction, the attached location has already written out some layer files for it to read.
+
+To avoid an availability gap while downloading data from S3, it is important that the hot
+secondary downloads new layer files before updating its layer map to de-reference replaced
+layers.
+
+### Handling missing layers/timelines
+
+If an incoming request references a timeline that the hot secondary is
+unaware of, it must go read from S3 to determine if the timeline exists, and if so then load it.
+
+The hot secondary should also be tipped off by the storage controller when
+timelines are created, so that in normal operation it is aware of timelines
+immediately rather than having to load on demand (loading on demand could
+have much higher latency for reads).
+
+Hot secondaries may also experience 404s reading layers from remote storage, because the layer might have been deleted by the attached location
+during compaction or GC.  If the hot secondary finds such a 404, it should
+trigger a re-download of the timeline index.
+
+### Transition from Hot Secondary to Attached
+
+While a hot secondary can serve writes independently for a short period of time (until
+too many L0s build up to efficiently serve reads), it needs to be promoted to be the attached
+location if the last attached location becomes unavailable (or if the storage controller
+determines that the tenant should be migrated).
+
+This can be done trivially by shutting down and starting up again in attached mode (on startup
+the layer map will be reset to the content of remote storage), but this can impose an availability gap, because:
+- After unexpected failure of an attached location, the hot secondary's local L0s may be
+  further ahead in WAL ingest than the contents of remote storage, so resetting to what's
+  in remote storage will make recent data unavailable until it is re-ingested.
+- Even if the remote data is up to date with latest WAL, it may take some time to download
+  layers. 
+
+To avoid an availability gap while re-ingesting WAL, it is necessary to stitch the local L0s with remote storage state.  We may do this at startup, by making an exception to our
+usual policy of only respecting remote storage state at startup.  This exception can
+be specific to L0 files, and perhaps also specific to when we can detect that these
+were written by a hot secondary (perhaps by marking these files with a suffix or magic 0xffff generation?)
+
+We should also only do this cutover once we're reasonably sure the old attached location
+isn't still uploading, so that on startup we do not see a whole new layer map with lots
+of layers that need downloading.
+
+We may still tolerate some availability gap in the <1s range while reloading the tenant
+in a different mode.  We should aim for this to be under 100ms under usual circumstances,
+as it should only require long enough to:
+- Flush ephemeral layer to L0 on shutdown (writing 128MB takes of the order 100ms)
+- Load remote index on startup (reading from S3 takes of the order 10ms)
+
+Doing many such cutovers concurrently may result in worse availability, so the controller
+should be tuned to understand that when cutting over multiple hot secondaries to attached,
+it is best not to rush it (as they are already in a readable state, it is less urgent
+than when activating warm secondaries).
+
+## Summary of a failover
+
+To summarize the order of operations when a pageserver instance fails while holding a tenant
+that has a hot secondary location:
+- after some short timeout (100s of ms), compute gives up on getpage requests to the primary and sends
+  them to the hot secondary.
+- after some much longer timeout (e.g. ~30s), controller decides that the hot secondary should
+  become attached, so that it can do its own compaction.
+- Hot secondary is instructed to do a compaction before shutting it down, so that during
+  its restart into attached mode it will not have to deal with any remote storage change.
+- Hot secondary shuts down, flushing ephemeral layer to L0.
+- Previously-secondary location starts up in attached mode with a new generation.  Downloads
+  index from remote storage, and identifies which L0 files to retain.  Adds these to LayerMap
+  and enqueues them for upload.
+- Now fully available for reads and able to proceed with compaction etc as normal.
+
+## Optimisations/details
+
+- We should add a read-only mode to RemoteTimelineClient
+
+## Alternatives considered
+
+### Full mirror
+
+We could make hot secondary locations do all compaction, gc, etc operations
+independently, and maintain their own set of layer files in S3.  These would essentially be separate tenants in pageserver terms, but consuming the same safekeeper timelines.
+
+These locations would on longer be anything special in pageserver terms, they'd simply be attached locations that use some modified path like `<tenant_id>.secondary` to avoid colliding with the primary data.
+
+The storage controller could have some `AttachedHotSecondary` placement
+policy that configures the hot secondary location with some flag to indicate that the alternative storage path should be used.
+
+Clearly the advantage of this approach is code simplicity.  However, the
+downsides are substantial:
+- Double object storage costs
+- Compaction costs are doubled (CPU & disk read I/O), whereas the proposed
+  implementation of hot secondaries only pays twice for the compaction _write_ IO as it writes compacted layers to local disk.
--- a/libs/compute_api/src/responses.rs
+++ b/libs/compute_api/src/responses.rs
@@ -134,10 +134,8 @@ pub struct CatalogObjects {
    pub databases: Vec<Database>,
 }

-#[derive(Clone, Debug, Deserialize, Serialize)]
+#[derive(Debug, Deserialize, Serialize)]
 pub struct ComputeCtlConfig {
-    /// Set of JSON web keys that the compute can use to authenticate
-    /// communication from the control plane.
    pub jwks: JwkSet,
 }

--- a/libs/compute_api/src/spec.rs
+++ b/libs/compute_api/src/spec.rs
@@ -155,16 +155,6 @@ pub struct ComputeSpec {
    /// over the same replication content from publisher.
    #[serde(default)] // Default false
    pub drop_subscriptions_before_start: bool,
-
-    /// Log level for audit logging:
-    ///
-    /// Disabled - no audit logging. This is the default.
-    /// log - log masked statements to the postgres log using pgaudit extension
-    /// hipaa - log unmasked statements to the file using pgaudit and pgauditlogtofile extension
-    ///
-    /// Extensions should be present in shared_preload_libraries
-    #[serde(default)]
-    pub audit_log_level: ComputeAudit,
 }

 /// Feature flag to signal `compute_ctl` to enable certain experimental functionality.
@@ -272,17 +262,6 @@ pub enum ComputeMode {
    Replica,
 }

-/// Log level for audit logging
-/// Disabled, log, hipaa
-/// Default is Disabled
-#[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)]
-pub enum ComputeAudit {
-    #[default]
-    Disabled,
-    Log,
-    Hipaa,
-}
-
 #[derive(Clone, Debug, Default, Deserialize, Serialize, PartialEq, Eq)]
 pub struct Cluster {
    pub cluster_id: Option<String>,
--- a/libs/http-utils/Cargo.toml
+++ b/libs/http-utils/Cargo.toml
@@ -6,8 +6,11 @@ license.workspace = true

 [dependencies]
 anyhow.workspace = true
+backtrace.workspace = true
 bytes.workspace = true
+inferno.workspace = true
 fail.workspace = true
+flate2.workspace = true
 hyper0.workspace = true
 itertools.workspace = true
 jemalloc_pprof.workspace = true
--- a/libs/http-utils/src/endpoint.rs
+++ b/libs/http-utils/src/endpoint.rs
@@ -3,6 +3,8 @@ use std::io::Write as _;
 use std::str::FromStr;
 use std::time::Duration;

+use ::pprof::ProfilerGuardBuilder;
+use ::pprof::protos::Message as _;
 use anyhow::{Context, anyhow};
 use bytes::{Bytes, BytesMut};
 use hyper::header::{AUTHORIZATION, CONTENT_DISPOSITION, CONTENT_TYPE, HeaderName};
@@ -10,8 +12,7 @@ use hyper::http::HeaderValue;
 use hyper::{Body, Method, Request, Response};
 use metrics::{Encoder, IntCounter, TextEncoder, register_int_counter};
 use once_cell::sync::Lazy;
-use pprof::ProfilerGuardBuilder;
-use pprof::protos::Message as _;
+use regex::Regex;
 use routerify::ext::RequestExt;
 use routerify::{Middleware, RequestInfo, Router, RouterBuilder};
 use tokio::sync::{Mutex, Notify, mpsc};
@@ -21,6 +22,7 @@ use tracing::{Instrument, debug, info, info_span, warn};
 use utils::auth::{AuthError, Claims, SwappableJwtAuth};

 use crate::error::{ApiError, api_error_handler, route_error_handler};
+use crate::pprof;
 use crate::request::{get_query_param, parse_query_param};

 static SERVE_METRICS_COUNT: Lazy<IntCounter> = Lazy::new(|| {
@@ -447,6 +449,20 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
        Some(format) => return Err(ApiError::BadRequest(anyhow!("invalid format {format}"))),
    };

+    // Functions and mappings to strip when symbolizing pprof profiles. If true,
+    // also remove child frames.
+    static STRIP_FUNCTIONS: Lazy<Vec<(Regex, bool)>> = Lazy::new(|| {
+        vec![
+            (Regex::new("^__rust").unwrap(), false),
+            (Regex::new("^_start$").unwrap(), false),
+            (Regex::new("^irallocx_prof").unwrap(), true),
+            (Regex::new("^prof_alloc_prep").unwrap(), true),
+            (Regex::new("^std::rt::lang_start").unwrap(), false),
+            (Regex::new("^std::sys::backtrace::__rust").unwrap(), false),
+        ]
+    });
+    const STRIP_MAPPINGS: &[&str] = &["libc", "libgcc", "pthread", "vdso"];
+
    // Obtain profiler handle.
    let mut prof_ctl = jemalloc_pprof::PROF_CTL
        .as_ref()
@@ -479,27 +495,45 @@ pub async fn profile_heap_handler(req: Request<Body>) -> Result<Response<Body>,
        }

        Format::Pprof => {
-            let data = tokio::task::spawn_blocking(move || prof_ctl.dump_pprof())
-                .await
-                .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
-                .map_err(ApiError::InternalServerError)?;
+            let data = tokio::task::spawn_blocking(move || {
+                let bytes = prof_ctl.dump_pprof()?;
+                // Symbolize the profile.
+                // TODO: consider moving this upstream to jemalloc_pprof and avoiding the
+                // serialization roundtrip.
+                let profile = pprof::decode(&bytes)?;
+                let profile = pprof::symbolize(profile)?;
+                let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
+                pprof::encode(&profile)
+            })
+            .await
+            .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
+            .map_err(ApiError::InternalServerError)?;
            Response::builder()
                .status(200)
                .header(CONTENT_TYPE, "application/octet-stream")
-                .header(CONTENT_DISPOSITION, "attachment; filename=\"heap.pb.gz\"")
+                .header(CONTENT_DISPOSITION, "attachment; filename=\"heap.pb\"")
                .body(Body::from(data))
                .map_err(|err| ApiError::InternalServerError(err.into()))
        }

        Format::Svg => {
-            let svg = tokio::task::spawn_blocking(move || prof_ctl.dump_flamegraph())
-                .await
-                .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
-                .map_err(ApiError::InternalServerError)?;
+            let body = tokio::task::spawn_blocking(move || {
+                let bytes = prof_ctl.dump_pprof()?;
+                let profile = pprof::decode(&bytes)?;
+                let profile = pprof::symbolize(profile)?;
+                let profile = pprof::strip_locations(profile, STRIP_MAPPINGS, &STRIP_FUNCTIONS);
+                let mut opts = inferno::flamegraph::Options::default();
+                opts.title = "Heap inuse".to_string();
+                opts.count_name = "bytes".to_string();
+                pprof::flamegraph(profile, &mut opts)
+            })
+            .await
+            .map_err(|join_err| ApiError::InternalServerError(join_err.into()))?
+            .map_err(ApiError::InternalServerError)?;
            Response::builder()
                .status(200)
                .header(CONTENT_TYPE, "image/svg+xml")
-                .body(Body::from(svg))
+                .body(Body::from(body))
                .map_err(|err| ApiError::InternalServerError(err.into()))
        }
    }
--- a/libs/http-utils/src/lib.rs
+++ b/libs/http-utils/src/lib.rs
@@ -2,6 +2,7 @@ pub mod endpoint;
 pub mod error;
 pub mod failpoints;
 pub mod json;
+pub mod pprof;
 pub mod request;

 extern crate hyper0 as hyper;
--- a/libs/http-utils/src/pprof.rs
+++ b/libs/http-utils/src/pprof.rs
@@ -0,0 +1,238 @@
+use std::borrow::Cow;
+use std::collections::{HashMap, HashSet};
+use std::ffi::c_void;
+use std::io::Write as _;
+
+use anyhow::bail;
+use flate2::Compression;
+use flate2::write::{GzDecoder, GzEncoder};
+use itertools::Itertools as _;
+use pprof::protos::{Function, Line, Location, Message as _, Profile};
+use regex::Regex;
+
+/// Decodes a gzip-compressed Protobuf-encoded pprof profile.
+pub fn decode(bytes: &[u8]) -> anyhow::Result<Profile> {
+    let mut gz = GzDecoder::new(Vec::new());
+    gz.write_all(bytes)?;
+    Ok(Profile::parse_from_bytes(&gz.finish()?)?)
+}
+
+/// Encodes a pprof profile as gzip-compressed Protobuf.
+pub fn encode(profile: &Profile) -> anyhow::Result<Vec<u8>> {
+    let mut gz = GzEncoder::new(Vec::new(), Compression::default());
+    profile.write_to_writer(&mut gz)?;
+    Ok(gz.finish()?)
+}
+
+/// Symbolizes a pprof profile using the current binary.
+pub fn symbolize(mut profile: Profile) -> anyhow::Result<Profile> {
+    if !profile.function.is_empty() {
+        return Ok(profile); // already symbolized
+    }
+
+    // Collect function names.
+    let mut functions: HashMap<String, Function> = HashMap::new();
+    let mut strings: HashMap<String, i64> = profile
+        .string_table
+        .into_iter()
+        .enumerate()
+        .map(|(i, s)| (s, i as i64))
+        .collect();
+
+    // Helper to look up or register a string.
+    let mut string_id = |s: &str| -> i64 {
+        // Don't use .entry() to avoid unnecessary allocations.
+        if let Some(id) = strings.get(s) {
+            return *id;
+        }
+        let id = strings.len() as i64;
+        strings.insert(s.to_string(), id);
+        id
+    };
+
+    for loc in &mut profile.location {
+        if !loc.line.is_empty() {
+            continue;
+        }
+
+        // Resolve the line and function for each location.
+        backtrace::resolve(loc.address as *mut c_void, |symbol| {
+            let Some(symbol_name) = symbol.name() else {
+                return;
+            };
+
+            let function_name = format!("{symbol_name:#}");
+            let functions_len = functions.len();
+            let function_id = functions
+                .entry(function_name)
+                .or_insert_with_key(|function_name| {
+                    let function_id = functions_len as u64 + 1;
+                    let system_name = String::from_utf8_lossy(symbol_name.as_bytes());
+                    let filename = symbol
+                        .filename()
+                        .map(|path| path.to_string_lossy())
+                        .unwrap_or(Cow::Borrowed(""));
+                    Function {
+                        id: function_id,
+                        name: string_id(function_name),
+                        system_name: string_id(&system_name),
+                        filename: string_id(&filename),
+                        ..Default::default()
+                    }
+                })
+                .id;
+            loc.line.push(Line {
+                function_id,
+                line: symbol.lineno().unwrap_or(0) as i64,
+                ..Default::default()
+            });
+        });
+    }
+
+    // Store the resolved functions, and mark the mapping as resolved.
+    profile.function = functions.into_values().sorted_by_key(|f| f.id).collect();
+    profile.string_table = strings
+        .into_iter()
+        .sorted_by_key(|(_, i)| *i)
+        .map(|(s, _)| s)
+        .collect();
+
+    for mapping in &mut profile.mapping {
+        mapping.has_functions = true;
+        mapping.has_filenames = true;
+    }
+
+    Ok(profile)
+}
+
+/// Strips locations (stack frames) matching the given mappings (substring) or function names
+/// (regex). The function bool specifies whether child frames should be stripped as well.
+///
+/// The string definitions are left behind in the profile for simplicity, to avoid rewriting all
+/// string references.
+pub fn strip_locations(
+    mut profile: Profile,
+    mappings: &[&str],
+    functions: &[(Regex, bool)],
+) -> Profile {
+    // Strip mappings.
+    let mut strip_mappings: HashSet<u64> = HashSet::new();
+
+    profile.mapping.retain(|mapping| {
+        let Some(name) = profile.string_table.get(mapping.filename as usize) else {
+            return true;
+        };
+        if mappings.iter().any(|substr| name.contains(substr)) {
+            strip_mappings.insert(mapping.id);
+            return false;
+        }
+        true
+    });
+
+    // Strip functions.
+    let mut strip_functions: HashMap<u64, bool> = HashMap::new();
+
+    profile.function.retain(|function| {
+        let Some(name) = profile.string_table.get(function.name as usize) else {
+            return true;
+        };
+        for (regex, strip_children) in functions {
+            if regex.is_match(name) {
+                strip_functions.insert(function.id, *strip_children);
+                return false;
+            }
+        }
+        true
+    });
+
+    // Strip locations. The bool specifies whether child frames should be stripped too.
+    let mut strip_locations: HashMap<u64, bool> = HashMap::new();
+
+    profile.location.retain(|location| {
+        for line in &location.line {
+            if let Some(strip_children) = strip_functions.get(&line.function_id) {
+                strip_locations.insert(location.id, *strip_children);
+                return false;
+            }
+        }
+        if strip_mappings.contains(&location.mapping_id) {
+            strip_locations.insert(location.id, false);
+            return false;
+        }
+        true
+    });
+
+    // Strip sample locations.
+    for sample in &mut profile.sample {
+        // First, find the uppermost function with child removal and truncate the stack.
+        if let Some(truncate) = sample
+            .location_id
+            .iter()
+            .rposition(|id| strip_locations.get(id) == Some(&true))
+        {
+            sample.location_id.drain(..=truncate);
+        }
+        // Next, strip any individual frames without child removal.
+        sample
+            .location_id
+            .retain(|id| !strip_locations.contains_key(id));
+    }
+
+    profile
+}
+
+/// Generates an SVG flamegraph from a symbolized pprof profile.
+pub fn flamegraph(
+    profile: Profile,
+    opts: &mut inferno::flamegraph::Options,
+) -> anyhow::Result<Vec<u8>> {
+    if profile.mapping.iter().any(|m| !m.has_functions) {
+        bail!("profile not symbolized");
+    }
+
+    // Index locations, functions, and strings.
+    let locations: HashMap<u64, Location> =
+        profile.location.into_iter().map(|l| (l.id, l)).collect();
+    let functions: HashMap<u64, Function> =
+        profile.function.into_iter().map(|f| (f.id, f)).collect();
+    let strings = profile.string_table;
+
+    // Resolve stacks as function names, and sum sample values per stack. Also reverse the stack,
+    // since inferno expects it bottom-up.
+    let mut stacks: HashMap<Vec<&str>, i64> = HashMap::new();
+    for sample in profile.sample {
+        let mut stack = Vec::with_capacity(sample.location_id.len());
+        for location in sample.location_id.into_iter().rev() {
+            let Some(location) = locations.get(&location) else {
+                bail!("missing location {location}");
+            };
+            for line in location.line.iter().rev() {
+                let Some(function) = functions.get(&line.function_id) else {
+                    bail!("missing function {}", line.function_id);
+                };
+                let Some(name) = strings.get(function.name as usize) else {
+                    bail!("missing string {}", function.name);
+                };
+                stack.push(name.as_str());
+            }
+        }
+        let Some(&value) = sample.value.first() else {
+            bail!("missing value");
+        };
+        *stacks.entry(stack).or_default() += value;
+    }
+
+    // Construct stack lines for inferno.
+    let lines = stacks
+        .into_iter()
+        .map(|(stack, value)| (stack.into_iter().join(";"), value))
+        .map(|(stack, value)| format!("{stack} {value}"))
+        .sorted()
+        .collect_vec();
+
+    // Construct the flamegraph.
+    let mut bytes = Vec::new();
+    let lines = lines.iter().map(|line| line.as_str());
+    inferno::flamegraph::from_lines(opts, lines, &mut bytes)?;
+    Ok(bytes)
+}
--- a/libs/pageserver_api/src/config.rs
+++ b/libs/pageserver_api/src/config.rs
@@ -123,10 +123,6 @@ pub struct ConfigToml {
    pub enable_read_path_debugging: Option<bool>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub validate_wal_contiguity: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub load_previous_heatmap: Option<bool>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub generate_unarchival_heatmap: Option<bool>,
 }

 #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
@@ -527,8 +523,6 @@ impl Default for ConfigToml {
                None
            },
            validate_wal_contiguity: None,
-            load_previous_heatmap: None,
-            generate_unarchival_heatmap: None,
        }
    }
 }
--- a/libs/pageserver_api/src/models.rs
+++ b/libs/pageserver_api/src/models.rs
@@ -1146,15 +1146,6 @@ pub struct TimelineArchivalConfigRequest {
    pub state: TimelineArchivalState,
 }

-#[derive(Serialize, Deserialize, PartialEq, Eq, Clone)]
-pub struct TimelinePatchIndexPartRequest {
-    pub rel_size_migration: Option<RelSizeMigration>,
-    pub gc_compaction_last_completed_lsn: Option<Lsn>,
-    pub applied_gc_cutoff_lsn: Option<Lsn>,
-    #[serde(default)]
-    pub force_index_update: bool,
-}
-
 #[derive(Debug, Serialize, Deserialize, Clone)]
 pub struct TimelinesInfoAndOffloaded {
    pub timelines: Vec<TimelineInfo>,
--- a/libs/utils/Cargo.toml
+++ b/libs/utils/Cargo.toml
@@ -15,6 +15,7 @@ arc-swap.workspace = true
 sentry.workspace = true
 async-compression.workspace = true
 anyhow.workspace = true
+backtrace.workspace = true
 bincode.workspace = true
 bytes.workspace = true
 camino.workspace = true
--- a/libs/utils/src/sentry_init.rs
+++ b/libs/utils/src/sentry_init.rs
@@ -3,24 +3,20 @@ use std::env;

 use sentry::ClientInitGuard;
 pub use sentry::release_name;
-use tracing::{error, info};

 #[must_use]
 pub fn init_sentry(
    release_name: Option<Cow<'static, str>>,
    extra_options: &[(&str, &str)],
 ) -> Option<ClientInitGuard> {
-    let Ok(dsn) = env::var("SENTRY_DSN") else {
-        info!("not initializing Sentry, no SENTRY_DSN given");
-        return None;
-    };
+    let dsn = env::var("SENTRY_DSN").ok()?;
    let environment = env::var("SENTRY_ENVIRONMENT").unwrap_or_else(|_| "development".into());

    let guard = sentry::init((
        dsn,
        sentry::ClientOptions {
-            release: release_name.clone(),
-            environment: Some(environment.clone().into()),
+            release: release_name,
+            environment: Some(environment.into()),
            ..Default::default()
        },
    ));
@@ -29,19 +25,5 @@ pub fn init_sentry(
            scope.set_extra(key, value.into());
        }
    });
-
-    if let Some(dsn) = guard.dsn() {
-        info!(
-            "initialized Sentry for project {}, environment {}, release {} (using API {})",
-            dsn.project_id(),
-            environment,
-            release_name.unwrap_or(Cow::Borrowed("None")),
-            dsn.envelope_api_url(),
-        );
-    } else {
-        // This should panic during sentry::init(), but we may as well cover it.
-        error!("failed to initialize Sentry, invalid DSN");
-    }
-
    Some(guard)
 }
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -98,7 +98,6 @@ criterion.workspace = true
 hex-literal.workspace = true
 tokio = { workspace = true, features = ["process", "sync", "fs", "rt", "io-util", "time", "test-util"] }
 indoc.workspace = true
-uuid.workspace = true

 [[bench]]
 name = "bench_layer_map"
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -194,13 +194,6 @@ pub struct PageServerConf {
    /// Interpreted protocol feature: if enabled, validate that the logical WAL received from
    /// safekeepers does not have gaps.
    pub validate_wal_contiguity: bool,
-
-    /// When set, the previously written to disk heatmap is loaded on tenant attach and used
-    /// to avoid clobbering the heatmap from new, cold, attached locations.
-    pub load_previous_heatmap: bool,
-
-    /// When set, include visible layers in the next uploaded heatmaps of an unarchived timeline.
-    pub generate_unarchival_heatmap: bool,
 }

 /// Token for authentication to safekeepers
@@ -365,8 +358,6 @@ impl PageServerConf {
            get_vectored_concurrent_io,
            enable_read_path_debugging,
            validate_wal_contiguity,
-            load_previous_heatmap,
-            generate_unarchival_heatmap,
        } = config_toml;

        let mut conf = PageServerConf {
@@ -456,8 +447,6 @@ impl PageServerConf {
            no_sync: no_sync.unwrap_or(false),
            enable_read_path_debugging: enable_read_path_debugging.unwrap_or(false),
            validate_wal_contiguity: validate_wal_contiguity.unwrap_or(false),
-            load_previous_heatmap: load_previous_heatmap.unwrap_or(true),
-            generate_unarchival_heatmap: generate_unarchival_heatmap.unwrap_or(true),
        };

        // ------------------------------------------------------------
@@ -491,9 +480,7 @@ impl PageServerConf {
    #[cfg(test)]
    pub fn test_repo_dir(test_name: &str) -> Utf8PathBuf {
        let test_output_dir = std::env::var("TEST_OUTPUT").unwrap_or("../tmp_check".into());
-
-        let test_id = uuid::Uuid::new_v4();
-        Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}_{test_id}"))
+        Utf8PathBuf::from(format!("{test_output_dir}/test_{test_name}"))
    }

    pub fn dummy_conf(repo_dir: Utf8PathBuf) -> Self {
@@ -506,8 +493,6 @@ impl PageServerConf {
            metric_collection_interval: Duration::from_secs(60),
            synthetic_size_calculation_interval: Duration::from_secs(60),
            background_task_maximum_delay: Duration::ZERO,
-            load_previous_heatmap: Some(true),
-            generate_unarchival_heatmap: Some(true),
            ..Default::default()
        };
        PageServerConf::parse_and_validate(NodeId(0), config_toml, &repo_dir).unwrap()
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -37,8 +37,7 @@ use pageserver_api::models::{
    TenantShardSplitResponse, TenantSorting, TenantState, TenantWaitLsnRequest,
    TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateRequestMode,
    TimelineCreateRequestModeImportPgdata, TimelineGcRequest, TimelineInfo,
-    TimelinePatchIndexPartRequest, TimelinesInfoAndOffloaded, TopTenantShardItem,
-    TopTenantShardsRequest, TopTenantShardsResponse,
+    TimelinesInfoAndOffloaded, TopTenantShardItem, TopTenantShardsRequest, TopTenantShardsResponse,
 };
 use pageserver_api::shard::{ShardCount, TenantShardId};
 use remote_storage::{DownloadError, GenericRemoteStorage, TimeTravelError};
@@ -64,7 +63,6 @@ use crate::tenant::mgr::{
    GetActiveTenantError, GetTenantError, TenantManager, TenantMapError, TenantMapInsertError,
    TenantSlot, TenantSlotError, TenantSlotUpsertError, TenantStateError, UpsertLocationError,
 };
-use crate::tenant::remote_timeline_client::index::GcCompactionState;
 use crate::tenant::remote_timeline_client::{
    download_index_part, list_remote_tenant_shards, list_remote_timelines,
 };
@@ -860,75 +858,6 @@ async fn timeline_archival_config_handler(
    json_response(StatusCode::OK, ())
 }

-/// This API is used to patch the index part of a timeline. You must ensure such patches are safe to apply. Use this API as an emergency
-/// measure only.
-///
-/// Some examples of safe patches:
-/// - Increase the gc_cutoff and gc_compaction_cutoff to a larger value in case of a bug that didn't bump the cutoff and cause read errors.
-/// - Force set the index part to use reldir v2 (migrating/migrated).
-///
-/// Some examples of unsafe patches:
-/// - Force set the index part from v2 to v1 (legacy). This will cause the code path to ignore anything written to the new keyspace and cause
-///   errors.
-/// - Decrease the gc_cutoff without validating the data really exists. It will cause read errors in the background.
-async fn timeline_patch_index_part_handler(
-    mut request: Request<Body>,
-    _cancel: CancellationToken,
-) -> Result<Response<Body>, ApiError> {
-    let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
-    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-
-    let request_data: TimelinePatchIndexPartRequest = json_request(&mut request).await?;
-    check_permission(&request, None)?; // require global permission for this request
-    let state = get_state(&request);
-
-    async {
-        let timeline =
-            active_timeline_of_active_tenant(&state.tenant_manager, tenant_shard_id, timeline_id)
-                .await?;
-
-        if let Some(rel_size_migration) = request_data.rel_size_migration {
-            timeline
-                .update_rel_size_v2_status(rel_size_migration)
-                .map_err(ApiError::InternalServerError)?;
-        }
-
-        if let Some(gc_compaction_last_completed_lsn) =
-            request_data.gc_compaction_last_completed_lsn
-        {
-            timeline
-                .update_gc_compaction_state(GcCompactionState {
-                    last_completed_lsn: gc_compaction_last_completed_lsn,
-                })
-                .map_err(ApiError::InternalServerError)?;
-        }
-
-        if let Some(applied_gc_cutoff_lsn) = request_data.applied_gc_cutoff_lsn {
-            {
-                let guard = timeline.applied_gc_cutoff_lsn.lock_for_write();
-                guard.store_and_unlock(applied_gc_cutoff_lsn);
-            }
-        }
-
-        if request_data.force_index_update {
-            timeline
-                .remote_client
-                .force_schedule_index_upload()
-                .context("force schedule index upload")
-                .map_err(ApiError::InternalServerError)?;
-        }
-
-        Ok::<_, ApiError>(())
-    }
-    .instrument(info_span!("timeline_patch_index_part",
-                tenant_id = %tenant_shard_id.tenant_id,
-                shard_id = %tenant_shard_id.shard_slug(),
-                %timeline_id))
-    .await?;
-
-    json_response(StatusCode::OK, ())
-}
-
 async fn timeline_detail_handler(
    request: Request<Body>,
    _cancel: CancellationToken,
@@ -3700,10 +3629,6 @@ pub fn make_router(
            "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/get_timestamp_of_lsn",
            |r| api_handler(r, get_timestamp_of_lsn_handler),
        )
-        .post(
-            "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/patch_index_part",
-            |r| api_handler(r, timeline_patch_index_part_handler),
-        )
        .post(
            "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/lsn_lease",
            |r| api_handler(r, lsn_lease_handler),
--- a/pageserver/src/metrics.rs
+++ b/pageserver/src/metrics.rs
@@ -143,29 +143,6 @@ pub(crate) static LAYERS_PER_READ_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
    .expect("failed to define a metric")
 });

-pub(crate) static LAYERS_PER_READ_BATCH_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
-    register_histogram!(
-        "pageserver_layers_per_read_batch_global",
-        "Layers visited to serve a single read batch (read amplification), regardless of number of reads.",
-        vec![
-            1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
-        ],
-    )
-    .expect("failed to define a metric")
-});
-
-pub(crate) static LAYERS_PER_READ_AMORTIZED_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
-    register_histogram!(
-        "pageserver_layers_per_read_amortized_global",
-        "Layers visited to serve a single read (read amplification). Amortized across a batch: \
-            all visited layers are divided by number of reads.",
-        vec![
-            1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0
-        ],
-    )
-    .expect("failed to define a metric")
-});
-
 pub(crate) static DELTAS_PER_READ_GLOBAL: Lazy<Histogram> = Lazy::new(|| {
    // We expect this to be low because of Postgres checkpoints. Let's see if that holds.
    register_histogram!(
@@ -4097,8 +4074,6 @@ pub fn preinitialize_metrics(conf: &'static PageServerConf) {
    // histograms
    [
        &LAYERS_PER_READ_GLOBAL,
-        &LAYERS_PER_READ_BATCH_GLOBAL,
-        &LAYERS_PER_READ_AMORTIZED_GLOBAL,
        &DELTAS_PER_READ_GLOBAL,
        &WAIT_LSN_TIME,
        &WAL_REDO_TIME,
--- a/pageserver/src/pgdatadir_mapping.rs
+++ b/pageserver/src/pgdatadir_mapping.rs
@@ -602,38 +602,30 @@ impl Timeline {
        let n_blocks = self
            .get_slru_segment_size(kind, segno, Version::Lsn(lsn), ctx)
            .await?;
-
-        let keyspace = KeySpace::single(
-            slru_block_to_key(kind, segno, 0)..slru_block_to_key(kind, segno, n_blocks),
-        );
-
-        let batches = keyspace.partition(
-            self.get_shard_identity(),
-            Timeline::MAX_GET_VECTORED_KEYS * BLCKSZ as u64,
-        );
-
-        let io_concurrency = IoConcurrency::spawn_from_conf(
-            self.conf,
-            self.gate
-                .enter()
-                .map_err(|_| PageReconstructError::Cancelled)?,
-        );
-
        let mut segment = BytesMut::with_capacity(n_blocks as usize * BLCKSZ as usize);
-        for batch in batches.parts {
-            let blocks = self
-                .get_vectored(batch, lsn, io_concurrency.clone(), ctx)
+        for blkno in 0..n_blocks {
+            let block = self
+                .get_slru_page_at_lsn(kind, segno, blkno, lsn, ctx)
                .await?;
-
-            for (_key, block) in blocks {
-                let block = block?;
-                segment.extend_from_slice(&block[..BLCKSZ as usize]);
-            }
+            segment.extend_from_slice(&block[..BLCKSZ as usize]);
        }
-
        Ok(segment.freeze())
    }

+    /// Look up given SLRU page version.
+    pub(crate) async fn get_slru_page_at_lsn(
+        &self,
+        kind: SlruKind,
+        segno: u32,
+        blknum: BlockNumber,
+        lsn: Lsn,
+        ctx: &RequestContext,
+    ) -> Result<Bytes, PageReconstructError> {
+        assert!(self.tenant_shard_id.is_shard_zero());
+        let key = slru_block_to_key(kind, segno, blknum);
+        self.get(key, lsn, ctx).await
+    }
+
    /// Get size of an SLRU segment
    pub(crate) async fn get_slru_segment_size(
        &self,
@@ -840,41 +832,19 @@ impl Timeline {
            let nblocks = self
                .get_slru_segment_size(SlruKind::Clog, segno, Version::Lsn(probe_lsn), ctx)
                .await?;
-
-            let keyspace = KeySpace::single(
-                slru_block_to_key(SlruKind::Clog, segno, 0)
-                    ..slru_block_to_key(SlruKind::Clog, segno, nblocks),
-            );
-
-            let batches = keyspace.partition(
-                self.get_shard_identity(),
-                Timeline::MAX_GET_VECTORED_KEYS * BLCKSZ as u64,
-            );
-
-            let io_concurrency = IoConcurrency::spawn_from_conf(
-                self.conf,
-                self.gate
-                    .enter()
-                    .map_err(|_| PageReconstructError::Cancelled)?,
-            );
-
-            for batch in batches.parts.into_iter().rev() {
-                let blocks = self
-                    .get_vectored(batch, probe_lsn, io_concurrency.clone(), ctx)
+            for blknum in (0..nblocks).rev() {
+                let clog_page = self
+                    .get_slru_page_at_lsn(SlruKind::Clog, segno, blknum, probe_lsn, ctx)
                    .await?;

-                for (_key, clog_page) in blocks.into_iter().rev() {
-                    let clog_page = clog_page?;
+                if clog_page.len() == BLCKSZ as usize + 8 {
+                    let mut timestamp_bytes = [0u8; 8];
+                    timestamp_bytes.copy_from_slice(&clog_page[BLCKSZ as usize..]);
+                    let timestamp = TimestampTz::from_be_bytes(timestamp_bytes);

-                    if clog_page.len() == BLCKSZ as usize + 8 {
-                        let mut timestamp_bytes = [0u8; 8];
-                        timestamp_bytes.copy_from_slice(&clog_page[BLCKSZ as usize..]);
-                        let timestamp = TimestampTz::from_be_bytes(timestamp_bytes);
-
-                        match f(timestamp) {
-                            ControlFlow::Break(b) => return Ok(b),
-                            ControlFlow::Continue(()) => (),
-                        }
+                    match f(timestamp) {
+                        ControlFlow::Break(b) => return Ok(b),
+                        ControlFlow::Continue(()) => (),
                    }
                }
            }
@@ -1964,12 +1934,14 @@ impl DatadirModification<'_> {
                .context("deserialize db")?
        };

+        // Add the new relation to the rel directory entry, and write it back
+        if !rel_dir.rels.insert((rel.relnode, rel.forknum)) {
+            return Err(RelationError::AlreadyExists);
+        }
+
        let v2_enabled = self.maybe_enable_rel_size_v2()?;

        if v2_enabled {
-            if rel_dir.rels.contains(&(rel.relnode, rel.forknum)) {
-                return Err(RelationError::AlreadyExists);
-            }
            let sparse_rel_dir_key =
                rel_tag_sparse_key(rel.spcnode, rel.dbnode, rel.relnode, rel.forknum);
            // check if the rel_dir_key exists in v2
@@ -2004,10 +1976,6 @@ impl DatadirModification<'_> {
            self.pending_directory_entries
                .push((DirectoryKind::RelV2, MetricsUpdate::Add(1)));
        } else {
-            // Add the new relation to the rel directory entry, and write it back
-            if !rel_dir.rels.insert((rel.relnode, rel.forknum)) {
-                return Err(RelationError::AlreadyExists);
-            }
            if !dbdir_exists {
                self.pending_directory_entries
                    .push((DirectoryKind::Rel, MetricsUpdate::Set(0)))
@@ -2021,7 +1989,6 @@ impl DatadirModification<'_> {
                )),
            );
        }
-
        // Put size
        let size_key = rel_size_to_key(rel);
        let buf = nblocks.to_le_bytes();
@@ -2144,7 +2111,7 @@ impl DatadirModification<'_> {
                    // Remove entry from relation size cache
                    self.tline.remove_cached_rel_size(&rel_tag);

-                    // Delete size entry, as well as all blocks; this is currently a no-op because we haven't implemented tombstones in storage.
+                    // Delete size entry, as well as all blocks
                    self.delete(rel_key_range(rel_tag));
                }
            }
--- a/pageserver/src/tenant.rs
+++ b/pageserver/src/tenant.rs
@@ -1150,7 +1150,7 @@ impl Tenant {
        // a previous heatmap which contains all visible layers in the layer map.
        // This previous heatmap will be used whenever a fresh heatmap is generated
        // for the timeline.
-        if self.conf.generate_unarchival_heatmap && matches!(cause, LoadTimelineCause::Unoffload) {
+        if matches!(cause, LoadTimelineCause::Unoffload) {
            let mut tline_ending_at = Some((&timeline, timeline.get_last_record_lsn()));
            while let Some((tline, end_lsn)) = tline_ending_at {
                let unarchival_heatmap = tline.generate_unarchival_heatmap(end_lsn).await;
@@ -1582,10 +1582,6 @@ impl Tenant {
    }

    async fn read_on_disk_heatmap(&self) -> Option<(HeatMapTenant, std::time::Instant)> {
-        if !self.conf.load_previous_heatmap {
-            return None;
-        }
-
        let on_disk_heatmap_path = self.conf.tenant_heatmap_path(&self.tenant_shard_id);
        match tokio::fs::read_to_string(on_disk_heatmap_path).await {
            Ok(heatmap) => match serde_json::from_str::<HeatMapTenant>(&heatmap) {
@@ -2451,7 +2447,6 @@ impl Tenant {
            create_guard,
            initdb_lsn,
            None,
-            None,
        )
        .await
    }
@@ -2783,7 +2778,6 @@ impl Tenant {
                    timeline_create_guard,
                    initdb_lsn,
                    None,
-                    None,
                )
                .await
            }
@@ -4871,7 +4865,6 @@ impl Tenant {
                timeline_create_guard,
                start_lsn + 1,
                Some(Arc::clone(src_timeline)),
-                Some(src_timeline.get_rel_size_v2_status()),
            )
            .await?;

@@ -5145,7 +5138,6 @@ impl Tenant {
                timeline_create_guard,
                pgdata_lsn,
                None,
-                None,
            )
            .await?;

@@ -5224,14 +5216,13 @@ impl Tenant {
        create_guard: TimelineCreateGuard,
        start_lsn: Lsn,
        ancestor: Option<Arc<Timeline>>,
-        rel_size_v2_status: Option<RelSizeMigration>,
    ) -> anyhow::Result<UninitializedTimeline<'a>> {
        let tenant_shard_id = self.tenant_shard_id;

        let resources = self.build_timeline_resources(new_timeline_id);
        resources
            .remote_client
-            .init_upload_queue_for_empty_remote(new_metadata, rel_size_v2_status.clone())?;
+            .init_upload_queue_for_empty_remote(new_metadata)?;

        let timeline_struct = self
            .create_timeline_struct(
@@ -5243,7 +5234,7 @@ impl Tenant {
                CreateTimelineCause::Load,
                create_guard.idempotency.clone(),
                None,
-                rel_size_v2_status,
+                None,
            )
            .context("Failed to create timeline data structure")?;

--- a/pageserver/src/tenant/remote_timeline_client.rs
+++ b/pageserver/src/tenant/remote_timeline_client.rs
@@ -437,13 +437,9 @@ impl RemoteTimelineClient {

    /// Initialize the upload queue for the case where the remote storage is empty,
    /// i.e., it doesn't have an `IndexPart`.
-    ///
-    /// `rel_size_v2_status` needs to be carried over during branching, and that's why
-    /// it's passed in here.
    pub fn init_upload_queue_for_empty_remote(
        &self,
        local_metadata: &TimelineMetadata,
-        rel_size_v2_status: Option<RelSizeMigration>,
    ) -> anyhow::Result<()> {
        // Set the maximum number of inprogress tasks to the remote storage concurrency. There's
        // certainly no point in starting more upload tasks than this.
@@ -453,9 +449,7 @@ impl RemoteTimelineClient {
            .as_ref()
            .map_or(0, |r| r.concurrency_limit());
        let mut upload_queue = self.upload_queue.lock().unwrap();
-        let initialized_queue =
-            upload_queue.initialize_empty_remote(local_metadata, inprogress_limit)?;
-        initialized_queue.dirty.rel_size_migration = rel_size_v2_status;
+        upload_queue.initialize_empty_remote(local_metadata, inprogress_limit)?;
        self.update_remote_physical_size_gauge(None);
        info!("initialized upload queue as empty");
        Ok(())
@@ -954,14 +948,6 @@ impl RemoteTimelineClient {
        Ok(())
    }

-    /// Only used in the `patch_index_part` HTTP API to force trigger an index upload.
-    pub fn force_schedule_index_upload(self: &Arc<Self>) -> Result<(), NotInitialized> {
-        let mut guard = self.upload_queue.lock().unwrap();
-        let upload_queue = guard.initialized_mut()?;
-        self.schedule_index_upload(upload_queue);
-        Ok(())
-    }
-
    /// Launch an index-file upload operation in the background (internal function)
    fn schedule_index_upload(self: &Arc<Self>, upload_queue: &mut UploadQueueInitialized) {
        let disk_consistent_lsn = upload_queue.dirty.metadata.disk_consistent_lsn();
--- a/pageserver/src/tenant/secondary/downloader.rs
+++ b/pageserver/src/tenant/secondary/downloader.rs
@@ -869,7 +869,8 @@ impl<'a> TenantDownloader<'a> {
                let heatmap_timeline = heatmap.timelines.get(heatmap_timeline_index).unwrap();

                let layers_in_heatmap = heatmap_timeline
-                    .hot_layers()
+                    .layers
+                    .iter()
                    .map(|l| (&l.name, l.metadata.generation))
                    .collect::<HashSet<_>>();
                let layers_on_disk = timeline_state
@@ -1014,8 +1015,7 @@ impl<'a> TenantDownloader<'a> {
        // Accumulate updates to the state
        let mut touched = Vec::new();

-        let timeline_id = timeline.timeline_id;
-        for layer in timeline.into_hot_layers() {
+        for layer in timeline.layers {
            if self.secondary_state.cancel.is_cancelled() {
                tracing::debug!("Cancelled -- dropping out of layer loop");
                return (Err(UpdateError::Cancelled), touched);
@@ -1040,7 +1040,7 @@ impl<'a> TenantDownloader<'a> {
            }

            match self
-                .download_layer(tenant_shard_id, &timeline_id, layer, ctx)
+                .download_layer(tenant_shard_id, &timeline.timeline_id, layer, ctx)
                .await
            {
                Ok(Some(layer)) => touched.push(layer),
@@ -1148,7 +1148,7 @@ impl<'a> TenantDownloader<'a> {
        let tenant_shard_id = self.secondary_state.get_tenant_shard_id();
        let timeline_id = timeline.timeline_id;

-        tracing::debug!(timeline_id=%timeline_id, "Downloading layers, {} in heatmap", timeline.hot_layers().count());
+        tracing::debug!(timeline_id=%timeline_id, "Downloading layers, {} in heatmap", timeline.layers.len());

        let (result, touched) = self
            .download_timeline_layers(tenant_shard_id, timeline, timeline_state, deadline, ctx)
@@ -1316,11 +1316,11 @@ async fn init_timeline_state(
    // As we iterate through layers found on disk, we will look up their metadata from this map.
    // Layers not present in metadata will be discarded.
    let heatmap_metadata: HashMap<&LayerName, &HeatMapLayer> =
-        heatmap.hot_layers().map(|l| (&l.name, l)).collect();
+        heatmap.layers.iter().map(|l| (&l.name, l)).collect();

    let last_heatmap_metadata: HashMap<&LayerName, &HeatMapLayer> =
        if let Some(last_heatmap) = last_heatmap {
-            last_heatmap.hot_layers().map(|l| (&l.name, l)).collect()
+            last_heatmap.layers.iter().map(|l| (&l.name, l)).collect()
        } else {
            HashMap::new()
        };
--- a/pageserver/src/tenant/secondary/heatmap.rs
+++ b/pageserver/src/tenant/secondary/heatmap.rs
@@ -42,7 +42,7 @@ pub(crate) struct HeatMapTimeline {
    #[serde_as(as = "DisplayFromStr")]
    pub(crate) timeline_id: TimelineId,

-    layers: Vec<HeatMapLayer>,
+    pub(crate) layers: Vec<HeatMapLayer>,
 }

 #[serde_as]
@@ -53,10 +53,8 @@ pub(crate) struct HeatMapLayer {

    #[serde_as(as = "TimestampSeconds<i64>")]
    pub(crate) access_time: SystemTime,
-
-    #[serde(default)]
-    pub(crate) cold: bool, // TODO: an actual 'heat' score that would let secondary locations prioritize downloading
-                           // the hottest layers, rather than trying to simply mirror whatever layers are on-disk on the primary.
+    // TODO: an actual 'heat' score that would let secondary locations prioritize downloading
+    // the hottest layers, rather than trying to simply mirror whatever layers are on-disk on the primary.
 }

 impl HeatMapLayer {
@@ -64,13 +62,11 @@ impl HeatMapLayer {
        name: LayerName,
        metadata: LayerFileMetadata,
        access_time: SystemTime,
-        cold: bool,
    ) -> Self {
        Self {
            name,
            metadata,
            access_time,
-            cold,
        }
    }
 }
@@ -82,18 +78,6 @@ impl HeatMapTimeline {
            layers,
        }
    }
-
-    pub(crate) fn into_hot_layers(self) -> impl Iterator<Item = HeatMapLayer> {
-        self.layers.into_iter().filter(|l| !l.cold)
-    }
-
-    pub(crate) fn hot_layers(&self) -> impl Iterator<Item = &HeatMapLayer> {
-        self.layers.iter().filter(|l| !l.cold)
-    }
-
-    pub(crate) fn all_layers(&self) -> impl Iterator<Item = &HeatMapLayer> {
-        self.layers.iter()
-    }
 }

 pub(crate) struct HeatMapStats {
@@ -108,7 +92,7 @@ impl HeatMapTenant {
            layers: 0,
        };
        for timeline in &self.timelines {
-            for layer in timeline.hot_layers() {
+            for layer in &timeline.layers {
                stats.layers += 1;
                stats.bytes += layer.metadata.file_size;
            }
--- a/pageserver/src/tenant/storage_layer/layer.rs
+++ b/pageserver/src/tenant/storage_layer/layer.rs
@@ -1563,10 +1563,10 @@ impl LayerInner {

        self.access_stats.record_residence_event();

-        *self.last_evicted_at.lock().unwrap() = Some(std::time::Instant::now());
-
        self.status.as_ref().unwrap().send_replace(Status::Evicted);

+        *self.last_evicted_at.lock().unwrap() = Some(std::time::Instant::now());
+
        Ok(())
    }

--- a/pageserver/src/tenant/timeline.rs
+++ b/pageserver/src/tenant/timeline.rs
@@ -99,8 +99,7 @@ use crate::disk_usage_eviction_task::{DiskUsageEvictionInfo, EvictionCandidate,
 use crate::keyspace::{KeyPartitioning, KeySpace};
 use crate::l0_flush::{self, L0FlushGlobalState};
 use crate::metrics::{
-    DELTAS_PER_READ_GLOBAL, LAYERS_PER_READ_AMORTIZED_GLOBAL, LAYERS_PER_READ_BATCH_GLOBAL,
-    LAYERS_PER_READ_GLOBAL, ScanLatencyOngoingRecording, TimelineMetrics,
+    DELTAS_PER_READ_GLOBAL, LAYERS_PER_READ_GLOBAL, ScanLatencyOngoingRecording, TimelineMetrics,
 };
 use crate::page_service::TenantManagerTypes;
 use crate::pgdatadir_mapping::{
@@ -1331,6 +1330,10 @@ impl Timeline {
        // (this is a requirement, not a bug). Skip updating the metric in these cases
        // to avoid infinite results.
        if !results.is_empty() {
+            // Record the total number of layers visited towards each key in the batch. While some
+            // layers may not intersect with a given read, and the cost of layer visits are
+            // amortized across the batch, each visited layer contributes directly to the observed
+            // latency for every read in the batch, which is what we care about.
            if layers_visited >= Self::LAYERS_VISITED_WARN_THRESHOLD {
                static LOG_PACER: Lazy<Mutex<RateLimit>> =
                    Lazy::new(|| Mutex::new(RateLimit::new(Duration::from_secs(60))));
@@ -1345,23 +1348,9 @@ impl Timeline {
                });
            }

-            // Records the number of layers visited in a few different ways:
-            //
-            // * LAYERS_PER_READ: all layers count towards every read in the batch, because each
-            //   layer directly affects its observed latency.
-            //
-            // * LAYERS_PER_READ_BATCH: all layers count towards each batch, to get the per-batch
-            //   layer visits and access cost.
-            //
-            // * LAYERS_PER_READ_AMORTIZED: the average layer count per read, to get the amortized
-            //   read amplification after batching.
-            let layers_visited = layers_visited as f64;
-            let avg_layers_visited = layers_visited / results.len() as f64;
-            LAYERS_PER_READ_BATCH_GLOBAL.observe(layers_visited);
            for _ in &results {
-                self.metrics.layers_per_read.observe(layers_visited);
-                LAYERS_PER_READ_GLOBAL.observe(layers_visited);
-                LAYERS_PER_READ_AMORTIZED_GLOBAL.observe(avg_layers_visited);
+                self.metrics.layers_per_read.observe(layers_visited as f64);
+                LAYERS_PER_READ_GLOBAL.observe(layers_visited as f64);
            }
        }

@@ -3648,7 +3637,7 @@ impl Timeline {
        let visible_non_resident = match previous_heatmap.as_deref() {
            Some(PreviousHeatmap::Active {
                heatmap, read_at, ..
-            }) => Some(heatmap.all_layers().filter_map(|hl| {
+            }) => Some(heatmap.layers.iter().filter_map(|hl| {
                let desc: PersistentLayerDesc = hl.name.clone().into();
                let layer = guard.try_get_from_key(&desc.key())?;

@@ -3664,7 +3653,7 @@ impl Timeline {
                    return None;
                }

-                Some((desc, hl.metadata.clone(), hl.access_time, hl.cold))
+                Some((desc, hl.metadata.clone(), hl.access_time))
            })),
            Some(PreviousHeatmap::Obsolete) => None,
            None => None,
@@ -3680,7 +3669,6 @@ impl Timeline {
                        layer.layer_desc().clone(),
                        layer.metadata(),
                        last_activity_ts,
-                        false, // these layers are not cold
                    ))
                }
                LayerVisibilityHint::Covered => {
@@ -3707,14 +3695,12 @@ impl Timeline {
        // Sort layers in order of which to download first.  For a large set of layers to download, we
        // want to prioritize those layers which are most likely to still be in the resident many minutes
        // or hours later:
-        // - Cold layers go last for convenience when a human inspects the heatmap.
        // - Download L0s last, because they churn the fastest: L0s on a fast-writing tenant might
        //   only exist for a few minutes before being compacted into L1s.
        // - For L1 & image layers, download most recent LSNs first: the older the LSN, the sooner
        //   the layer is likely to be covered by an image layer during compaction.
-        layers.sort_by_key(|(desc, _meta, _atime, cold)| {
+        layers.sort_by_key(|(desc, _meta, _atime)| {
            std::cmp::Reverse((
-                *cold,
                !LayerMap::is_l0(&desc.key_range, desc.is_delta),
                desc.lsn_range.end,
            ))
@@ -3722,9 +3708,7 @@ impl Timeline {

        let layers = layers
            .into_iter()
-            .map(|(desc, meta, atime, cold)| {
-                HeatMapLayer::new(desc.layer_name(), meta, atime, cold)
-            })
+            .map(|(desc, meta, atime)| HeatMapLayer::new(desc.layer_name(), meta, atime))
            .collect();

        Some(HeatMapTimeline::new(self.timeline_id, layers))
@@ -3744,7 +3728,6 @@ impl Timeline {
                name: vl.layer_desc().layer_name(),
                metadata: vl.metadata(),
                access_time: now,
-                cold: true,
            };
            heatmap_layers.push(hl);
        }
@@ -7046,7 +7029,6 @@ mod tests {

    use pageserver_api::key::Key;
    use pageserver_api::value::Value;
-    use std::iter::Iterator;
    use tracing::Instrument;
    use utils::id::TimelineId;
    use utils::lsn::Lsn;
@@ -7060,8 +7042,8 @@ mod tests {
    use crate::tenant::{PreviousHeatmap, Timeline};

    fn assert_heatmaps_have_same_layers(lhs: &HeatMapTimeline, rhs: &HeatMapTimeline) {
-        assert_eq!(lhs.all_layers().count(), rhs.all_layers().count());
-        let lhs_rhs = lhs.all_layers().zip(rhs.all_layers());
+        assert_eq!(lhs.layers.len(), rhs.layers.len());
+        let lhs_rhs = lhs.layers.iter().zip(rhs.layers.iter());
        for (l, r) in lhs_rhs {
            assert_eq!(l.name, r.name);
            assert_eq!(l.metadata, r.metadata);
@@ -7139,11 +7121,10 @@ mod tests {
        assert_eq!(heatmap.timeline_id, timeline.timeline_id);

        // L0 should come last
-        let heatmap_layers = heatmap.all_layers().collect::<Vec<_>>();
-        assert_eq!(heatmap_layers.last().unwrap().name, l0_delta.layer_name());
+        assert_eq!(heatmap.layers.last().unwrap().name, l0_delta.layer_name());

        let mut last_lsn = Lsn::MAX;
-        for layer in heatmap_layers {
+        for layer in &heatmap.layers {
            // Covered layer should be omitted
            assert!(layer.name != covered_delta.layer_name());

@@ -7272,7 +7253,7 @@ mod tests {
            .expect("Infallible while timeline is not shut down");

        // Both layers should be in the heatmap
-        assert!(heatmap.all_layers().count() > 0);
+        assert!(!heatmap.layers.is_empty());

        // Now simulate a migration.
        timeline
@@ -7298,7 +7279,7 @@ mod tests {
            .await
            .expect("Infallible while timeline is not shut down");

-        assert_eq!(post_eviction_heatmap.all_layers().count(), 0);
+        assert!(post_eviction_heatmap.layers.is_empty());
        assert!(matches!(
            timeline.previous_heatmap.load().as_deref(),
            Some(PreviousHeatmap::Obsolete)
--- a/pageserver/src/tenant/timeline/compaction.rs
+++ b/pageserver/src/tenant/timeline/compaction.rs
@@ -7,7 +7,6 @@
 use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
 use std::ops::{Deref, Range};
 use std::sync::Arc;
-use std::time::Instant;

 use super::layer_manager::LayerManager;
 use super::{
@@ -16,11 +15,10 @@ use super::{
    Timeline,
 };

-use anyhow::{Context, anyhow};
+use anyhow::{Context, anyhow, bail};
 use bytes::Bytes;
 use enumset::EnumSet;
 use fail::fail_point;
-use futures::FutureExt;
 use itertools::Itertools;
 use once_cell::sync::Lazy;
 use pageserver_api::config::tenant_conf_defaults::DEFAULT_CHECKPOINT_DISTANCE;
@@ -236,12 +234,6 @@ impl GcCompactionQueue {
            // it enough in staging yet.
            return Ok(());
        }
-        if timeline.get_gc_compaction_watermark() == Lsn::INVALID {
-            // If the gc watermark is not set, we don't need to trigger auto compaction.
-            // This check is the same as in `gc_compaction_split_jobs` but we don't log
-            // here and we can also skip the computation of the trigger condition earlier.
-            return Ok(());
-        }

        let Ok(permit) = CONCURRENT_GC_COMPACTION_TASKS.clone().try_acquire_owned() else {
            // Only allow one compaction run at a time. TODO: As we do `try_acquire_owned`, we cannot ensure
@@ -365,7 +357,8 @@ impl GcCompactionQueue {
                GcCompactJob::from_compact_options(options.clone()),
                options.sub_compaction_max_job_size_mb,
            )
-            .await?;
+            .await
+            .map_err(CompactionError::Other)?;
        if jobs.is_empty() {
            info!("no jobs to run, skipping scheduled compaction task");
            self.notify_and_unblock(id);
@@ -444,7 +437,6 @@ impl GcCompactionQueue {
            ));
        };
        let has_pending_tasks;
-        let mut yield_for_l0 = false;
        let Some((id, item)) = ({
            let mut guard = self.inner.lock().unwrap();
            if let Some((id, item)) = guard.queued.pop_front() {
@@ -494,23 +486,13 @@ impl GcCompactionQueue {
                        let mut guard = self.inner.lock().unwrap();
                        guard.guards.entry(id).or_default().gc_guard = Some(gc_guard);
                    }
-                    let compaction_result =
-                        timeline.compact_with_options(cancel, options, ctx).await?;
+                    let _ = timeline.compact_with_options(cancel, options, ctx).await?;
                    self.notify_and_unblock(id);
-                    if compaction_result == CompactionOutcome::YieldForL0 {
-                        yield_for_l0 = true;
-                    }
                }
            }
            GcCompactionQueueItem::SubCompactionJob(options) => {
                // TODO: error handling, clear the queue if any task fails?
-                let compaction_result = timeline.compact_with_options(cancel, options, ctx).await?;
-                if compaction_result == CompactionOutcome::YieldForL0 {
-                    // We will permenantly give up a task if we yield for L0 compaction: the preempted subcompaction job won't be running
-                    // again. This ensures that we don't keep doing duplicated work within gc-compaction. Not directly returning here because
-                    // we need to clean things up before returning from the function.
-                    yield_for_l0 = true;
-                }
+                let _ = timeline.compact_with_options(cancel, options, ctx).await?;
            }
            GcCompactionQueueItem::Notify(id, l2_lsn) => {
                self.notify_and_unblock(id);
@@ -539,10 +521,7 @@ impl GcCompactionQueue {
            let mut guard = self.inner.lock().unwrap();
            guard.running = None;
        }
-        Ok(if yield_for_l0 {
-            tracing::info!("give up gc-compaction: yield for L0 compaction");
-            CompactionOutcome::YieldForL0
-        } else if has_pending_tasks {
+        Ok(if has_pending_tasks {
            CompactionOutcome::Pending
        } else {
            CompactionOutcome::Done
@@ -740,41 +719,17 @@ struct CompactionStatisticsNumSize {

 #[derive(Debug, Serialize, Default)]
 pub struct CompactionStatistics {
-    /// Delta layer visited (maybe compressed, physical size)
    delta_layer_visited: CompactionStatisticsNumSize,
-    /// Image layer visited (maybe compressed, physical size)
    image_layer_visited: CompactionStatisticsNumSize,
-    /// Delta layer produced (maybe compressed, physical size)
    delta_layer_produced: CompactionStatisticsNumSize,
-    /// Image layer produced (maybe compressed, physical size)
    image_layer_produced: CompactionStatisticsNumSize,
-    /// Delta layer discarded (maybe compressed, physical size of the layer being discarded instead of the original layer)
-    delta_layer_discarded: CompactionStatisticsNumSize,
-    /// Image layer discarded (maybe compressed, physical size of the layer being discarded instead of the original layer)
-    image_layer_discarded: CompactionStatisticsNumSize,
+    num_delta_layer_discarded: usize,
+    num_image_layer_discarded: usize,
    num_unique_keys_visited: usize,
-    /// Delta visited (uncompressed, original size)
    wal_keys_visited: CompactionStatisticsNumSize,
-    /// Image visited (uncompressed, original size)
    image_keys_visited: CompactionStatisticsNumSize,
-    /// Delta produced (uncompressed, original size)
    wal_produced: CompactionStatisticsNumSize,
-    /// Image produced (uncompressed, original size)
    image_produced: CompactionStatisticsNumSize,
-
-    // Time spent in each phase
-    time_acquire_lock_secs: f64,
-    time_analyze_secs: f64,
-    time_download_layer_secs: f64,
-    time_main_loop_secs: f64,
-    time_final_phase_secs: f64,
-    time_total_secs: f64,
-
-    // Summary
-    /// Ratio of the key-value size before/after gc-compaction.
-    uncompressed_size_ratio: f64,
-    /// Ratio of the physical size before/after gc-compaction.
-    physical_size_ratio: f64,
 }

 impl CompactionStatistics {
@@ -824,13 +779,11 @@ impl CompactionStatistics {
        self.image_produced.num += 1;
        self.image_produced.size += val.len() as u64 + Self::estimated_size_of_key() as u64;
    }
-    fn discard_delta_layer(&mut self, original_size: u64) {
-        self.delta_layer_discarded.num += 1;
-        self.delta_layer_discarded.size += original_size;
+    fn discard_delta_layer(&mut self) {
+        self.num_delta_layer_discarded += 1;
    }
-    fn discard_image_layer(&mut self, original_size: u64) {
-        self.image_layer_discarded.num += 1;
-        self.image_layer_discarded.size += original_size;
+    fn discard_image_layer(&mut self) {
+        self.num_image_layer_discarded += 1;
    }
    fn produce_delta_layer(&mut self, size: u64) {
        self.delta_layer_produced.num += 1;
@@ -840,19 +793,6 @@ impl CompactionStatistics {
        self.image_layer_produced.num += 1;
        self.image_layer_produced.size += size;
    }
-    fn finalize(&mut self) {
-        let original_key_value_size = self.image_keys_visited.size + self.wal_keys_visited.size;
-        let produced_key_value_size = self.image_produced.size + self.wal_produced.size;
-        self.uncompressed_size_ratio =
-            original_key_value_size as f64 / (produced_key_value_size as f64 + 1.0); // avoid div by 0
-        let original_physical_size = self.image_layer_visited.size + self.delta_layer_visited.size;
-        let produced_physical_size = self.image_layer_produced.size
-            + self.delta_layer_produced.size
-            + self.image_layer_discarded.size
-            + self.delta_layer_discarded.size; // Also include the discarded layers to make the ratio accurate
-        self.physical_size_ratio =
-            original_physical_size as f64 / (produced_physical_size as f64 + 1.0); // avoid div by 0
-    }
 }

 #[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
@@ -885,7 +825,9 @@ impl Timeline {
            .flags
            .contains(CompactFlags::EnhancedGcBottomMostCompaction)
        {
-            self.compact_with_gc(cancel, options, ctx).await?;
+            self.compact_with_gc(cancel, options, ctx)
+                .await
+                .map_err(CompactionError::Other)?;
            return Ok(CompactionOutcome::Done);
        }

@@ -2403,19 +2345,12 @@ impl Timeline {
    async fn check_compaction_space(
        self: &Arc<Self>,
        layer_selection: &[Layer],
-    ) -> Result<(), CompactionError> {
-        let available_space = self
-            .check_available_space()
-            .await
-            .map_err(CompactionError::Other)?;
+    ) -> anyhow::Result<()> {
+        let available_space = self.check_available_space().await?;
        let mut remote_layer_size = 0;
        let mut all_layer_size = 0;
        for layer in layer_selection {
-            let needs_download = layer
-                .needs_download()
-                .await
-                .context("failed to check if layer needs download")
-                .map_err(CompactionError::Other)?;
+            let needs_download = layer.needs_download().await?;
            if needs_download.is_some() {
                remote_layer_size += layer.layer_desc().file_size;
            }
@@ -2424,14 +2359,14 @@ impl Timeline {
        let allocated_space = (available_space as f64 * 0.8) as u64; /* reserve 20% space for other tasks */
        if all_layer_size /* space needed for newly-generated file */ + remote_layer_size /* space for downloading layers */ > allocated_space
        {
-            return Err(CompactionError::Other(anyhow!(
+            return Err(anyhow!(
                "not enough space for compaction: available_space={}, allocated_space={}, all_layer_size={}, remote_layer_size={}, required_space={}",
                available_space,
                allocated_space,
                all_layer_size,
                remote_layer_size,
                all_layer_size + remote_layer_size
-            )));
+            ));
        }
        Ok(())
    }
@@ -2462,7 +2397,7 @@ impl Timeline {
        self: &Arc<Self>,
        job: GcCompactJob,
        sub_compaction_max_job_size_mb: Option<u64>,
-    ) -> Result<Vec<GcCompactJob>, CompactionError> {
+    ) -> anyhow::Result<Vec<GcCompactJob>> {
        let compact_below_lsn = if job.compact_lsn_range.end != Lsn::MAX {
            job.compact_lsn_range.end
        } else {
@@ -2613,7 +2548,7 @@ impl Timeline {
        cancel: &CancellationToken,
        options: CompactOptions,
        ctx: &RequestContext,
-    ) -> Result<CompactionOutcome, CompactionError> {
+    ) -> anyhow::Result<()> {
        let sub_compaction = options.sub_compaction;
        let job = GcCompactJob::from_compact_options(options.clone());
        if sub_compaction {
@@ -2635,7 +2570,7 @@ impl Timeline {
            if jobs_len == 0 {
                info!("no jobs to run, skipping gc bottom-most compaction");
            }
-            return Ok(CompactionOutcome::Done);
+            return Ok(());
        }
        self.compact_with_gc_inner(cancel, job, ctx).await
    }
@@ -2645,24 +2580,19 @@ impl Timeline {
        cancel: &CancellationToken,
        job: GcCompactJob,
        ctx: &RequestContext,
-    ) -> Result<CompactionOutcome, CompactionError> {
+    ) -> anyhow::Result<()> {
        // Block other compaction/GC tasks from running for now. GC-compaction could run along
        // with legacy compaction tasks in the future. Always ensure the lock order is compaction -> gc.
        // Note that we already acquired the compaction lock when the outer `compact` function gets called.

-        let timer = Instant::now();
-        let begin_timer = timer;
-
        let gc_lock = async {
            tokio::select! {
                guard = self.gc_lock.lock() => Ok(guard),
-                _ = cancel.cancelled() => Err(CompactionError::ShuttingDown),
+                // TODO: refactor to CompactionError to correctly pass cancelled error
+                _ = cancel.cancelled() => Err(anyhow!("cancelled")),
            }
        };

-        let time_acquire_lock = timer.elapsed();
-        let timer = Instant::now();
-
        let gc_lock = crate::timed(
            gc_lock,
            "acquires gc lock",
@@ -2714,7 +2644,7 @@ impl Timeline {
                        tracing::warn!(
                            "no layers to compact with gc: gc_cutoff not generated yet, skipping gc bottom-most compaction"
                        );
-                        return Ok(CompactionOutcome::Skipped);
+                        return Ok(());
                    }
                    real_gc_cutoff
                } else {
@@ -2752,7 +2682,7 @@ impl Timeline {
                    "no layers to compact with gc: no historic layers below gc_cutoff, gc_cutoff={}",
                    gc_cutoff
                );
-                return Ok(CompactionOutcome::Done);
+                return Ok(());
            };
            // Next, if the user specifies compact_lsn_range.start, we need to filter some layers out. All the layers (strictly) below
            // the min_layer_lsn computed as below will be filtered out and the data will be accessed using the normal read path, as if
@@ -2773,7 +2703,7 @@ impl Timeline {
                    "no layers to compact with gc: no historic layers above compact_above_lsn, compact_above_lsn={}",
                    compact_lsn_range.end
                );
-                return Ok(CompactionOutcome::Done);
+                return Ok(());
            };
            // Then, pick all the layers that are below the max_layer_lsn. This is to ensure we can pick all single-key
            // layers to compact.
@@ -2799,7 +2729,7 @@ impl Timeline {
                    "no layers to compact with gc: no layers within the key range, gc_cutoff={}, key_range={}..{}",
                    gc_cutoff, compact_key_range.start, compact_key_range.end
                );
-                return Ok(CompactionOutcome::Done);
+                return Ok(());
            }
            retain_lsns_below_horizon.sort();
            GcCompactionJobDescription {
@@ -2852,9 +2782,6 @@ impl Timeline {
            has_data_below,
        );

-        let time_analyze = timer.elapsed();
-        let timer = Instant::now();
-
        for layer in &job_desc.selected_layers {
            debug!("read layer: {}", layer.layer_desc().key());
        }
@@ -2883,10 +2810,10 @@ impl Timeline {
            .map(|layer| layer.layer_desc().layer_name())
            .collect_vec();
        if let Some(err) = check_valid_layermap(&layer_names) {
-            return Err(CompactionError::Other(anyhow!(
+            bail!(
                "gc-compaction layer map check failed because {}, cannot proceed with compaction due to potential data loss",
                err
-            )));
+            );
        }
        // The maximum LSN we are processing in this compaction loop
        let end_lsn = job_desc
@@ -2901,33 +2828,11 @@ impl Timeline {
        let mut total_downloaded_size = 0;
        let mut total_layer_size = 0;
        for layer in &job_desc.selected_layers {
-            if layer
-                .needs_download()
-                .await
-                .context("failed to check if layer needs download")
-                .map_err(CompactionError::Other)?
-                .is_some()
-            {
+            if layer.needs_download().await?.is_some() {
                total_downloaded_size += layer.layer_desc().file_size;
            }
            total_layer_size += layer.layer_desc().file_size;
-            if cancel.is_cancelled() {
-                return Err(CompactionError::ShuttingDown);
-            }
-            let should_yield = self
-                .l0_compaction_trigger
-                .notified()
-                .now_or_never()
-                .is_some();
-            if should_yield {
-                tracing::info!("preempt gc-compaction when downloading layers: too many L0 layers");
-                return Ok(CompactionOutcome::YieldForL0);
-            }
-            let resident_layer = layer
-                .download_and_keep_resident(ctx)
-                .await
-                .context("failed to download and keep resident layer")
-                .map_err(CompactionError::Other)?;
+            let resident_layer = layer.download_and_keep_resident(ctx).await?;
            downloaded_layers.push(resident_layer);
        }
        info!(
@@ -2938,36 +2843,19 @@ impl Timeline {
        );
        for resident_layer in &downloaded_layers {
            if resident_layer.layer_desc().is_delta() {
-                let layer = resident_layer
-                    .get_as_delta(ctx)
-                    .await
-                    .context("failed to get delta layer")
-                    .map_err(CompactionError::Other)?;
+                let layer = resident_layer.get_as_delta(ctx).await?;
                delta_layers.push(layer);
            } else {
-                let layer = resident_layer
-                    .get_as_image(ctx)
-                    .await
-                    .context("failed to get image layer")
-                    .map_err(CompactionError::Other)?;
+                let layer = resident_layer.get_as_image(ctx).await?;
                image_layers.push(layer);
            }
        }
-        let (dense_ks, sparse_ks) = self
-            .collect_gc_compaction_keyspace()
-            .await
-            .context("failed to collect gc compaction keyspace")
-            .map_err(CompactionError::Other)?;
+        let (dense_ks, sparse_ks) = self.collect_gc_compaction_keyspace().await?;
        let mut merge_iter = FilterIterator::create(
            MergeIterator::create(&delta_layers, &image_layers, ctx),
            dense_ks,
            sparse_ks,
-        )
-        .context("failed to create filter iterator")
-        .map_err(CompactionError::Other)?;
-
-        let time_download_layer = timer.elapsed();
-        let timer = Instant::now();
+        )?;

        // Step 2: Produce images+deltas.
        let mut accumulated_values = Vec::new();
@@ -2986,9 +2874,7 @@ impl Timeline {
                    self.get_compaction_target_size(),
                    ctx,
                )
-                .await
-                .context("failed to create image layer writer")
-                .map_err(CompactionError::Other)?,
+                .await?,
            )
        } else {
            None
@@ -3001,9 +2887,7 @@ impl Timeline {
            lowest_retain_lsn..end_lsn,
            self.get_compaction_target_size(),
        )
-        .await
-        .context("failed to create delta layer writer")
-        .map_err(CompactionError::Other)?;
+        .await?;

        #[derive(Default)]
        struct RewritingLayers {
@@ -3043,28 +2927,9 @@ impl Timeline {
        // the key and LSN range are determined. However, to keep things simple here, we still
        // create this writer, and discard the writer in the end.

-        let mut keys_processed = 0;
-
-        while let Some(((key, lsn, val), desc)) = merge_iter
-            .next_with_trace()
-            .await
-            .context("failed to get next key-value pair")
-            .map_err(CompactionError::Other)?
-        {
+        while let Some(((key, lsn, val), desc)) = merge_iter.next_with_trace().await? {
            if cancel.is_cancelled() {
-                return Err(CompactionError::ShuttingDown);
-            }
-            keys_processed += 1;
-            if keys_processed % 1000 == 0 {
-                let should_yield = self
-                    .l0_compaction_trigger
-                    .notified()
-                    .now_or_never()
-                    .is_some();
-                if should_yield {
-                    tracing::info!("preempt gc-compaction in the main loop: too many L0 layers");
-                    return Ok(CompactionOutcome::YieldForL0);
-                }
+                return Err(anyhow!("cancelled")); // TODO: refactor to CompactionError and pass cancel error
            }
            if self.shard_identity.is_key_disposable(&key) {
                // If this shard does not need to store this key, simply skip it.
@@ -3095,9 +2960,7 @@ impl Timeline {
                                desc.lsn_range.clone(),
                                ctx,
                            )
-                            .await
-                            .context("failed to create delta layer writer")
-                            .map_err(CompactionError::Other)?,
+                            .await?,
                        );
                    }
                    rewriter.before.as_mut().unwrap()
@@ -3112,20 +2975,14 @@ impl Timeline {
                                desc.lsn_range.clone(),
                                ctx,
                            )
-                            .await
-                            .context("failed to create delta layer writer")
-                            .map_err(CompactionError::Other)?,
+                            .await?,
                        );
                    }
                    rewriter.after.as_mut().unwrap()
                } else {
                    unreachable!()
                };
-                rewriter
-                    .put_value(key, lsn, val, ctx)
-                    .await
-                    .context("failed to put value")
-                    .map_err(CompactionError::Other)?;
+                rewriter.put_value(key, lsn, val, ctx).await?;
                continue;
            }
            match val {
@@ -3148,13 +3005,9 @@ impl Timeline {
                        &job_desc.retain_lsns_below_horizon,
                        COMPACTION_DELTA_THRESHOLD,
                        get_ancestor_image(self, *last_key, ctx, has_data_below, lowest_retain_lsn)
-                            .await
-                            .context("failed to get ancestor image")
-                            .map_err(CompactionError::Other)?,
+                            .await?,
                    )
-                    .await
-                    .context("failed to generate key retention")
-                    .map_err(CompactionError::Other)?;
+                    .await?;
                retention
                    .pipe_to(
                        *last_key,
@@ -3163,9 +3016,7 @@ impl Timeline {
                        &mut stat,
                        ctx,
                    )
-                    .await
-                    .context("failed to pipe to delta layer writer")
-                    .map_err(CompactionError::Other)?;
+                    .await?;
                accumulated_values.clear();
                *last_key = key;
                accumulated_values.push((key, lsn, val));
@@ -3183,14 +3034,9 @@ impl Timeline {
                job_desc.gc_cutoff,
                &job_desc.retain_lsns_below_horizon,
                COMPACTION_DELTA_THRESHOLD,
-                get_ancestor_image(self, last_key, ctx, has_data_below, lowest_retain_lsn)
-                    .await
-                    .context("failed to get ancestor image")
-                    .map_err(CompactionError::Other)?,
+                get_ancestor_image(self, last_key, ctx, has_data_below, lowest_retain_lsn).await?,
            )
-            .await
-            .context("failed to generate key retention")
-            .map_err(CompactionError::Other)?;
+            .await?;
        retention
            .pipe_to(
                last_key,
@@ -3199,36 +3045,21 @@ impl Timeline {
                &mut stat,
                ctx,
            )
-            .await
-            .context("failed to pipe to delta layer writer")
-            .map_err(CompactionError::Other)?;
+            .await?;
        // end: move the above part to the loop body

-        let time_main_loop = timer.elapsed();
-        let timer = Instant::now();
-
        let mut rewrote_delta_layers = Vec::new();
        for (key, writers) in delta_layer_rewriters {
            if let Some(delta_writer_before) = writers.before {
                let (desc, path) = delta_writer_before
                    .finish(job_desc.compaction_key_range.start, ctx)
-                    .await
-                    .context("failed to finish delta layer writer")
-                    .map_err(CompactionError::Other)?;
-                let layer = Layer::finish_creating(self.conf, self, desc, &path)
-                    .context("failed to finish creating delta layer")
-                    .map_err(CompactionError::Other)?;
+                    .await?;
+                let layer = Layer::finish_creating(self.conf, self, desc, &path)?;
                rewrote_delta_layers.push(layer);
            }
            if let Some(delta_writer_after) = writers.after {
-                let (desc, path) = delta_writer_after
-                    .finish(key.key_range.end, ctx)
-                    .await
-                    .context("failed to finish delta layer writer")
-                    .map_err(CompactionError::Other)?;
-                let layer = Layer::finish_creating(self.conf, self, desc, &path)
-                    .context("failed to finish creating delta layer")
-                    .map_err(CompactionError::Other)?;
+                let (desc, path) = delta_writer_after.finish(key.key_range.end, ctx).await?;
+                let layer = Layer::finish_creating(self.conf, self, desc, &path)?;
                rewrote_delta_layers.push(layer);
            }
        }
@@ -3243,9 +3074,7 @@ impl Timeline {
                let end_key = job_desc.compaction_key_range.end;
                writer
                    .finish_with_discard_fn(self, ctx, end_key, discard)
-                    .await
-                    .context("failed to finish image layer writer")
-                    .map_err(CompactionError::Other)?
+                    .await?
            } else {
                drop(writer);
                Vec::new()
@@ -3257,9 +3086,7 @@ impl Timeline {
        let produced_delta_layers = if !dry_run {
            delta_layer_writer
                .finish_with_discard_fn(self, ctx, discard)
-                .await
-                .context("failed to finish delta layer writer")
-                .map_err(CompactionError::Other)?
+                .await?
        } else {
            drop(delta_layer_writer);
            Vec::new()
@@ -3271,13 +3098,6 @@ impl Timeline {
        let mut keep_layers = HashSet::new();
        let produced_delta_layers_len = produced_delta_layers.len();
        let produced_image_layers_len = produced_image_layers.len();
-
-        let layer_selection_by_key = job_desc
-            .selected_layers
-            .iter()
-            .map(|l| (l.layer_desc().key(), l.layer_desc().clone()))
-            .collect::<HashMap<_, _>>();
-
        for action in produced_delta_layers {
            match action {
                BatchWriterResult::Produced(layer) => {
@@ -3291,16 +3111,8 @@ impl Timeline {
                    if cfg!(debug_assertions) {
                        info!("discarded delta layer: {}", l);
                    }
-                    if let Some(layer_desc) = layer_selection_by_key.get(&l) {
-                        stat.discard_delta_layer(layer_desc.file_size());
-                    } else {
-                        tracing::warn!(
-                            "discarded delta layer not in layer_selection: {}, produced a layer outside of the compaction key range?",
-                            l
-                        );
-                        stat.discard_delta_layer(0);
-                    }
                    keep_layers.insert(l);
+                    stat.discard_delta_layer();
                }
            }
        }
@@ -3309,9 +3121,6 @@ impl Timeline {
                "produced rewritten delta layer: {}",
                layer.layer_desc().key()
            );
-            // For now, we include rewritten delta layer size in the "produce_delta_layer". We could
-            // make it a separate statistics in the future.
-            stat.produce_delta_layer(layer.layer_desc().file_size());
        }
        compact_to.extend(rewrote_delta_layers);
        for action in produced_image_layers {
@@ -3323,16 +3132,8 @@ impl Timeline {
                }
                BatchWriterResult::Discarded(l) => {
                    debug!("discarded image layer: {}", l);
-                    if let Some(layer_desc) = layer_selection_by_key.get(&l) {
-                        stat.discard_image_layer(layer_desc.file_size());
-                    } else {
-                        tracing::warn!(
-                            "discarded image layer not in layer_selection: {}, produced a layer outside of the compaction key range?",
-                            l
-                        );
-                        stat.discard_image_layer(0);
-                    }
                    keep_layers.insert(l);
+                    stat.discard_image_layer();
                }
            }
        }
@@ -3365,9 +3166,7 @@ impl Timeline {
                    &layer.layer_desc().key_range,
                    &job_desc.compaction_key_range,
                ) {
-                    return Err(CompactionError::Other(anyhow!(
-                        "violated constraint: image layer outside of compaction key range"
-                    )));
+                    bail!("violated constraint: image layer outside of compaction key range");
                }
                if !fully_contains(
                    &job_desc.compaction_key_range,
@@ -3380,25 +3179,13 @@ impl Timeline {

        layer_selection.retain(|x| !keep_layers.contains(&x.layer_desc().key()));

-        let time_final_phase = timer.elapsed();
-
-        stat.time_final_phase_secs = time_final_phase.as_secs_f64();
-        stat.time_main_loop_secs = time_main_loop.as_secs_f64();
-        stat.time_acquire_lock_secs = time_acquire_lock.as_secs_f64();
-        stat.time_download_layer_secs = time_download_layer.as_secs_f64();
-        stat.time_analyze_secs = time_analyze.as_secs_f64();
-        stat.time_total_secs = begin_timer.elapsed().as_secs_f64();
-        stat.finalize();
-
        info!(
            "gc-compaction statistics: {}",
-            serde_json::to_string(&stat)
-                .context("failed to serialize gc-compaction statistics")
-                .map_err(CompactionError::Other)?
+            serde_json::to_string(&stat)?
        );

        if dry_run {
-            return Ok(CompactionOutcome::Done);
+            return Ok(());
        }

        info!(
@@ -3433,10 +3220,10 @@ impl Timeline {
        // the writer, so potentially, we will need a function like `ImageLayerBatchWriter::get_all_pending_layer_keys` to get all the keys that are
        // in the writer before finalizing the persistent layers. Now we would leave some dangling layers on the disk if the check fails.
        if let Some(err) = check_valid_layermap(&final_layers) {
-            return Err(CompactionError::Other(anyhow!(
+            bail!(
                "gc-compaction layer map check failed after compaction because {}, compaction result not applied to the layer map due to potential data loss",
                err
-            )));
+            );
        }

        // Between the sanity check and this compaction update, there could be new layers being flushed, but it should be fine because we only
@@ -3488,9 +3275,7 @@ impl Timeline {
        // find_gc_cutoffs will try accessing things below the cutoff. TODO: ideally, this should
        // be batched into `schedule_compaction_update`.
        let disk_consistent_lsn = self.disk_consistent_lsn.load();
-        self.schedule_uploads(disk_consistent_lsn, None)
-            .context("failed to schedule uploads")
-            .map_err(CompactionError::Other)?;
+        self.schedule_uploads(disk_consistent_lsn, None)?;
        // If a layer gets rewritten throughout gc-compaction, we need to keep that layer only in `compact_to` instead
        // of `compact_from`.
        let compact_from = {
@@ -3517,7 +3302,7 @@ impl Timeline {

        drop(gc_lock);

-        Ok(CompactionOutcome::Done)
+        Ok(())
    }
 }

--- a/pageserver/src/tenant/timeline/heatmap_layers_downloader.rs
+++ b/pageserver/src/tenant/timeline/heatmap_layers_downloader.rs
@@ -61,11 +61,11 @@ impl HeatmapLayersDownloader {

                tracing::info!(
                    resident_size=%timeline.resident_physical_size(),
-                    heatmap_layers=%heatmap.all_layers().count(),
+                    heatmap_layers=%heatmap.layers.len(),
                    "Starting heatmap layers download"
                );

-                let stream = futures::stream::iter(heatmap.all_layers().cloned().filter_map(
+                let stream = futures::stream::iter(heatmap.layers.into_iter().filter_map(
                    |layer| {
                        let ctx = ctx.attached_child();
                        let tl = timeline.clone();
--- a/pgxn/neon/file_cache.c
+++ b/pgxn/neon/file_cache.c
@@ -1369,10 +1369,6 @@ neon_get_lfc_stats(PG_FUNCTION_ARGS)
 			if (lfc_ctl)
 				value = lfc_ctl->limit;
 			break;
-		case 8:
-			key = "file_cache_chunk_size_pages";
-			value = BLOCKS_PER_CHUNK;
-			break;
 		default:
 			SRF_RETURN_DONE(funcctx);
 	}
--- a/pgxn/neon/pagestore_smgr.c
+++ b/pgxn/neon/pagestore_smgr.c
@@ -1026,19 +1026,6 @@ prefetch_lookupv(NRelFileInfo rinfo, ForkNumber forknum, BlockNumber blocknum, n
 			if (!neon_prefetch_response_usable(&lsns[i], slot))
 				continue;

-			/*
-			 * Ignore errors
-			 */
-			if (slot->response->tag != T_NeonGetPageResponse)
-			{
-				if (slot->response->tag != T_NeonErrorResponse)
-				{
-					NEON_PANIC_CONNECTION_STATE(slot->shard_no, PANIC,
-											"Expected GetPage (0x%02x) or Error (0x%02x) response to GetPageRequest, but got 0x%02x",
-											T_NeonGetPageResponse, T_NeonErrorResponse, slot->response->tag);
-				}
-				continue;
-			}
 			memcpy(buffers[i], ((NeonGetPageResponse*)slot->response)->page, BLCKSZ);
 			prefetch_set_unused(ring_index);
 			BITMAP_SET(mask, i);
--- a/poetry.lock
+++ b/poetry.lock
@@ -1414,14 +1414,14 @@ files = [

 [[package]]
 name = "jinja2"
-version = "3.1.6"
+version = "3.1.5"
 description = "A very fast and expressive template engine."
 optional = false
 python-versions = ">=3.7"
 groups = ["main"]
 files = [
-    {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"},
-    {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"},
+    {file = "jinja2-3.1.5-py3-none-any.whl", hash = "sha256:aba0f4dc9ed8013c424088f68a5c226f7d6097ed89b246d7749c2ec4175c6adb"},
+    {file = "jinja2-3.1.5.tar.gz", hash = "sha256:8fefff8dc3034e27bb80d67c671eb8a9bc424c0ef4c0826edbff304cceff43bb"},
 ]

 [package.dependencies]
@@ -3820,4 +3820,4 @@ cffi = ["cffi (>=1.11)"]
 [metadata]
 lock-version = "2.1"
 python-versions = "^3.11"
-content-hash = "010ffce959bb256880ab5a267048c182e4612b3151f9a94e3bf5d3a7807962fe"
+content-hash = "9711c5479c867fa614ce3d352f1bbc63dba1cb2376d347f96fbeda6f512ee308"
--- a/proxy/Cargo.toml
+++ b/proxy/Cargo.toml
@@ -53,7 +53,7 @@ measured = { workspace = true, features = ["lasso"] }
 metrics.workspace = true
 once_cell.workspace = true
 opentelemetry = { workspace = true, features = ["trace"] }
-papaya = "0.2.0"
+papaya = "0.1.8"
 parking_lot.workspace = true
 parquet.workspace = true
 parquet_derive.workspace = true
--- a/proxy/src/logging.rs
+++ b/proxy/src/logging.rs
@@ -1,11 +1,9 @@
 use std::cell::{Cell, RefCell};
 use std::collections::HashMap;
 use std::hash::BuildHasher;
-use std::sync::atomic::{AtomicU32, Ordering};
-use std::{array, env, fmt, io};
+use std::{env, io};

 use chrono::{DateTime, Utc};
-use indexmap::IndexSet;
 use opentelemetry::trace::TraceContextExt;
 use scopeguard::defer;
 use serde::ser::{SerializeMap, Serializer};
@@ -19,7 +17,6 @@ use tracing_subscriber::fmt::{FormatEvent, FormatFields};
 use tracing_subscriber::layer::{Context, Layer};
 use tracing_subscriber::prelude::*;
 use tracing_subscriber::registry::{LookupSpan, SpanRef};
-use try_lock::TryLock;

 /// Initialize logging and OpenTelemetry tracing and exporter.
 ///
@@ -49,13 +46,13 @@ pub async fn init() -> anyhow::Result<LoggingGuard> {
    let otlp_layer = tracing_utils::init_tracing("proxy").await;

    let json_log_layer = if logfmt == LogFormat::Json {
-        Some(JsonLoggingLayer::new(
-            RealClock,
-            StderrWriter {
+        Some(JsonLoggingLayer {
+            clock: RealClock,
+            skipped_field_indices: papaya::HashMap::default(),
+            writer: StderrWriter {
                stderr: std::io::stderr(),
            },
-            ["request_id", "session_id", "conn_id"],
-        ))
+        })
    } else {
        None
    };
@@ -194,39 +191,13 @@ thread_local! {
 }

 /// Implements tracing layer to handle events specific to logging.
-struct JsonLoggingLayer<C: Clock, W: MakeWriter, const F: usize> {
+struct JsonLoggingLayer<C: Clock, W: MakeWriter> {
    clock: C,
    skipped_field_indices: papaya::HashMap<callsite::Identifier, SkippedFieldIndices>,
-    callsite_ids: papaya::HashMap<callsite::Identifier, CallsiteId>,
    writer: W,
-    // We use a const generic and arrays to bypass one heap allocation.
-    extract_fields: IndexSet<&'static str>,
-    _marker: std::marker::PhantomData<[&'static str; F]>,
 }

-impl<C: Clock, W: MakeWriter, const F: usize> JsonLoggingLayer<C, W, F> {
-    fn new(clock: C, writer: W, extract_fields: [&'static str; F]) -> Self {
-        JsonLoggingLayer {
-            clock,
-            skipped_field_indices: papaya::HashMap::default(),
-            callsite_ids: papaya::HashMap::default(),
-            writer,
-            extract_fields: IndexSet::from_iter(extract_fields),
-            _marker: std::marker::PhantomData,
-        }
-    }
-
-    #[inline]
-    fn callsite_id(&self, cs: callsite::Identifier) -> CallsiteId {
-        *self
-            .callsite_ids
-            .pin()
-            .get_or_insert_with(cs, CallsiteId::next)
-    }
-}
-
-impl<S, C: Clock + 'static, W: MakeWriter + 'static, const F: usize> Layer<S>
-    for JsonLoggingLayer<C, W, F>
+impl<S, C: Clock + 'static, W: MakeWriter + 'static> Layer<S> for JsonLoggingLayer<C, W>
 where
    S: Subscriber + for<'a> LookupSpan<'a>,
 {
@@ -240,14 +211,7 @@ where
        let res: io::Result<()> = REENTRANCY_GUARD.with(move |entered| {
            if entered.get() {
                let mut formatter = EventFormatter::new();
-                formatter.format::<S, F>(
-                    now,
-                    event,
-                    &ctx,
-                    &self.skipped_field_indices,
-                    &self.callsite_ids,
-                    &self.extract_fields,
-                )?;
+                formatter.format(now, event, &ctx, &self.skipped_field_indices)?;
                self.writer.make_writer().write_all(formatter.buffer())
            } else {
                entered.set(true);
@@ -255,14 +219,7 @@ where

                EVENT_FORMATTER.with_borrow_mut(move |formatter| {
                    formatter.reset();
-                    formatter.format::<S, F>(
-                        now,
-                        event,
-                        &ctx,
-                        &self.skipped_field_indices,
-                        &self.callsite_ids,
-                        &self.extract_fields,
-                    )?;
+                    formatter.format(now, event, &ctx, &self.skipped_field_indices)?;
                    self.writer.make_writer().write_all(formatter.buffer())
                })
            }
@@ -289,13 +246,10 @@ where
        let span = ctx.span(id).expect("span must exist");
        let fields = SpanFields::default();
        fields.record_fields(attrs);
-
        // This could deadlock when there's a panic somewhere in the tracing
        // event handling and a read or write guard is still held. This includes
        // the OTel subscriber.
-        let mut exts = span.extensions_mut();
-
-        exts.insert(fields);
+        span.extensions_mut().insert(fields);
    }

    fn on_record(&self, id: &span::Id, values: &span::Record<'_>, ctx: Context<'_, S>) {
@@ -311,7 +265,6 @@ where
    /// wins.
    fn register_callsite(&self, metadata: &'static Metadata<'static>) -> Interest {
        if !metadata.is_event() {
-            self.callsite_id(metadata.callsite());
            // Must not be never because we wouldn't get trace and span data.
            return Interest::always();
        }
@@ -344,26 +297,6 @@ where
    }
 }

-#[derive(Copy, Clone, Debug, Default)]
-#[repr(transparent)]
-struct CallsiteId(u32);
-
-impl CallsiteId {
-    #[inline]
-    fn next() -> Self {
-        // Start at 1 to reserve 0 for default.
-        static COUNTER: AtomicU32 = AtomicU32::new(1);
-        CallsiteId(COUNTER.fetch_add(1, Ordering::Relaxed))
-    }
-}
-
-impl fmt::Display for CallsiteId {
-    #[inline]
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
 /// Stores span field values recorded during the spans lifetime.
 #[derive(Default)]
 struct SpanFields {
@@ -515,14 +448,12 @@ impl EventFormatter {
        self.logline_buffer.clear();
    }

-    fn format<S, const F: usize>(
+    fn format<S>(
        &mut self,
        now: DateTime<Utc>,
        event: &Event<'_>,
        ctx: &Context<'_, S>,
        skipped_field_indices: &papaya::HashMap<callsite::Identifier, SkippedFieldIndices>,
-        callsite_ids: &papaya::HashMap<callsite::Identifier, CallsiteId>,
-        extract_fields: &IndexSet<&'static str>,
    ) -> io::Result<()>
    where
        S: Subscriber + for<'a> LookupSpan<'a>,
@@ -554,7 +485,6 @@ impl EventFormatter {
            event.record(&mut message_extractor);
            let mut serializer = message_extractor.into_serializer()?;

-            // Direct message fields.
            let mut fields_present = FieldsPresent(false, skipped_field_indices);
            event.record(&mut fields_present);
            if fields_present.0 {
@@ -564,16 +494,7 @@ impl EventFormatter {
                )?;
            }

-            let spans = SerializableSpans {
-                ctx,
-                callsite_ids,
-                extract: ExtractedSpanFields::<'_, F>::new(extract_fields),
-            };
-            serializer.serialize_entry("spans", &spans)?;
-
-            // TODO: thread-local cache?
            let pid = std::process::id();
-            // Skip adding pid 1 to reduce noise for services running in containers.
            if pid != 1 {
                serializer.serialize_entry("process_id", &pid)?;
            }
@@ -593,7 +514,6 @@ impl EventFormatter {

            serializer.serialize_entry("target", meta.target())?;

-            // Skip adding module if it's the same as target.
            if let Some(module) = meta.module_path() {
                if module != meta.target() {
                    serializer.serialize_entry("module", module)?;
@@ -620,10 +540,7 @@ impl EventFormatter {
                }
            }

-            if spans.extract.has_values() {
-                // TODO: add fields from event, too?
-                serializer.serialize_entry("extract", &spans.extract)?;
-            }
+            serializer.serialize_entry("spans", &SerializableSpanStack(ctx))?;

            serializer.end()
        };
@@ -901,20 +818,15 @@ impl<S: serde::ser::SerializeMap> tracing::field::Visit for MessageFieldSkipper<
    }
 }

-/// Serializes the span stack from root to leaf (parent of event) as object
-/// with the span names as keys. To prevent collision we append a numberic value
-/// to the name. Also, collects any span fields we're interested in. Last one
-/// wins.
-struct SerializableSpans<'a, 'ctx, Span, const F: usize>
+/// Serializes the span stack from root to leaf (parent of event) enumerated
+/// inside an object where the keys are just the number padded with zeroes
+/// to retain sorting order.
+// The object is necessary because Loki cannot flatten arrays.
+struct SerializableSpanStack<'a, 'b, Span>(&'b Context<'a, Span>)
 where
-    Span: Subscriber + for<'lookup> LookupSpan<'lookup>,
-{
-    ctx: &'a Context<'ctx, Span>,
-    callsite_ids: &'a papaya::HashMap<callsite::Identifier, CallsiteId>,
-    extract: ExtractedSpanFields<'a, F>,
-}
+    Span: Subscriber + for<'lookup> LookupSpan<'lookup>;

-impl<Span, const F: usize> serde::ser::Serialize for SerializableSpans<'_, '_, Span, F>
+impl<Span> serde::ser::Serialize for SerializableSpanStack<'_, '_, Span>
 where
    Span: Subscriber + for<'lookup> LookupSpan<'lookup>,
 {
@@ -924,24 +836,9 @@ where
    {
        let mut serializer = serializer.serialize_map(None)?;

-        if let Some(leaf_span) = self.ctx.lookup_current() {
-            for span in leaf_span.scope().from_root() {
-                // Append a numeric callsite ID to the span name to keep the name unique
-                // in the JSON object.
-                let cid = self
-                    .callsite_ids
-                    .pin()
-                    .get(&span.metadata().callsite())
-                    .copied()
-                    .unwrap_or_default();
-
-                // Loki turns the # into an underscore during field name concatenation.
-                serializer.serialize_key(&format_args!("{}#{}", span.metadata().name(), &cid))?;
-
-                serializer.serialize_value(&SerializableSpanFields {
-                    span: &span,
-                    extract: &self.extract,
-                })?;
+        if let Some(leaf_span) = self.0.lookup_current() {
+            for (i, span) in leaf_span.scope().from_root().enumerate() {
+                serializer.serialize_entry(&format_args!("{i:02}"), &SerializableSpan(&span))?;
            }
        }

@@ -949,79 +846,28 @@ where
    }
 }

-/// Serializes the span fields as object.
-struct SerializableSpanFields<'a, 'span, Span, const F: usize>
+/// Serializes a single span. Include the span ID, name and its fields as
+/// recorded up to this point.
+struct SerializableSpan<'a, 'b, Span>(&'b SpanRef<'a, Span>)
 where
-    Span: for<'lookup> LookupSpan<'lookup>,
-{
-    span: &'a SpanRef<'span, Span>,
-    extract: &'a ExtractedSpanFields<'a, F>,
-}
+    Span: for<'lookup> LookupSpan<'lookup>;

-impl<Span, const F: usize> serde::ser::Serialize for SerializableSpanFields<'_, '_, Span, F>
+impl<Span> serde::ser::Serialize for SerializableSpan<'_, '_, Span>
 where
    Span: for<'lookup> LookupSpan<'lookup>,
 {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    fn serialize<Ser>(&self, serializer: Ser) -> Result<Ser::Ok, Ser::Error>
    where
-        S: serde::ser::Serializer,
+        Ser: serde::ser::Serializer,
    {
        let mut serializer = serializer.serialize_map(None)?;
+        // TODO: the span ID is probably only useful for debugging tracing.
+        serializer.serialize_entry("span_id", &format_args!("{:016x}", self.0.id().into_u64()))?;
+        serializer.serialize_entry("span_name", self.0.metadata().name())?;

-        let ext = self.span.extensions();
+        let ext = self.0.extensions();
        if let Some(data) = ext.get::<SpanFields>() {
-            for (name, value) in &data.fields.pin() {
-                serializer.serialize_entry(name, value)?;
-                // TODO: replace clone with reference, if possible.
-                self.extract.set(name, value.clone());
-            }
-        }
-
-        serializer.end()
-    }
-}
-
-struct ExtractedSpanFields<'a, const F: usize> {
-    names: &'a IndexSet<&'static str>,
-    // TODO: replace TryLock with something local thread and interior mutability.
-    //       serde API doesn't let us use `mut`.
-    values: TryLock<([Option<serde_json::Value>; F], bool)>,
-}
-
-impl<'a, const F: usize> ExtractedSpanFields<'a, F> {
-    fn new(names: &'a IndexSet<&'static str>) -> Self {
-        ExtractedSpanFields {
-            names,
-            values: TryLock::new((array::from_fn(|_| Option::default()), false)),
-        }
-    }
-
-    #[inline]
-    fn set(&self, name: &'static str, value: serde_json::Value) {
-        if let Some((index, _)) = self.names.get_full(name) {
-            let mut fields = self.values.try_lock().expect("thread-local use");
-            fields.0[index] = Some(value);
-            fields.1 = true;
-        }
-    }
-
-    #[inline]
-    fn has_values(&self) -> bool {
-        self.values.try_lock().expect("thread-local use").1
-    }
-}
-
-impl<const F: usize> serde::ser::Serialize for ExtractedSpanFields<'_, F> {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: serde::ser::Serializer,
-    {
-        let mut serializer = serializer.serialize_map(None)?;
-
-        let values = self.values.try_lock().expect("thread-local use");
-        for (i, value) in values.0.iter().enumerate() {
-            if let Some(value) = value {
-                let key = self.names[i];
+            for (key, value) in &data.fields.pin() {
                serializer.serialize_entry(key, value)?;
            }
        }
@@ -1033,7 +879,6 @@ impl<const F: usize> serde::ser::Serialize for ExtractedSpanFields<'_, F> {
 #[cfg(test)]
 #[allow(clippy::unwrap_used)]
 mod tests {
-    use std::marker::PhantomData;
    use std::sync::{Arc, Mutex, MutexGuard};

    use assert_json_diff::assert_json_eq;
@@ -1082,17 +927,14 @@ mod tests {
        let log_layer = JsonLoggingLayer {
            clock: clock.clone(),
            skipped_field_indices: papaya::HashMap::default(),
-            callsite_ids: papaya::HashMap::default(),
            writer: buffer.clone(),
-            extract_fields: IndexSet::from_iter(["x"]),
-            _marker: PhantomData::<[&'static str; 1]>,
        };

        let registry = tracing_subscriber::Registry::default().with(log_layer);

        tracing::subscriber::with_default(registry, || {
-            info_span!("some_span", x = 24).in_scope(|| {
-                info_span!("some_span", x = 40, x = 41, x = 42).in_scope(|| {
+            info_span!("span1", x = 40, x = 41, x = 42).in_scope(|| {
+                info_span!("span2").in_scope(|| {
                    tracing::error!(
                        a = 1,
                        a = 2,
@@ -1118,16 +960,16 @@ mod tests {
                    "a": 3,
                },
                "spans": {
-                    "some_span#1":{
-                        "x": 24,
-                    },
-                    "some_span#2": {
+                    "00":{
+                        "span_id": "0000000000000001",
+                        "span_name": "span1",
                        "x": 42,
+                    },
+                    "01": {
+                        "span_id": "0000000000000002",
+                        "span_name": "span2",
                    }
                },
-                "extract": {
-                    "x": 42,
-                },
                "src": actual.as_object().unwrap().get("src").unwrap().as_str().unwrap(),
                "target": "proxy::logging::tests",
                "process_id": actual.as_object().unwrap().get("process_id").unwrap().as_number().unwrap(),
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -13,7 +13,7 @@ requests = "^2.32.3"
 pytest-xdist = "^3.3.1"
 asyncpg = "^0.30.0"
 aiopg = "^1.4.0"
-Jinja2 = "^3.1.6"
+Jinja2 = "^3.1.5"
 types-requests = "^2.31.0.0"
 types-psycopg2 = "^2.9.21.20241019"
 boto3 = "^1.34.11"
--- a/safekeeper/src/send_interpreted_wal.rs
+++ b/safekeeper/src/send_interpreted_wal.rs
@@ -430,10 +430,7 @@ impl InterpretedWalReader {
                        .with_context(|| "Failed to interpret WAL")?;

                        for (shard, record) in interpreted {
-                            // Shard zero needs to track the start LSN of the latest record
-                            // in adition to the LSN of the next record to ingest. The former
-                            // is included in basebackup persisted by the compute in WAL.
-                            if !shard.is_shard_zero() && record.is_empty() {
+                            if record.is_empty() {
                                continue;
                            }

@@ -743,7 +740,7 @@ mod tests {
            .unwrap();

        let resident_tli = tli.wal_residence_guard().await.unwrap();
-        let end_watch = Env::write_wal(tli, start_lsn, SIZE, MSG_COUNT, c"neon-file:", None)
+        let end_watch = Env::write_wal(tli, start_lsn, SIZE, MSG_COUNT, None)
            .await
            .unwrap();
        let end_pos = end_watch.get();
@@ -886,16 +883,10 @@ mod tests {

        let resident_tli = tli.wal_residence_guard().await.unwrap();
        let mut next_record_lsns = Vec::default();
-        let end_watch = Env::write_wal(
-            tli,
-            start_lsn,
-            SIZE,
-            MSG_COUNT,
-            c"neon-file:",
-            Some(&mut next_record_lsns),
-        )
-        .await
-        .unwrap();
+        let end_watch =
+            Env::write_wal(tli, start_lsn, SIZE, MSG_COUNT, Some(&mut next_record_lsns))
+                .await
+                .unwrap();
        let end_pos = end_watch.get();

        let streaming_wal_reader = StreamingWalReader::new(
@@ -1036,16 +1027,10 @@ mod tests {
            .unwrap();

        let resident_tli = tli.wal_residence_guard().await.unwrap();
-        let end_watch = Env::write_wal(
-            tli,
-            start_lsn,
-            SIZE,
-            MSG_COUNT,
-            c"neon-file:",
-            Some(&mut next_record_lsns),
-        )
-        .await
-        .unwrap();
+        let end_watch =
+            Env::write_wal(tli, start_lsn, SIZE, MSG_COUNT, Some(&mut next_record_lsns))
+                .await
+                .unwrap();

        assert!(next_record_lsns.len() > 3);
        let shard_0_start_lsn = next_record_lsns[3];
@@ -1139,88 +1124,4 @@ mod tests {
            }
        }
    }
-
-    #[tokio::test]
-    async fn test_shard_zero_does_not_skip_empty_records() {
-        let _ = env_logger::builder().is_test(true).try_init();
-
-        const SIZE: usize = 8 * 1024;
-        const MSG_COUNT: usize = 10;
-        const PG_VERSION: u32 = 17;
-
-        let start_lsn = Lsn::from_str("0/149FD18").unwrap();
-        let env = Env::new(true).unwrap();
-        let tli = env
-            .make_timeline(NodeId(1), TenantTimelineId::generate(), start_lsn)
-            .await
-            .unwrap();
-
-        let resident_tli = tli.wal_residence_guard().await.unwrap();
-        let mut next_record_lsns = Vec::new();
-        let end_watch = Env::write_wal(
-            tli,
-            start_lsn,
-            SIZE,
-            MSG_COUNT,
-            // This is a logical message prefix that is not persisted to key value storage.
-            // We use it in order to validate that shard zero receives emtpy interpreted records.
-            c"test:",
-            Some(&mut next_record_lsns),
-        )
-        .await
-        .unwrap();
-        let end_pos = end_watch.get();
-
-        let streaming_wal_reader = StreamingWalReader::new(
-            resident_tli,
-            None,
-            start_lsn,
-            end_pos,
-            end_watch,
-            MAX_SEND_SIZE,
-        );
-
-        let shard = ShardIdentity::unsharded();
-        let (records_tx, mut records_rx) = tokio::sync::mpsc::channel::<Batch>(MSG_COUNT * 2);
-
-        let handle = InterpretedWalReader::spawn(
-            streaming_wal_reader,
-            start_lsn,
-            records_tx,
-            shard,
-            PG_VERSION,
-            &Some("pageserver".to_string()),
-        );
-
-        let mut interpreted_records = Vec::new();
-        while let Some(batch) = records_rx.recv().await {
-            interpreted_records.push(batch.records);
-            if batch.wal_end_lsn == batch.available_wal_end_lsn {
-                break;
-            }
-        }
-
-        let received_next_record_lsns = interpreted_records
-            .into_iter()
-            .flat_map(|b| b.records)
-            .map(|rec| rec.next_record_lsn)
-            .collect::<Vec<_>>();
-
-        // By default this also includes the start LSN. Trim it since it shouldn't be received.
-        let next_record_lsns = next_record_lsns.into_iter().skip(1).collect::<Vec<_>>();
-
-        assert_eq!(received_next_record_lsns, next_record_lsns);
-
-        handle.abort();
-        let mut done = false;
-        for _ in 0..5 {
-            if handle.current_position().is_none() {
-                done = true;
-                break;
-            }
-            tokio::time::sleep(Duration::from_millis(1)).await;
-        }
-
-        assert!(done);
-    }
 }
--- a/safekeeper/src/test_utils.rs
+++ b/safekeeper/src/test_utils.rs
@@ -1,4 +1,3 @@
-use std::ffi::CStr;
 use std::sync::Arc;

 use camino_tempfile::Utf8TempDir;
@@ -125,7 +124,6 @@ impl Env {
        start_lsn: Lsn,
        msg_size: usize,
        msg_count: usize,
-        prefix: &CStr,
        mut next_record_lsns: Option<&mut Vec<Lsn>>,
    ) -> anyhow::Result<EndWatch> {
        let (msg_tx, msg_rx) = tokio::sync::mpsc::channel(receive_wal::MSG_QUEUE_SIZE);
@@ -135,6 +133,7 @@ impl Env {

        WalAcceptor::spawn(tli.wal_residence_guard().await?, msg_rx, reply_tx, Some(0));

+        let prefix = c"neon-file:";
        let prefixlen = prefix.to_bytes_with_nul().len();
        assert!(msg_size >= prefixlen);
        let message = vec![0; msg_size - prefixlen];
--- a/safekeeper/src/wal_reader_stream.rs
+++ b/safekeeper/src/wal_reader_stream.rs
@@ -246,7 +246,7 @@ mod tests {
            .unwrap();

        let resident_tli = tli.wal_residence_guard().await.unwrap();
-        let end_watch = Env::write_wal(tli, start_lsn, SIZE, MSG_COUNT, c"neon-file:", None)
+        let end_watch = Env::write_wal(tli, start_lsn, SIZE, MSG_COUNT, None)
            .await
            .unwrap();
        let end_pos = end_watch.get();
--- a/storage_controller/src/main.rs
+++ b/storage_controller/src/main.rs
@@ -143,6 +143,10 @@ struct Cli {
    // Flag to use https for requests to pageserver API.
    #[arg(long, default_value = "false")]
    use_https_pageserver_api: bool,
+
+    /// Whether to load safekeeprs from the database and heartbeat them
+    #[arg(long, default_value = "false")]
+    load_safekeepers: bool,
 }

 enum StrictMode {
@@ -356,6 +360,7 @@ async fn async_main() -> anyhow::Result<()> {
        start_as_candidate: args.start_as_candidate,
        http_service_port: args.listen.port() as i32,
        use_https_pageserver_api: args.use_https_pageserver_api,
+        load_safekeepers: args.load_safekeepers,
    };

    // Validate that we can connect to the database
--- a/storage_controller/src/service.rs
+++ b/storage_controller/src/service.rs
@@ -394,6 +394,8 @@ pub struct Config {
    pub long_reconcile_threshold: Duration,

    pub use_https_pageserver_api: bool,
+
+    pub load_safekeepers: bool,
 }

 impl From<DatabaseError> for ApiError {
@@ -1410,15 +1412,20 @@ impl Service {
            .set(nodes.len() as i64);

        tracing::info!("Loading safekeepers from database...");
-        let safekeepers = persistence
-            .list_safekeepers()
-            .await?
-            .into_iter()
-            .map(|skp| Safekeeper::from_persistence(skp, CancellationToken::new()))
-            .collect::<Vec<_>>();
+        let safekeepers = if config.load_safekeepers {
+            persistence
+                .list_safekeepers()
+                .await?
+                .into_iter()
+                .map(|skp| Safekeeper::from_persistence(skp, CancellationToken::new()))
+                .collect::<Vec<_>>()
+        } else {
+            tracing::info!("Skipping safekeeper loading");
+            Default::default()
+        };
+
        let safekeepers: HashMap<NodeId, Safekeeper> =
            safekeepers.into_iter().map(|n| (n.get_id(), n)).collect();
-        tracing::info!("Loaded {} safekeepers from database.", safekeepers.len());

        tracing::info!("Loading shards from database...");
        let mut tenant_shard_persistence = persistence.load_active_tenant_shards().await?;
@@ -8059,7 +8066,8 @@ impl Service {
    ) -> Result<(), DatabaseError> {
        let node_id = NodeId(record.id as u64);
        self.persistence.safekeeper_upsert(record.clone()).await?;
-        {
+
+        if self.config.load_safekeepers {
            let mut locked = self.inner.write().unwrap();
            let mut safekeepers = (*locked.safekeepers).clone();
            match safekeepers.entry(node_id) {
@@ -8091,7 +8099,7 @@ impl Service {
            .await?;
        let node_id = NodeId(id as u64);
        // After the change has been persisted successfully, update the in-memory state
-        {
+        if self.config.load_safekeepers {
            let mut locked = self.inner.write().unwrap();
            let mut safekeepers = (*locked.safekeepers).clone();
            let sk = safekeepers
--- a/test_runner/fixtures/neon_fixtures.py
+++ b/test_runner/fixtures/neon_fixtures.py
@@ -1133,6 +1133,13 @@ class NeonEnv:
        if self.storage_controller_config is not None:
            cfg["storage_controller"] = self.storage_controller_config

+        # Disable new storcon flag in compat tests
+        if config.test_may_use_compatibility_snapshot_binaries:
+            if "storage_controller" in cfg:
+                cfg["storage_controller"]["load_safekeepers"] = False
+            else:
+                cfg["storage_controller"] = {"load_safekeepers": False}
+
        # Create config for pageserver
        http_auth_type = "NeonJWT" if config.auth_enabled else "Trust"
        pg_auth_type = "NeonJWT" if config.auth_enabled else "Trust"
@@ -1162,8 +1169,6 @@ class NeonEnv:
                # Disable pageserver disk syncs in tests: when running tests concurrently, this avoids
                # the pageserver taking a long time to start up due to syncfs flushing other tests' data
                "no_sync": True,
-                # Look for gaps in WAL received from safekeepeers
-                "validate_wal_contiguity": True,
            }

            # Batching (https://github.com/neondatabase/neon/issues/9377):
@@ -1174,6 +1179,14 @@ class NeonEnv:
                "max_batch_size": 32,
            }

+            if config.test_may_use_compatibility_snapshot_binaries:
+                log.info(
+                    "Skipping WAL contiguity validation to avoid forward-compatibility related test failures"
+                )
+            else:
+                # Look for gaps in WAL received from safekeepeers
+                ps_cfg["validate_wal_contiguity"] = True
+
            get_vectored_concurrent_io = self.pageserver_get_vectored_concurrent_io
            if get_vectored_concurrent_io is not None:
                ps_cfg["get_vectored_concurrent_io"] = {
@@ -1188,9 +1201,6 @@ class NeonEnv:
                    config.pageserver_default_tenant_config_compaction_algorithm
                )

-            tenant_config = ps_cfg.setdefault("tenant_config", {})
-            tenant_config["rel_size_v2_enabled"] = True  # Enable relsize_v2 by default in tests
-
            if self.pageserver_remote_storage is not None:
                ps_cfg["remote_storage"] = remote_storage_to_toml_dict(
                    self.pageserver_remote_storage
--- a/test_runner/fixtures/pageserver/http.py
+++ b/test_runner/fixtures/pageserver/http.py
@@ -375,19 +375,6 @@ class PageserverHttpClient(requests.Session, MetricsGetter):
        res = self.post(f"http://localhost:{self.port}/v1/tenant/{tenant_id}/reset", params=params)
        self.verbose_error(res)

-    def timeline_patch_index_part(
-        self,
-        tenant_id: TenantId | TenantShardId,
-        timeline_id: TimelineId,
-        data: dict[str, Any],
-    ):
-        res = self.post(
-            f"http://localhost:{self.port}/v1/tenant/{tenant_id}/timeline/{timeline_id}/patch_index_part",
-            json=data,
-        )
-        self.verbose_error(res)
-        return res.json()
-
    def tenant_location_conf(
        self,
        tenant_id: TenantId | TenantShardId,
--- a/test_runner/fixtures/utils.py
+++ b/test_runner/fixtures/utils.py
@@ -337,8 +337,6 @@ def allure_add_grafana_link(host: str, timeline_id: TimelineId, start_ms: int, e
    """
    # We expect host to be in format like ep-holy-mouse-w2u462gi.us-east-2.aws.neon.build
    endpoint_id, region_id, _ = host.split(".", 2)
-    # Remove "-pooler" suffix if present
-    endpoint_id = endpoint_id.removesuffix("-pooler")

    params = {
        "orgId": 1,
--- a/test_runner/regress/test_compute_catalog.py
+++ b/test_runner/regress/test_compute_catalog.py
@@ -5,59 +5,34 @@ import logging
 import requests
 from fixtures.neon_fixtures import NeonEnv, logical_replication_sync

-TEST_ROLE_NAMES = [
-    {"name": "neondb_owner"},
-    {"name": "role with spaces"},
-    {"name": "role with%20spaces "},
-    {"name": "role with whitespaces	"},
-    {"name": "injective role with spaces'; SELECT pg_sleep(1000);"},
-    {"name": "role with #pound-sign and &ampersands=true"},
-    {"name": "role with emoji 🌍"},
-    {"name": "role \";with ';injections $$ $x$ $ %I !/\\&#@"},
-    {"name": '"role in double quotes"'},
-    {"name": "'role in single quotes'"},
-]
-
 TEST_DB_NAMES = [
    {
        "name": "neondb",
-        "owner": "neondb_owner",
+        "owner": "cloud_admin",
    },
    {
        "name": "db with spaces",
-        "owner": "role with spaces",
+        "owner": "cloud_admin",
    },
    {
        "name": "db with%20spaces ",
-        "owner": "role with%20spaces ",
+        "owner": "cloud_admin",
    },
    {
        "name": "db with whitespaces	",
-        "owner": "role with whitespaces	",
+        "owner": "cloud_admin",
    },
    {
-        "name": "injective db with spaces'; SELECT pg_sleep(1000);",
-        "owner": "injective role with spaces'; SELECT pg_sleep(1000);",
+        "name": "injective db with spaces'; SELECT pg_sleep(10);",
+        "owner": "cloud_admin",
    },
    {
        "name": "db with #pound-sign and &ampersands=true",
-        "owner": "role with #pound-sign and &ampersands=true",
+        "owner": "cloud_admin",
    },
    {
        "name": "db with emoji 🌍",
-        "owner": "role with emoji 🌍",
-    },
-    {
-        "name": "db \";with ';injections $$ $x$ $ %I !/\\&#@",
-        "owner": "role \";with ';injections $$ $x$ $ %I !/\\&#@",
-    },
-    {
-        "name": '"db in double quotes"',
-        "owner": '"role in double quotes"',
-    },
-    {
-        "name": "'db in single quotes'",
-        "owner": "'role in single quotes'",
+        "owner": "cloud_admin",
    },
 ]

@@ -77,7 +52,6 @@ def test_compute_catalog(neon_simple_env: NeonEnv):
        **{
            "skip_pg_catalog_updates": False,
            "cluster": {
-                "roles": TEST_ROLE_NAMES,
                "databases": TEST_DB_NAMES,
            },
        }
@@ -125,10 +99,10 @@ def test_compute_catalog(neon_simple_env: NeonEnv):
        ), f"Expected 404 status code, but got {e.response.status_code}"


-def test_compute_create_drop_dbs_and_roles(neon_simple_env: NeonEnv):
+def test_compute_create_databases(neon_simple_env: NeonEnv):
    """
-    Test that compute_ctl can create and work with databases and roles
-    with special characters (whitespaces, %, tabs, etc.) in the name.
+    Test that compute_ctl can create and work with databases with special
+    characters (whitespaces, %, tabs, etc.) in the name.
    """
    env = neon_simple_env

@@ -142,7 +116,6 @@ def test_compute_create_drop_dbs_and_roles(neon_simple_env: NeonEnv):
        **{
            "skip_pg_catalog_updates": False,
            "cluster": {
-                "roles": TEST_ROLE_NAMES,
                "databases": TEST_DB_NAMES,
            },
        }
@@ -166,43 +139,6 @@ def test_compute_create_drop_dbs_and_roles(neon_simple_env: NeonEnv):
            assert len(curr_db) == 1
            assert curr_db[0] == db["name"]

-    for role in TEST_ROLE_NAMES:
-        with endpoint.cursor() as cursor:
-            cursor.execute("SELECT rolname FROM pg_roles WHERE rolname = %s", (role["name"],))
-            catalog_role = cursor.fetchone()
-            assert catalog_role is not None
-            assert catalog_role[0] == role["name"]
-
-    delta_operations = []
-    for db in TEST_DB_NAMES:
-        delta_operations.append({"action": "delete_db", "name": db["name"]})
-    for role in TEST_ROLE_NAMES:
-        delta_operations.append({"action": "delete_role", "name": role["name"]})
-
-    endpoint.respec_deep(
-        **{
-            "skip_pg_catalog_updates": False,
-            "cluster": {
-                "roles": [],
-                "databases": [],
-            },
-            "delta_operations": delta_operations,
-        }
-    )
-    endpoint.reconfigure()
-
-    for db in TEST_DB_NAMES:
-        with endpoint.cursor() as cursor:
-            cursor.execute("SELECT datname FROM pg_database WHERE datname = %s", (db["name"],))
-            catalog_db = cursor.fetchone()
-            assert catalog_db is None
-
-    for role in TEST_ROLE_NAMES:
-        with endpoint.cursor() as cursor:
-            cursor.execute("SELECT rolname FROM pg_roles WHERE rolname = %s", (role["name"],))
-            catalog_role = cursor.fetchone()
-            assert catalog_role is None
-

 def test_dropdb_with_subscription(neon_simple_env: NeonEnv):
    """
@@ -214,19 +150,17 @@ def test_dropdb_with_subscription(neon_simple_env: NeonEnv):
    # stuff into the spec.json file.
    endpoint = env.endpoints.create_start("main")

-    SUB_DB_NAME = "';subscriber_db $$ $x$ $;"
-    PUB_DB_NAME = "publisher_db"
    TEST_DB_NAMES = [
        {
            "name": "neondb",
            "owner": "cloud_admin",
        },
        {
-            "name": SUB_DB_NAME,
+            "name": "subscriber_db",
            "owner": "cloud_admin",
        },
        {
-            "name": PUB_DB_NAME,
+            "name": "publisher_db",
            "owner": "cloud_admin",
        },
    ]
@@ -243,47 +177,47 @@ def test_dropdb_with_subscription(neon_simple_env: NeonEnv):
    )
    endpoint.reconfigure()

-    # Connect to the PUB_DB_NAME and create a publication
-    with endpoint.cursor(dbname=PUB_DB_NAME) as cursor:
+    # connect to the publisher_db and create a publication
+    with endpoint.cursor(dbname="publisher_db") as cursor:
        cursor.execute("CREATE PUBLICATION mypub FOR ALL TABLES")
        cursor.execute("select pg_catalog.pg_create_logical_replication_slot('mysub', 'pgoutput');")
        cursor.execute("CREATE TABLE t(a int)")
        cursor.execute("INSERT INTO t VALUES (1)")
        cursor.execute("CHECKPOINT")

-    # Connect to the SUB_DB_NAME and create a subscription
-    # Note that we need to create subscription with the following connstr:
-    connstr = endpoint.connstr(dbname=PUB_DB_NAME).replace("'", "''")
-    with endpoint.cursor(dbname=SUB_DB_NAME) as cursor:
+    # connect to the subscriber_db and create a subscription
+    # Note that we need to create subscription with
+    connstr = endpoint.connstr(dbname="publisher_db").replace("'", "''")
+    with endpoint.cursor(dbname="subscriber_db") as cursor:
        cursor.execute("CREATE TABLE t(a int)")
        cursor.execute(
-            f"CREATE SUBSCRIPTION mysub CONNECTION '{connstr}' PUBLICATION mypub WITH (create_slot = false) "
+            f"CREATE SUBSCRIPTION mysub CONNECTION '{connstr}' PUBLICATION mypub  WITH (create_slot = false) "
        )

-    # Wait for the subscription to be active
+    # wait for the subscription to be active
    logical_replication_sync(
        endpoint,
        endpoint,
        "mysub",
-        sub_dbname=SUB_DB_NAME,
-        pub_dbname=PUB_DB_NAME,
+        sub_dbname="subscriber_db",
+        pub_dbname="publisher_db",
    )

    # Check that replication is working
-    with endpoint.cursor(dbname=SUB_DB_NAME) as cursor:
+    with endpoint.cursor(dbname="subscriber_db") as cursor:
        cursor.execute("SELECT * FROM t")
        rows = cursor.fetchall()
        assert len(rows) == 1
        assert rows[0][0] == 1

-    # Drop the SUB_DB_NAME from the list
+    # drop the subscriber_db from the list
    TEST_DB_NAMES_NEW = [
        {
            "name": "neondb",
            "owner": "cloud_admin",
        },
        {
-            "name": PUB_DB_NAME,
+            "name": "publisher_db",
            "owner": "cloud_admin",
        },
    ]
@@ -296,7 +230,7 @@ def test_dropdb_with_subscription(neon_simple_env: NeonEnv):
                "databases": TEST_DB_NAMES_NEW,
            },
            "delta_operations": [
-                {"action": "delete_db", "name": SUB_DB_NAME},
+                {"action": "delete_db", "name": "subscriber_db"},
                # also test the case when we try to delete a non-existent database
                # shouldn't happen in normal operation,
                # but can occur when failed operations are retried
@@ -305,22 +239,22 @@ def test_dropdb_with_subscription(neon_simple_env: NeonEnv):
        }
    )

-    logging.info(f"Reconfiguring the endpoint to drop the {SUB_DB_NAME} database")
+    logging.info("Reconfiguring the endpoint to drop the subscriber_db")
    endpoint.reconfigure()

-    # Check that the SUB_DB_NAME is dropped
+    # Check that the subscriber_db is dropped
    with endpoint.cursor() as cursor:
-        cursor.execute("SELECT datname FROM pg_database WHERE datname = %s", (SUB_DB_NAME,))
+        cursor.execute("SELECT datname FROM pg_database WHERE datname = %s", ("subscriber_db",))
        catalog_db = cursor.fetchone()
        assert catalog_db is None

-    # Check that we can still connect to the PUB_DB_NAME
-    with endpoint.cursor(dbname=PUB_DB_NAME) as cursor:
+    # Check that we can still connect to the publisher_db
+    with endpoint.cursor(dbname="publisher_db") as cursor:
        cursor.execute("SELECT * FROM current_database()")
        curr_db = cursor.fetchone()
        assert curr_db is not None
        assert len(curr_db) == 1
-        assert curr_db[0] == PUB_DB_NAME
+        assert curr_db[0] == "publisher_db"


 def test_compute_drop_role(neon_simple_env: NeonEnv):
@@ -331,7 +265,6 @@ def test_compute_drop_role(neon_simple_env: NeonEnv):
    """
    env = neon_simple_env
    TEST_DB_NAME = "db_with_permissions"
-    TEST_GRANTEE = "'); MALFORMED SQL $$ $x$ $/;5%$ %I"

    endpoint = env.endpoints.create_start("main")

@@ -368,18 +301,16 @@ def test_compute_drop_role(neon_simple_env: NeonEnv):
        cursor.execute("create view test_view as select * from test_table")

    with endpoint.cursor(dbname=TEST_DB_NAME, user="neon") as cursor:
-        cursor.execute(f'create role "{TEST_GRANTEE}"')
+        cursor.execute("create role readonly")
        # We (`compute_ctl`) make 'neon' the owner of schema 'public' in the owned database.
        # Postgres has all sorts of permissions and grants that we may not handle well,
        # but this is the shortest repro grant for the issue
        # https://github.com/neondatabase/cloud/issues/13582
-        cursor.execute(f'grant select on all tables in schema public to "{TEST_GRANTEE}"')
+        cursor.execute("grant select on all tables in schema public to readonly")

    # Check that role was created
    with endpoint.cursor() as cursor:
-        cursor.execute(
-            "SELECT rolname FROM pg_roles WHERE rolname = %(role)s", {"role": TEST_GRANTEE}
-        )
+        cursor.execute("SELECT rolname FROM pg_roles WHERE rolname = 'readonly'")
        role = cursor.fetchone()
        assert role is not None

@@ -387,8 +318,7 @@ def test_compute_drop_role(neon_simple_env: NeonEnv):
    # that may block our ability to drop the role.
    with endpoint.cursor(dbname=TEST_DB_NAME) as cursor:
        cursor.execute(
-            "select grantor from information_schema.role_table_grants where grantee = %(grantee)s",
-            {"grantee": TEST_GRANTEE},
+            "select grantor from information_schema.role_table_grants where grantee = 'readonly'"
        )
        res = cursor.fetchall()
        assert len(res) == 2, f"Expected 2 table grants, got {len(res)}"
@@ -402,7 +332,7 @@ def test_compute_drop_role(neon_simple_env: NeonEnv):
            "delta_operations": [
                {
                    "action": "delete_role",
-                    "name": TEST_GRANTEE,
+                    "name": "readonly",
                },
            ],
        }
@@ -411,9 +341,7 @@ def test_compute_drop_role(neon_simple_env: NeonEnv):

    # Check that role is dropped
    with endpoint.cursor() as cursor:
-        cursor.execute(
-            "SELECT rolname FROM pg_roles WHERE rolname = %(role)s", {"role": TEST_GRANTEE}
-        )
+        cursor.execute("SELECT rolname FROM pg_roles WHERE rolname = 'readonly'")
        role = cursor.fetchone()
        assert role is None

--- a/test_runner/regress/test_download_extensions.py
+++ b/test_runner/regress/test_download_extensions.py
@@ -137,8 +137,6 @@ def test_remote_extensions(
    metrics = parse_metrics(raw_metrics)
    remote_ext_requests = metrics.query_all(
        "compute_ctl_remote_ext_requests_total",
-        # Check that we properly report the filename in the metrics
-        {"filename": "anon.tar.zst"},
    )
    assert len(remote_ext_requests) == 1
    for sample in remote_ext_requests:
--- a/test_runner/regress/test_pageserver_api.py
+++ b/test_runner/regress/test_pageserver_api.py
@@ -7,7 +7,7 @@ from fixtures.neon_fixtures import (
    NeonEnvBuilder,
 )
 from fixtures.pageserver.http import PageserverHttpClient
-from fixtures.utils import run_only_on_default_postgres, wait_until
+from fixtures.utils import wait_until


 def check_client(env: NeonEnv, client: PageserverHttpClient):
@@ -138,25 +138,3 @@ def test_pageserver_http_api_client_auth_enabled(neon_env_builder: NeonEnvBuilde

    with env.pageserver.http_client(auth_token=pageserver_token) as client:
        check_client(env, client)
-
-
-@run_only_on_default_postgres("it does not use any postgres functionality")
-def test_pageserver_http_index_part_force_patch(neon_env_builder: NeonEnvBuilder):
-    env = neon_env_builder.init_start()
-    tenant_id = env.initial_tenant
-    timeline_id = env.initial_timeline
-    with env.pageserver.http_client() as client:
-        client.timeline_patch_index_part(
-            tenant_id,
-            timeline_id,
-            {"rel_size_migration": "migrating"},
-        )
-        assert client.timeline_detail(tenant_id, timeline_id)["rel_size_migration"] == "migrating"
-        # This is invalid in practice: we should never rollback the migrating state to legacy.
-        # But we do it here to test the API.
-        client.timeline_patch_index_part(
-            tenant_id,
-            timeline_id,
-            {"rel_size_migration": "legacy"},
-        )
-        assert client.timeline_detail(tenant_id, timeline_id)["rel_size_migration"] == "legacy"
--- a/test_runner/regress/test_pageserver_secondary.py
+++ b/test_runner/regress/test_pageserver_secondary.py
@@ -955,17 +955,6 @@ def test_migration_to_cold_secondary(neon_env_builder: NeonEnvBuilder):

        raise RuntimeError(f"No heatmap for timeline: {tlid}")

-    def count_timeline_heatmap_layers(tlid) -> tuple[int, int]:
-        cold, hot = 0, 0
-        layers = timeline_heatmap(tlid)["layers"]
-        for layer in layers:
-            if layer["cold"]:
-                cold += 1
-            else:
-                hot += 1
-
-        return cold, hot
-
    env.storage_controller.allowed_errors.extend(
        [
            ".*Timed out.*downloading layers.*",
@@ -999,19 +988,13 @@ def test_migration_to_cold_secondary(neon_env_builder: NeonEnvBuilder):
        TenantShardId(tenant_id, shard_number=0, shard_count=0), timeline_id
    )

-    def all_layers_downloaded(node, expected_layer_count: int):
-        local_layers_count = len(node.list_layers(tenant_id, timeline_id))
+    def all_layers_downloaded(expected_layer_count: int):
+        local_layers_count = len(ps_secondary.list_layers(tenant_id, timeline_id))

        log.info(f"{local_layers_count=} {after_migration_heatmap_layers_count=}")
        assert local_layers_count >= expected_layer_count

-    def no_layers_downloaded(node):
-        local_layers_count = len(node.list_layers(tenant_id, timeline_id))
-
-        log.info(f"{local_layers_count=} {after_migration_heatmap_layers_count=}")
-        assert local_layers_count == 0
-
-    wait_until(lambda: all_layers_downloaded(ps_secondary, after_migration_heatmap_layers_count))
+    wait_until(lambda: all_layers_downloaded(after_migration_heatmap_layers_count))

    # Read everything and make sure that we're not downloading anything extra.
    # All hot layers should be available locally now.
@@ -1064,35 +1047,13 @@ def test_migration_to_cold_secondary(neon_env_builder: NeonEnvBuilder):
    wait_until(lambda: check_archival_state(TimelineArchivalState.UNARCHIVED, child_timeline_id))

    ps_secondary.http_client().tenant_heatmap_upload(tenant_id)
+    log.info(f"Parent timeline heatmap size: {len(timeline_heatmap(timeline_id)['layers'])}")
+    log.info(f"Child timeline heatmap size: {len(timeline_heatmap(child_timeline_id)['layers'])}")

-    parent_cold, parent_hot = count_timeline_heatmap_layers(timeline_id)
-    child_cold, child_hot = count_timeline_heatmap_layers(child_timeline_id)
-
-    log.info(f"Parent timeline heatmap size: cold={parent_cold}, hot={parent_hot}")
-    log.info(f"Child timeline heatmap size: cold={child_cold}, hot={child_hot}")
-
-    # All layers in the heatmap should come from the generation on unarchival.
-    # Hence, they should be cold.
-    assert parent_cold > 0
-    assert parent_hot == 0
-
-    expected_locally = parent_cold
+    expected_locally = len(timeline_heatmap(timeline_id)["layers"])
+    assert expected_locally > 0

    env.storage_controller.download_heatmap_layers(
        TenantShardId(tenant_id, shard_number=0, shard_count=0), child_timeline_id, recurse=True
    )
-    wait_until(lambda: all_layers_downloaded(ps_secondary, expected_locally))
-
-    for ps in env.pageservers:
-        ps.http_client().configure_failpoints([("secondary-layer-download-sleep", "off")])
-
-    # The uploaded heatmap is still empty. Clean up all layers on the secondary.
-    ps_attached.http_client().tenant_secondary_download(tenant_id, wait_ms=100)
-    wait_until(lambda: no_layers_downloaded(ps_attached))
-
-    # Upload a new heatmap. The previously cold layers become hot since they're now resident.
-    ps_secondary.http_client().tenant_heatmap_upload(tenant_id)
-
-    # Warm up the current secondary.
-    ps_attached.http_client().tenant_secondary_download(tenant_id, wait_ms=100)
-    wait_until(lambda: all_layers_downloaded(ps_secondary, expected_locally))
+    wait_until(lambda: all_layers_downloaded(expected_locally))
--- a/test_runner/regress/test_pg_regress.py
+++ b/test_runner/regress/test_pg_regress.py
@@ -5,7 +5,7 @@ from __future__ import annotations

 from concurrent.futures import ThreadPoolExecutor
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, cast

 import pytest
 from fixtures.log_helper import log
@@ -118,20 +118,10 @@ def post_checks(env: NeonEnv, test_output_dir: Path, db_name: str, endpoint: End
        pageserver.http_client().timeline_gc(shard, env.initial_timeline, None)


-def patch_tenant_conf(tenant_conf: dict[str, Any], reldir_type: str) -> dict[str, Any]:
-    tenant_conf = tenant_conf.copy()
-    if reldir_type == "v2":
-        tenant_conf["rel_size_v2_enabled"] = "true"
-    else:
-        tenant_conf["rel_size_v2_enabled"] = "false"
-    return tenant_conf
-
-
 # Run the main PostgreSQL regression tests, in src/test/regress.
 #
@pytest.mark.timeout(3000)  # Contains many sub-tests, is slow in debug builds
@pytest.mark.parametrize("shard_count", [None, 4])
-@pytest.mark.parametrize("reldir_type", ["v1", "v2"])
 def test_pg_regress(
    neon_env_builder: NeonEnvBuilder,
    test_output_dir: Path,
@@ -140,7 +130,6 @@ def test_pg_regress(
    base_dir: Path,
    pg_distrib_dir: Path,
    shard_count: int | None,
-    reldir_type: str,
 ):
    DBNAME = "regression"

@@ -153,7 +142,7 @@ def test_pg_regress(

    neon_env_builder.enable_pageserver_remote_storage(s3_storage())
    env = neon_env_builder.init_start(
-        initial_tenant_conf=patch_tenant_conf(TENANT_CONF, reldir_type),
+        initial_tenant_conf=TENANT_CONF,
        initial_tenant_shard_count=shard_count,
    )

@@ -207,7 +196,6 @@ def test_pg_regress(
 #
@pytest.mark.timeout(1500)  # Contains many sub-tests, is slow in debug builds
@pytest.mark.parametrize("shard_count", [None, 4])
-@pytest.mark.parametrize("reldir_type", ["v1", "v2"])
 def test_isolation(
    neon_env_builder: NeonEnvBuilder,
    test_output_dir: Path,
@@ -216,7 +204,6 @@ def test_isolation(
    base_dir: Path,
    pg_distrib_dir: Path,
    shard_count: int | None,
-    reldir_type: str,
 ):
    DBNAME = "isolation_regression"

@@ -224,8 +211,7 @@ def test_isolation(
        neon_env_builder.num_pageservers = shard_count
    neon_env_builder.enable_pageserver_remote_storage(s3_storage())
    env = neon_env_builder.init_start(
-        initial_tenant_conf=patch_tenant_conf(TENANT_CONF, reldir_type),
-        initial_tenant_shard_count=shard_count,
+        initial_tenant_conf=TENANT_CONF, initial_tenant_shard_count=shard_count
    )

    # Connect to postgres and create a database called "regression".
@@ -281,7 +267,6 @@ def test_isolation(
 # Run extra Neon-specific pg_regress-based tests. The tests and their
 # schedule file are in the sql_regress/ directory.
@pytest.mark.parametrize("shard_count", [None, 4])
-@pytest.mark.parametrize("reldir_type", ["v1", "v2"])
 def test_sql_regress(
    neon_env_builder: NeonEnvBuilder,
    test_output_dir: Path,
@@ -290,7 +275,6 @@ def test_sql_regress(
    base_dir: Path,
    pg_distrib_dir: Path,
    shard_count: int | None,
-    reldir_type: str,
 ):
    DBNAME = "regression"

@@ -298,8 +282,7 @@ def test_sql_regress(
        neon_env_builder.num_pageservers = shard_count
    neon_env_builder.enable_pageserver_remote_storage(s3_storage())
    env = neon_env_builder.init_start(
-        initial_tenant_conf=patch_tenant_conf(TENANT_CONF, reldir_type),
-        initial_tenant_shard_count=shard_count,
+        initial_tenant_conf=TENANT_CONF, initial_tenant_shard_count=shard_count
    )

    # Connect to postgres and create a database called "regression".
@@ -362,7 +345,9 @@ def test_tx_abort_with_many_relations(
    """

    env = neon_env_builder.init_start(
-        initial_tenant_conf=patch_tenant_conf({}, reldir_type),
+        initial_tenant_conf={
+            "rel_size_v2_enabled": "true" if reldir_type == "v2" else "false",
+        }
    )
    ep = env.endpoints.create_start(
        "main",
@@ -390,8 +375,12 @@ def test_tx_abort_with_many_relations(

    # How many relations: this number is tuned to be long enough to take tens of seconds
    # if the rollback code path is buggy, tripping the test's timeout.
-    n = 5000
-    step = 2500
+    if reldir_type == "v1":
+        n = 4000
+        step = 4000
+    else:
+        n = 20000
+        step = 5000

    def create():
        # Create many relations
--- a/test_runner/regress/test_vm_bits.py
+++ b/test_runner/regress/test_vm_bits.py
@@ -327,9 +327,9 @@ def test_check_visibility_map(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin):
        log.info(f"pgbench run {i+1}/{PGBENCH_RUNS}")
        endpoint.safe_psql(f"create database {dbname}")
        connstr = endpoint.connstr(dbname=dbname)
-        # Initialize the data set, but don't vacuum yet.
-        pg_bin.run(["pgbench", "-i", "-s", "8", "-n", connstr])
-        # Vacuum to create the visibility map, and freeze the tuples to set the frozen bit.
+        # pgbench -i will automatically vacuum the tables. This creates the visibility map.
+        pg_bin.run(["pgbench", "-i", "-s", "10", connstr])
+        # Freeze the tuples to set the initial frozen bit.
        endpoint.safe_psql("vacuum freeze", dbname=dbname)
        # Run pgbench.
        pg_bin.run(["pgbench", "-c", "32", "-j", "8", "-T", "10", connstr])
@@ -354,3 +354,19 @@ def test_check_visibility_map(neon_env_builder: NeonEnvBuilder, pg_bin: PgBin):
            row = cur.fetchone()
            assert row is not None
            assert row[0] == 0, f"{row[0]} inconsistent VM pages (frozen)"
+
+    # Vacuum and freeze the tables, and check that the visibility map is still accurate.
+    for dbname in dbnames:
+        log.info(f"Vacuuming and checking visibility map for {dbname}")
+        with endpoint.cursor(dbname=dbname) as cur:
+            cur.execute("vacuum freeze")
+
+            cur.execute("select count(*) from pg_check_visible('pgbench_accounts')")
+            row = cur.fetchone()
+            assert row is not None
+            assert row[0] == 0, f"{row[0]} inconsistent VM pages (visible)"
+
+            cur.execute("select count(*) from pg_check_frozen('pgbench_accounts')")
+            row = cur.fetchone()
+            assert row is not None
+            assert row[0] == 0, f"{row[0]} inconsistent VM pages (frozen)"
--- a/vendor/postgres-v14
+++ b/vendor/postgres-v14
--- a/vendor/postgres-v15
+++ b/vendor/postgres-v15
--- a/vendor/postgres-v16
+++ b/vendor/postgres-v16
--- a/vendor/postgres-v17
+++ b/vendor/postgres-v17
--- a/vendor/revisions.json
+++ b/vendor/revisions.json
@@ -1,18 +1,18 @@
 {
  "v17": [
    "17.4",
-    "780efda2ef8d629495cc289624534ba8cde40779"
+    "517b8dc244abf3e56f0089849e464af76f70b94e"
  ],
  "v16": [
    "16.8",
-    "78050f965f2e550fd6e58f837394cb3d080d7d42"
+    "799e7a08dd171aa06a7395dd326f4243aaeb9f93"
  ],
  "v15": [
    "15.12",
-    "533be42f7da97e614ce1c494fafe3e49f53991b1"
+    "9b118b1cffa6e4ca0d63389b57b54d11e207e9a8"
  ],
  "v14": [
    "14.17",
-    "b1425505c6f9a622a5aadf3ee362740519993310"
+    "6254ab9b4496c3e481bc037ae69d859bbc2bdd7d"
  ]
 }
--- a/workspace_hack/Cargo.toml
+++ b/workspace_hack/Cargo.toml
@@ -18,7 +18,7 @@ license.workspace = true
 ahash = { version = "0.8" }
 anyhow = { version = "1", features = ["backtrace"] }
 base64-594e8ee84c453af0 = { package = "base64", version = "0.13", features = ["alloc"] }
-base64-647d43efb71741da = { package = "base64", version = "0.21" }
+base64-647d43efb71741da = { package = "base64", version = "0.21", features = ["alloc"] }
 base64ct = { version = "1", default-features = false, features = ["std"] }
 bytes = { version = "1", features = ["serde"] }
 camino = { version = "1", default-features = false, features = ["serde1"] }
Author	SHA1	Message	Date
John Spray	ce3d23e223	Notes on cutover	2025-03-14 19:19:28 +00:00
John Spray	7d60895310	rename	2025-03-14 15:33:23 +00:00
John Spray	f441b1efb4	docs: add hot secondaries rfc	2025-03-13 17:22:56 +00:00
				`@@ -1 +0,0 @@`
				`SELECT lfc_value AS lfc_chunk_size_pages FROM neon.neon_lfc_stats WHERE lfc_key = 'file_cache_chunk_size_pages';`