pageserver: handle rel drops correctly in rel size cache

.github/actions/neon-project-create/action.yml

@@ -49,6 +49,10 @@ inputs:
     description: 'A JSON object with project settings'
     required: false
     default: '{}'
+  default_endpoint_settings:
+    description: 'A JSON object with the default endpoint settings'
+    required: false
+    default: '{}'
 
 outputs:
   dsn:
@@ -135,6 +139,21 @@ runs:
             -H "Accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer ${ADMIN_API_KEY}" \
             -d "{\"scheduling\": \"Essential\"}"
         fi
+        # XXX
+        # This is a workaround for the default endpoint settings, which currently do not allow some settings in the public API.
+        # https://github.com/neondatabase/cloud/issues/27108
+        if [[ -n ${DEFAULT_ENDPOINT_SETTINGS} && ${DEFAULT_ENDPOINT_SETTINGS} != "{}" ]] ; then
+          PROJECT_DATA=$(curl -X GET \
+              "https://${API_HOST}/regions/${REGION_ID}/api/v1/admin/projects/${project_id}" \
+              -H "Accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer ${ADMIN_API_KEY}" \
+              -d "{\"scheduling\": \"Essential\"}"
+          )
+          NEW_DEFAULT_ENDPOINT_SETTINGS=$(echo ${PROJECT_DATA} | jq -rc ".project.default_endpoint_settings + ${DEFAULT_ENDPOINT_SETTINGS}")
+          curl -X POST --fail \
+                "https://${API_HOST}/regions/${REGION_ID}/api/v1/admin/projects/${project_id}/default_endpoint_settings" \
+                -H "Accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer ${ADMIN_API_KEY}" \
+                --data "${NEW_DEFAULT_ENDPOINT_SETTINGS}"
+        fi
         
 
       env:
@@ -152,3 +171,4 @@ runs:
         PSQL: ${{ inputs.psql_path }}
         LD_LIBRARY_PATH: ${{ inputs.libpq_lib_path }}
         PROJECT_SETTINGS: ${{ inputs.project_settings }}
+        DEFAULT_ENDPOINT_SETTINGS: ${{ inputs.default_endpoint_settings }}

.github/workflows/_build-and-test-locally.yml

@@ -279,14 +279,18 @@ jobs:
           # run all non-pageserver tests
           ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E '!package(pageserver)'
 
-          # run pageserver tests
-          # (When developing new pageserver features gated by config fields, we commonly make the rust
-          # unit tests sensitive to an environment variable NEON_PAGESERVER_UNIT_TEST_FEATURENAME.
-          # Then run the nextest invocation below for all relevant combinations. Singling out the
-          # pageserver tests from non-pageserver tests cuts down the time it takes for this CI step.)
-          NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=tokio-epoll-uring  \
-          ${cov_prefix} \
-          cargo nextest run $CARGO_FLAGS $CARGO_FEATURES  -E 'package(pageserver)'
+          # run pageserver tests with different settings
+          for get_vectored_concurrent_io in sequential sidecar-task; do
+            for io_engine in std-fs tokio-epoll-uring ; do
+                for io_mode in buffered direct direct-rw ; do
+                  NEON_PAGESERVER_UNIT_TEST_GET_VECTORED_CONCURRENT_IO=$get_vectored_concurrent_io \
+                  NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=$io_engine \
+                  NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IO_MODE=$io_mode \
+                  ${cov_prefix} \
+                  cargo nextest run $CARGO_FLAGS $CARGO_FEATURES  -E 'package(pageserver)'
+              done
+            done
+          done
 
           # Run separate tests for real S3
           export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
@@ -401,6 +405,8 @@ jobs:
           CHECK_ONDISK_DATA_COMPATIBILITY: nonempty
           BUILD_TAG: ${{ inputs.build-tag }}
           PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring
+          PAGESERVER_GET_VECTORED_CONCURRENT_IO: sidecar-task
+          PAGESERVER_VIRTUAL_FILE_IO_MODE: direct-rw
           USE_LFC: ${{ matrix.lfc_state == 'with-lfc' && 'true' || 'false' }}
 
       # Temporary disable this step until we figure out why it's so flaky

.github/workflows/build_and_test.yml

@@ -323,6 +323,8 @@ jobs:
           PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
           TEST_RESULT_CONNSTR: "${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}"
           PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring
+          PAGESERVER_GET_VECTORED_CONCURRENT_IO: sidecar-task
+          PAGESERVER_VIRTUAL_FILE_IO_MODE: direct-rw
           SYNC_BETWEEN_TESTS: true
       # XXX: no coverage data handling here, since benchmarks are run on release builds,
       # while coverage is currently collected for the debug ones
@@ -963,7 +965,7 @@ jobs:
           fi
 
       - name: Verify docker-compose example and test extensions
-        timeout-minutes: 60
+        timeout-minutes: 20
         env:
           TAG: >-
             ${{

.github/workflows/cloud-extensions.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         pg-version: [16, 17]
 
-    runs-on: us-east-2
+    runs-on: [ self-hosted, small ]
     container:
       # We use the neon-test-extensions image here as it contains the source code for the extensions.
       image: ghcr.io/neondatabase/neon-test-extensions-v${{ matrix.pg-version }}:latest
@@ -71,7 +71,20 @@ jobs:
           region_id: ${{ inputs.region_id || 'aws-us-east-2' }}
           postgres_version: ${{ matrix.pg-version }}
           project_settings: ${{ steps.project-settings.outputs.settings }}
+          # We need these settings to get the expected output results.
+          # We cannot use the environment variables e.g. PGTZ due to
+          # https://github.com/neondatabase/neon/issues/1287
+          default_endpoint_settings: >
+            {
+              "pg_settings": {
+                "DateStyle": "Postgres,MDY",
+                "TimeZone": "America/Los_Angeles",
+                "compute_query_id": "off",
+                "neon.allow_unstable_extensions": "on"
+              }
+            }
           api_key: ${{ secrets.NEON_STAGING_API_KEY }}
+          admin_api_key: ${{ secrets.NEON_STAGING_ADMIN_API_KEY }}
 
       - name: Run the regression tests
         run: /run-tests.sh -r /ext-src

.github/workflows/neon_extra_builds.yml

@@ -63,10 +63,8 @@ jobs:
 
       - name: Filter out only v-string for build matrix
         id: postgres_changes
-        env:
-          CHANGES: ${{ steps.files_changed.outputs.changes }}
         run: |
-          v_strings_only_as_json_array=$(echo ${CHANGES} | jq '.[]|select(test("v\\d+"))' | jq --slurp -c)
+          v_strings_only_as_json_array=$(echo ${{ steps.files_changed.outputs.chnages }} | jq '.[]|select(test("v\\d+"))' | jq --slurp -c)
           echo "changes=${v_strings_only_as_json_array}" | tee -a "${GITHUB_OUTPUT}"
 
   check-macos-build:

Cargo.lock

@@ -1112,12 +1112,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cfg_aliases"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
-
 [[package]]
 name = "cgroups-rs"
 version = "0.3.3"
@@ -1312,7 +1306,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonwebtoken",
  "metrics",
- "nix 0.30.1",
+ "nix 0.27.1",
  "notify",
  "num_cpus",
  "once_cell",
@@ -1435,7 +1429,7 @@ dependencies = [
  "humantime-serde",
  "hyper 0.14.30",
  "jsonwebtoken",
- "nix 0.30.1",
+ "nix 0.27.1",
  "once_cell",
  "pageserver_api",
  "pageserver_client",
@@ -3518,9 +3512,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.172"
+version = "0.2.169"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
+checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
 
 [[package]]
 name = "libloading"
@@ -3794,16 +3788,6 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"
 
-[[package]]
-name = "neon-shmem"
-version = "0.1.0"
-dependencies = [
- "nix 0.30.1",
- "tempfile",
- "thiserror 1.0.69",
- "workspace_hack",
-]
-
 [[package]]
 name = "never-say-never"
 version = "6.6.666"
@@ -3837,13 +3821,12 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.30.1"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
+checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
 dependencies = [
  "bitflags 2.8.0",
  "cfg-if",
- "cfg_aliases",
  "libc",
  "memoffset 0.9.0",
 ]
@@ -4297,7 +4280,7 @@ dependencies = [
  "jsonwebtoken",
  "md5",
  "metrics",
- "nix 0.30.1",
+ "nix 0.27.1",
  "num-traits",
  "num_cpus",
  "once_cell",
@@ -4348,7 +4331,6 @@ dependencies = [
  "toml_edit",
  "tracing",
  "tracing-utils",
- "twox-hash",
  "url",
  "utils",
  "uuid",
@@ -4373,7 +4355,7 @@ dependencies = [
  "humantime",
  "humantime-serde",
  "itertools 0.10.5",
- "nix 0.30.1",
+ "nix 0.27.1",
  "once_cell",
  "postgres_backend",
  "postgres_ffi",
@@ -5205,7 +5187,6 @@ dependencies = [
  "rustls 0.23.18",
  "rustls-native-certs 0.8.0",
  "rustls-pemfile 2.1.1",
- "ryu",
  "scopeguard",
  "serde",
  "serde_json",
@@ -7917,7 +7898,7 @@ dependencies = [
  "humantime",
  "jsonwebtoken",
  "metrics",
- "nix 0.30.1",
+ "nix 0.27.1",
  "once_cell",
  "pem",
  "pin-project-lite",
@@ -8493,7 +8474,6 @@ dependencies = [
  "log",
  "memchr",
  "nix 0.26.4",
- "nix 0.30.1",
  "nom",
  "num",
  "num-bigint",

Cargo.toml

@@ -23,7 +23,6 @@ members = [
     "libs/postgres_ffi",
     "libs/safekeeper_api",
     "libs/desim",
-    "libs/neon-shmem",
     "libs/utils",
     "libs/consumption_metrics",
     "libs/postgres_backend",
@@ -128,7 +127,7 @@ md5 = "0.7.0"
 measured = { version = "0.0.22", features=["lasso"] }
 measured-process = { version = "0.0.22" }
 memoffset = "0.9"
-nix = { version = "0.30.1", features = ["dir", "fs", "mman", "process", "socket", "signal", "poll"] }
+nix = { version = "0.27", features = ["dir", "fs", "process", "socket", "signal", "poll"] }
 # Do not update to >= 7.0.0, at least. The update will have a significant impact
 # on compute startup metrics (start_postgres_ms), >= 25% degradation.
 notify = "6.0.0"

build-tools.Dockerfile

@@ -292,7 +292,7 @@ WORKDIR /home/nonroot
 
 # Rust
 # Please keep the version of llvm (installed above) in sync with rust llvm (`rustc --version --verbose | grep LLVM`)
-ENV RUSTC_VERSION=1.87.0
+ENV RUSTC_VERSION=1.86.0
 ENV RUSTUP_HOME="/home/nonroot/.rustup"
 ENV PATH="/home/nonroot/.cargo/bin:${PATH}"
 ARG RUSTFILT_VERSION=0.2.1

control_plane/src/background_process.rs

@@ -14,7 +14,7 @@
 
 use std::ffi::OsStr;
 use std::io::Write;
-use std::os::fd::AsFd;
+use std::os::unix::prelude::AsRawFd;
 use std::os::unix::process::CommandExt;
 use std::path::Path;
 use std::process::Command;
@@ -356,7 +356,7 @@ where
             let file = pid_file::claim_for_current_process(&path).expect("claim pid file");
             // Remove the FD_CLOEXEC flag on the pidfile descriptor so that the pidfile
             // remains locked after exec.
-            nix::fcntl::fcntl(file.as_fd(), FcntlArg::F_SETFD(FdFlag::empty()))
+            nix::fcntl::fcntl(file.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty()))
                 .expect("remove FD_CLOEXEC");
             // Don't run drop(file), it would close the file before we actually exec.
             std::mem::forget(file);

control_plane/src/bin/neon_local.rs

@@ -8,6 +8,7 @@
 use std::borrow::Cow;
 use std::collections::{BTreeSet, HashMap};
 use std::fs::File;
+use std::os::fd::AsRawFd;
 use std::path::PathBuf;
 use std::process::exit;
 use std::str::FromStr;
@@ -30,7 +31,7 @@ use control_plane::safekeeper::SafekeeperNode;
 use control_plane::storage_controller::{
     NeonStorageControllerStartArgs, NeonStorageControllerStopArgs, StorageController,
 };
-use nix::fcntl::{Flock, FlockArg};
+use nix::fcntl::{FlockArg, flock};
 use pageserver_api::config::{
     DEFAULT_HTTP_LISTEN_PORT as DEFAULT_PAGESERVER_HTTP_PORT,
     DEFAULT_PG_LISTEN_PORT as DEFAULT_PAGESERVER_PG_PORT,
@@ -748,16 +749,16 @@ struct TimelineTreeEl {
 
 /// A flock-based guard over the neon_local repository directory
 struct RepoLock {
-    _file: Flock<File>,
+    _file: File,
 }
 
 impl RepoLock {
     fn new() -> Result<Self> {
         let repo_dir = File::open(local_env::base_path())?;
-        match Flock::lock(repo_dir, FlockArg::LockExclusive) {
-            Ok(f) => Ok(Self { _file: f }),
-            Err((_, e)) => Err(e).context("flock error"),
-        }
+        let repo_dir_fd = repo_dir.as_raw_fd();
+        flock(repo_dir_fd, FlockArg::LockExclusive)?;
+
+        Ok(Self { _file: repo_dir })
     }
 }
 

docker-compose/ext-src/alter_db.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-# We need these settings to get the expected output results.
-# We cannot use the environment variables e.g. PGTZ due to
-# https://github.com/neondatabase/neon/issues/1287
-export DATABASE=${1:-contrib_regression}
-psql -c "ALTER DATABASE ${DATABASE} SET neon.allow_unstable_extensions='on'" \
-     -c "ALTER DATABASE ${DATABASE} SET DateStyle='Postgres,MDY'" \
-     -c "ALTER DATABASE ${DATABASE} SET TimeZone='America/Los_Angeles'" \

docker-compose/ext-src/pg_graphql-src/regular-test.sh

@@ -18,7 +18,6 @@ TESTS=${TESTS/row_level_security/}
 TESTS=${TESTS/sqli_connection/}
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 psql -v ON_ERROR_STOP=1 -f test/fixtures.sql -d contrib_regression
 ${REGRESS} --use-existing --dbname=contrib_regression --inputdir=${TESTDIR} ${TESTS}
 

docker-compose/ext-src/pgrag-src/regular-test.sh

@@ -3,7 +3,6 @@ set -ex
 cd "$(dirname "${0}")"
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 psql -d contrib_regression -c "CREATE EXTENSION vector" -c "CREATE EXTENSION rag"
 PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
 ${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin'    --use-existing --load-extension=vector --load-extension=rag --dbname=contrib_regression basic_functions text_processing api_keys chunking_functions document_processing embedding_api_functions voyageai_functions

docker-compose/ext-src/pgx_ulid-src/Makefile

@@ -20,6 +20,5 @@ installcheck: regression-test
 regression-test:
 	dropdb --if-exists contrib_regression
 	createdb contrib_regression
-	../alter_db.sh
 	psql -d contrib_regression -c "CREATE EXTENSION $(EXTNAME)"
 	$(PG_REGRESS) --inputdir=. --outputdir=. --use-existing --dbname=contrib_regression $(REGRESS)

docker-compose/ext-src/plv8-src/regular-test.sh

@@ -3,7 +3,6 @@ set -ex
 cd "$(dirname ${0})"
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
 REGRESS="$(make -n installcheck | awk '{print substr($0,index($0,"init-extension"));}')"
 REGRESS="${REGRESS/startup_perms/}"

docker-compose/ext-src/rag_bge_small_en_v15-src/Makefile

@@ -11,6 +11,5 @@ PG_REGRESS := $(dir $(PGXS))../../src/test/regress/pg_regress
 installcheck:
 	dropdb --if-exists contrib_regression
 	createdb contrib_regression
-	../alter_db.sh
 	psql -d contrib_regression -c "CREATE EXTENSION vector" -c "CREATE EXTENSION rag_bge_small_en_v15"
 	$(PG_REGRESS) --use-existing --dbname=contrib_regression $(REGRESS)

docker-compose/ext-src/rag_jina_reranker_v1_tiny_en-src/Makefile

@@ -11,6 +11,5 @@ PG_REGRESS := $(dir $(PGXS))../../src/test/regress/pg_regress
 installcheck:
 	dropdb --if-exists contrib_regression
 	createdb contrib_regression
-	../alter_db.sh
 	psql -d contrib_regression -c "CREATE EXTENSION vector" -c "CREATE EXTENSION rag_jina_reranker_v1_tiny_en"
 	$(PG_REGRESS) --use-existing --dbname=contrib_regression $(REGRESS)

docker-compose/ext-src/rum-src/regular-test.sh

@@ -3,6 +3,5 @@ set -ex
 cd "$(dirname ${0})"
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
 ${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression rum rum_hash ruminv timestamp orderby orderby_hash altorder altorder_hash limits int2 int4 int8 float4 float8 money oid time timetz date interval macaddr inet cidr text varchar char bytea bit varbit numeric rum_weight expr array

docs/rfcs/043-bottom-most-gc-compaction.md

@@ -1,194 +0,0 @@
-# Bottommost Garbage-Collection Compaction
-
-## Summary
-
-The goal of this doc is to propose a way to reliably collect garbages below the GC horizon. This process is called bottom-most garbage-collect-compaction, and is part of the broader legacy-enhanced compaction that we plan to implement in the future.
-
-## Motivation
-
-The current GC algorithm will wait until the covering via image layers before collecting the garbages of a key region. Relying on image layer generation to generate covering images is not reliable. There are prior arts to generate feedbacks from the GC algorithm to the image generation process to accelerate garbage collection, but it slows down the system and creates write amplification.
-
-# Basic Idea
-
-![](images/036-bottom-most-gc-compaction/01-basic-idea.svg)
-
-The idea of bottom-most compaction is simple: we rewrite all layers that are below or intersect with the GC horizon to produce a flat level of image layers at the GC horizon and deltas above the GC horizon. In this process,
-
-- All images and deltas ≤ GC horizon LSN will be dropped. This process collects garbages.
-- We produce images for all keys involved in the compaction process at the GC horizon.
-
-Therefore, it can precisely collect all garbages below the horizon, and reduce the space amplification, i.e., in the staircase pattern (test_gc_feedback).
-
-![The staircase pattern in test_gc_feedback in the original compaction algorithm. The goal is to collect garbage below the red horizontal line.](images/036-bottom-most-gc-compaction/12-staircase-test-gc-feedback.png)
-
-The staircase pattern in test_gc_feedback in the original compaction algorithm. The goal is to collect garbage below the red horizontal line.
-
-# Branches
-
-With branches, the bottom-most compaction should retain a snapshot of the keyspace at the `retain_lsn` so that the child branch can access data at the branch point. This requires some modifications to the basic bottom-most compaction algorithm that we sketched above. 
-
-![](images/036-bottom-most-gc-compaction/03-retain-lsn.svg)
-
-## Single Timeline w/ Snapshots: handle `retain_lsn`
-
-First let’s look into the case where we create branches over the main branch but don’t write any data to them (aka “snapshots”).
-
-The bottom-most compaction algorithm collects all deltas and images of a key and can make decisions on what data to retain. Given that we have a single key’s history as below:
-
-```
-LSN 0x10 -> A
-LSN 0x20 -> append B
-retain_lsn: 0x20
-LSN 0x30 -> append C
-LSN 0x40 -> append D
-retain_lsn: 0x40
-LSN 0x50 -> append E
-GC horizon: 0x50
-LSN 0x60 -> append F
-```
-
-The algorithm will produce:
-
-```
-LSN 0x20 -> AB
-(drop all history below the earliest retain_lsn)
-LSN 0x40 -> ABCD
-(assume the cost of replaying 2 deltas is higher than storing the full image, we generate an image here)
-LSN 0x50 -> append E
-(replay one delta is cheap)
-LSN 0x60 -> append F
-(keep everything as-is above the GC horizon)
-```
-
-![](images/036-bottom-most-gc-compaction/05-btmgc-parent.svg)
-
-What happens is that we balance the space taken by each retain_lsn and the cost of replaying deltas during the bottom-most compaction process. This is controlled by a threshold. If `count(deltas) < $threshold`, the deltas will be retained. Otherwise, an image will be generated and the deltas will be dropped.
-
-In the example above, the `$threshold` is 2.
-
-## Child Branches with data: pull + partial images
-
-In the previous section we have shown how bottom-most compaction respects `retain_lsn` so that all data that was readable at branch creation remains readable. But branches can have data on their own, and that data can fall out of the branch’s PITR window. So, this section explains how we deal with that.
-
-We will run the same bottom-most compaction for these branches, to ensure the space amplification on the child branch is reasonable. 
-
-```
-branch_lsn: 0x20
-LSN 0x30 -> append P
-LSN 0x40 -> append Q
-LSN 0x50 -> append R
-GC horizon: 0x50
-LSN 0x60 -> append S
-```
-
-Note that bottom-most compaction happens on a per-timeline basis. When it processes this key, it only reads the history from LSN 0x30 without a base image. Therefore, on child branches, the bottom-most compaction process will make image creation decisions based on the same `count(deltas) < $threshold` criteria, and if it decides to create an image, the base image will be retrieved from the ancestor branch.
-
-```
-branch_lsn: 0x20
-LSN 0x50 -> ABPQR
-(we pull the image at LSN 0x20 from the ancestor branch to get AB, and then apply append PQ to the page; we replace the record at 0x40 with an image and drop the delta)
-GC horizon: 0x50
-LSN 0x60 -> append S
-```
-
-![](images/036-bottom-most-gc-compaction/06-btmgc-child.svg)
-
-Note that for child branches, we do not create image layers for the images when bottom-most compaction runs. Instead, we drop the 0x30/0x40/0x50 delta records and directly place the image ABPQR@0x50 into the delta layer, which serves as a sparse image layer. For child branches, if we create image layers, we will need to put all keys in the range into the image layer. This causes space bloat and slow compactions. In this proposal, the compaction process will only compact and process keys modified inside the child branch.
-
-# Result
-
-Bottom-most compaction ensures all garbage under the GC horizon gets collected right away (compared with “eventually” in the current algorithm). Meanwhile, it generates images at each of the retain_lsn to ensure branch reads are fast. As we make per-key decisions on whether to generate an image or not, the theoretical lower bound of the storage space we need to retain for a branch is lower than before.
-
-Before: min(sum(logs for each key), sum(image for each key)), for each partition — we always generate image layers on a key range
-
-After: sum(min(logs for each key, image for each key))
-
-# Compaction Trigger
-
-The bottom-most compaction can be automatically triggered. The goal of the trigger is that it should ensure a constant factor for write amplification. Say that the user write 1GB of WAL into the system, we should write 1GB x C data to S3. The legacy compaction algorithm does not have such a constant factor C. The data we write to S3 is quadratic to the logical size of the database (see [A Theoretical View of Neon Storage](https://www.notion.so/A-Theoretical-View-of-Neon-Storage-8d7ad7555b0c41b2a3597fa780911194?pvs=21)).
-
-We propose the following compaction trigger that generates a constant write amplification factor. Write amplification >= total writes to S3 / total user writes. We only analyze the write amplification caused by the bottom-most GC-compaction process, ignoring the legacy create image layers amplification.
-
-Given that we have ***X*** bytes of the delta layers above the GC horizon, ***A*** bytes of the delta layers intersecting with the GC horizon, ***B*** bytes of the delta layers below the GC horizon, and ***C*** bytes of the image layers below the GC horizon.
-
-The legacy GC + compaction loop will always keep ***A*** unchanged, reduce ***B and C*** when there are image layers covering the key range. This yields 0 write amplification (only file deletions) and extra ***B*** bytes of space.
-
-![](images/036-bottom-most-gc-compaction/09-btmgc-analysis-2.svg)
-
-The bottom-most compaction proposed here will split ***A*** into deltas above the GC horizon and below the GC horizon. Everything below the GC horizon will be image layers after the compaction (not considering branches). Therefore, this yields ***A+C*** extra write traffic each iteration, plus 0 extra space.
-
-![](images/036-bottom-most-gc-compaction/07-btmgc-analysis-1.svg)
-
-Also considering read amplification (below the GC horizon). When a read request reaches the GC horizon, the read amplification will be (A+B+C)/C=1+(A+B)/C. Reducing ***A*** and ***B*** can help reduce the read amplification below the GC horizon.
-
-The metrics-based trigger will wait until a point that space amplification is not that large and write amplification is not that large before the compaction gets triggered. The trigger is defined as **(A+B)/C ≥ 1 (or some other ratio)**.
-
-To reason about this trigger, consider the two cases:
-
-**Data Ingestion**
-
-User keeps ingesting data into the database, which indicates that WAL size roughly equals to the database logical size. The compaction gets triggered only when the newly-written WAL roughly equals to the current bottom-most image size (=X). Therefore, it’s triggered when the database size gets doubled. This is a reasonable amount of work. Write amplification is 2X/X=1 for the X amount of data written.
-
-![](images/036-bottom-most-gc-compaction/10-btmgc-analysis-3.svg)
-
-**Updates/Deletion**
-
-In this case, WAL size will be larger than the database logical size ***D***. The compaction gets triggered for every ***D*** bytes of WAL written. Therefore, for every ***D*** bytes of WAL, we rewrite the bottom-most layer, which produces an extra ***D*** bytes of write amplification. This incurs exactly 2x write amplification (by the write of D), 1.5x write amplification (if we count from the start of the process) and no space amplification. 
-
-![](images/036-bottom-most-gc-compaction/11-btmgc-analysis-4.svg)
-
-Note that here I try to reason that write amplification is a constant (i.e., the data we write to S3 is proportional to the data the user write). The main problem with the current legacy compaction algorithm is that write amplification is proportional to the database size.
-
-The next step is to optimize the write amplification above the GC horizon (i.e., change the image creation criteria, top-most compaction, or introduce tiered compaction), to ensure the write amplification of the whole system is a constant factor.
-
-20GB layers → +20GB layers → delete 20GB, need 40GB temporary space
-
-# Sub-Compactions
-
-The gc-compaction algorithm may take a long time and we need to split the job into multiple sub-compaction jobs.
-
-![](images/036-bottom-most-gc-compaction/13-job-split.svg)
-
-As in the figure, the auto-trigger schedules a compaction job covering the full keyspace below a specific LSN. In such case that we cannot finish compacting it in one run in a reasonable amount of time, the algorithm will vertically split it into multiple jobs (in this case, 5).
-
-Each gc-compaction job will create one level of delta layers and one flat level of image layers for each LSN. Those layers will be automatically split based on size, which means that if the sub-compaction job produces 1GB of deltas, it will produce 4 * 256MB delta layers. For those layers that is not fully contained within the sub-compaction job rectangles, it will be rewritten to only contain the keys outside of the key range.
-
-# Implementation
-
-The main implementation of gc-compaction is in `compaction.rs`.
-
-* `compact_with_gc`: The main loop of gc-compaction. It takes a rectangle range of the layer map and compact that specific range. It selects layers intersecting with the rectangle, downloads the layers, creates the k-merge iterator to read those layers in the key-lsn order, and decide which keys to keep or insert a reconstructed page. The process is the basic unit of a gc-compaction and is not interruptable. If the process gets preempted by L0 compaction, it has to be restarted from scratch. For layers overlaps with the rectangle but not fully inside, the main loop will also rewrite them so that the new layer (or two layers if both left and right ends are outside of the rectangle) has the same LSN range as the original one but only contain the keys outside of the compaction range.
-* `gc_compaction_split_jobs`: Splits a big gc-compaction job into sub-compactions based on heuristics in the layer map. The function looks at the layer map and splits the compaction job based on the size of the layers so that each compaction job only pulls ~4GB of layer files.
-* `generate_key_retention` and `KeyHistoryRetention`: Implements the algorithm described in the "basic idea" and "branch" chapter of this RFC. It takes a vector of history of a key (key-lsn-value) and decides which LSNs of the key to retain. If there are too many deltas between two retain_lsns, it will reconstruct the page and insert an image into the compaction result. Also, we implement `KeyHistoryRetention::verify` to ensure the generated result is not corrupted -- all retain_lsns and all LSNs above the gc-horizon should be accessible.
-* `GcCompactionQueue`: the automatic trigger implementation for gc-compaction. `GcCompactionQueue::iteration` is called at the end of the tenant compaction loop. It will then call `trigger_auto_compaction` to decide whether to trigger a gc-compaction job for this tenant. If yes, the compaction-job will be added to the compaction queue, and the queue will be slowly drained once there are no other compaction jobs running. gc-compaction has the lowest priority. If a sub-compaction job is not successful or gets preempted by L0 compaction (see limitations for reasons why a compaction job would fail), it will _not_ be retried.
-* Changes to `index_part.json`: we added a `last_completed_lsn` field to the index part for the auto-trigger to decide when to trigger a compaction.
-* Changes to the read path: when gc-compaction updates the layer map, all reads need to wait. See `gc_compaction_layer_update_lock` and comments in the code path for more information.
-
-Gc-compaction can also be scheduled over the HTTP API. Example:
-
-```
-curl 'localhost:9898/v1/tenant/:tenant_id/timeline/:timeline_id/compact?enhanced_gc_bottom_most_compaction=true&dry_run=true' -X PUT -H "Content-Type: application/json" -d '{"scheduled": true, "compact_key_range": { "start": "000000067F0000A0000002A1CF0100000000", "end": "000000067F0000A0000002A1D70100000000" } }'
-```
-
-The `dry_run` mode can be specified in the query string so that the compaction will go through all layers to estimate how much space can be saved without writing the compaction result into the layer map.
-
-The auto-trigger is controlled by tenant-level flag `gc_compaction_enabled`. If this is set to false, no gc-compaction will be automatically scheduled on this tenant (but manual trigger still works).
-
-# Next Steps
-
-There are still some limitations of gc-compaction itself that needs to be resolved and tested,
-
-- gc-compaction is currently only automatically triggered on root branches. We have not tested gc-compaction on child branches in staging.
-- gc-compaction will skip aux key regions because of the possible conflict with the assumption of aux file tombstones.
-- gc-compaction does not consider keyspaces at retain_lsns and only look at keys in the layers. This also causes us giving up some sub-compaction jobs because a key might have part of its history available due to traditional GC removing part of the history.
-- We limit gc-compaction to run over shards <= 150GB to avoid gc-compaction taking too much time blocking other compaction jobs. The sub-compaction split algorithm needs to be improved to be able to split vertically and horizontally. Also, we need to move the download layer process out of the compaction loop so that we don't block other compaction jobs for too long.
-- The compaction trigger always schedules gc-compaction from the lowest LSN to the gc-horizon. Currently we do not schedule compaction jobs that only selects layers in the middle. Allowing this could potentially reduce the number of layers read/write throughout the process.
-- gc-compaction will give up if there are too many layers to rewrite or if there are not enough disk space for the compaction.
-- gc-compaction sometimes fails with "no key produced during compaction", which means that all existing keys within the compaction range can be collected; but we don't have a way to write this information back to the layer map -- we cannot generate an empty image layer.
-- We limit the maximum size of deltas for a single key to 512MB. If above this size, gc-compaction will give up. This can be resolved by changing `generate_key_retention` to be a stream instead of requiring to collect all the key history.
-
-In the future,
-
-- Top-most compaction: ensure we always have an image coverage for the latest data (or near the latest data), so that reads will be fast at the latest LSN.
-- Tiered compaction on deltas: ensure read from any LSN is fast.
-- Per-timeline compaction → tenant-wide compaction?

docs/rfcs/images/036-bottom-most-gc-compaction/01-basic-idea.svg

@@ -1,135 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="82 284 863 375" width="863" height="375">
-  <defs/>
-  <g id="01-basic-idea" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>01-basic-idea</title>
-    <rect fill="white" x="82" y="284" width="863" height="375"/>
-    <g id="01-basic-idea_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_2">
-        <rect x="234" y="379.5" width="203.5" height="17.5" fill="white"/>
-        <rect x="234" y="379.5" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_3">
-        <rect x="453.5" y="379.5" width="203.5" height="17.5" fill="white"/>
-        <rect x="453.5" y="379.5" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_4">
-        <rect x="672.5" y="379.5" width="203.5" height="17.5" fill="white"/>
-        <rect x="672.5" y="379.5" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_5">
-        <rect x="234" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="234" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_6">
-        <rect x="375" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="375" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <rect x="516" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="516" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_8">
-        <rect x="657" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="657" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_9">
-        <rect x="798" y="288.5" width="78" height="77.5" fill="white"/>
-        <rect x="798" y="288.5" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_11">
-        <line x1="185.5" y1="326.75" x2="943.7734" y2="326.75" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_12">
-        <text transform="translate(87 318.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_13">
-        <text transform="translate(106.41 372.886)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.39" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="29132252e-19" y="28.447998" xml:space="preserve">at earlier LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_14">
-        <text transform="translate(121.92 289.578)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_15">
-        <path d="M 517.125 423.5 L 553.375 423.5 L 553.375 482 L 571.5 482 L 535.25 512 L 499 482 L 517.125 482 Z" fill="white"/>
-        <path d="M 517.125 423.5 L 553.375 423.5 L 553.375 482 L 571.5 482 L 535.25 512 L 499 482 L 517.125 482 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <rect x="234" y="599.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="234" y="599.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <rect x="453.5" y="599.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="453.5" y="599.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_24">
-        <rect x="672.5" y="599.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="672.5" y="599.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_23">
-        <rect x="234" y="533" width="127" height="52.974" fill="white"/>
-        <rect x="234" y="533" width="127" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_22">
-        <rect x="375" y="533" width="310.5" height="52.974" fill="white"/>
-        <rect x="375" y="533" width="310.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_21">
-        <rect x="702.5" y="533" width="173.5" height="52.974" fill="white"/>
-        <rect x="702.5" y="533" width="173.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_18">
-        <line x1="185.5" y1="607.724" x2="943.7734" y2="607.724" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_16">
-        <text transform="translate(121.92 538)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_27">
-        <text transform="translate(114.8 592.86)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="3488765e-18" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="4.01" y="28.447998" xml:space="preserve">at GC LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <rect x="243.06836" y="300" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(248.06836 301.068)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_30">
-        <rect x="243.06836" y="335.5" width="624.3633" height="17.5" fill="#c0ffff"/>
-        <text transform="translate(248.06836 336.568)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.89414" y="12" xml:space="preserve">Deltas below GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_32">
-        <rect x="243.06836" y="550.737" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(248.06836 551.805)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_33">
-        <rect x="304" y="630.474" width="485.5" height="28.447998" fill="#c0ffff"/>
-        <text transform="translate(309 637.016)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="63.095" y="12" xml:space="preserve">Deltas and image below GC Horizon gets garbage-collected</tspan>
-        </text>
-      </g>
-      <g id="Graphic_34">
-        <text transform="translate(576.5 444.0325)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="0" y="11" xml:space="preserve">WAL replay of deltas+image below GC Horizon</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="0" y="25.336" xml:space="preserve">Reshuffle deltas</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/03-retain-lsn.svg

@@ -1,141 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-104 215 863 335" width="863" height="335">
-  <defs>
-    <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -4 10 8" markerWidth="10" markerHeight="8" color="#7f8080">
-      <g>
-        <path d="M 8 0 L 0 -3 L 0 3 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
-      </g>
-    </marker>
-  </defs>
-  <g id="03-retain-lsn" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>03-retain-lsn</title>
-    <rect fill="white" x="-104" y="215" width="863" height="335"/>
-    <g id="03-retain-lsn_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_28">
-        <rect x="48" y="477" width="203.5" height="9.990005" fill="white"/>
-        <rect x="48" y="477" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_27">
-        <rect x="267.5" y="477" width="203.5" height="9.990005" fill="white"/>
-        <rect x="267.5" y="477" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <rect x="486.5" y="477" width="203.5" height="9.990005" fill="white"/>
-        <rect x="486.5" y="477" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-.5" y1="387.172" x2="757.7734" y2="387.172" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-99 378.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_31">
-        <rect x="48.25" y="410" width="203.5" height="9.990005" fill="white"/>
-        <rect x="48.25" y="410" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_30">
-        <rect x="267.75" y="410" width="203.5" height="9.990005" fill="white"/>
-        <rect x="267.75" y="410" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <rect x="486.75" y="410" width="203.5" height="9.990005" fill="white"/>
-        <rect x="486.75" y="410" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_34">
-        <rect x="48.25" y="431.495" width="113.75" height="34" fill="white"/>
-        <rect x="48.25" y="431.495" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_33">
-        <rect x="172.5" y="431.495" width="203.5" height="34" fill="white"/>
-        <rect x="172.5" y="431.495" width="203.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_32">
-        <rect x="386.5" y="431.495" width="303.5" height="34" fill="white"/>
-        <rect x="386.5" y="431.495" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_37">
-        <rect x="48" y="498.495" width="203.5" height="9.990005" fill="white"/>
-        <rect x="48" y="498.495" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_36">
-        <rect x="267.5" y="498.495" width="203.5" height="9.990005" fill="white"/>
-        <rect x="267.5" y="498.495" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_35">
-        <rect x="486.5" y="498.495" width="203.5" height="9.990005" fill="white"/>
-        <rect x="486.5" y="498.495" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_38">
-        <line x1="-10.48" y1="535.5395" x2="39.318294" y2="508.24794" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_39">
-        <text transform="translate(-96.984 526.3155)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 1</tspan>
-        </text>
-      </g>
-      <g id="Line_41">
-        <line x1="-10.48" y1="507.0915" x2="38.90236" y2="485.8992" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_40">
-        <text transform="translate(-96.984 497.8675)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 2</tspan>
-        </text>
-      </g>
-      <g id="Line_43">
-        <line x1="-10.48" y1="478.6435" x2="39.44267" y2="453.01616" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_42">
-        <text transform="translate(-96.984 469.4195)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 3</tspan>
-        </text>
-      </g>
-      <g id="Line_45">
-        <line x1="-10.48" y1="448.495" x2="39.65061" y2="419.90015" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_44">
-        <text transform="translate(-96.984 439.271)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 4</tspan>
-        </text>
-      </g>
-      <g id="Graphic_46">
-        <rect x="335.46477" y="215.5" width="353.4299" height="125.495" fill="white"/>
-        <rect x="335.46477" y="215.5" width="353.4299" height="125.495" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_48">
-        <text transform="translate(549.3766 317.547)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="6536993e-19" y="15" xml:space="preserve">Dependent Branch</tspan>
-        </text>
-      </g>
-      <g id="Graphic_50">
-        <text transform="translate(340.43824 317.547)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 3</tspan>
-        </text>
-      </g>
-      <g id="Line_57">
-        <line x1="323.90685" y1="248.8045" x2="714.9232" y2="248.8045" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_56">
-        <text transform="translate(165.91346 240.0805)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="35811354e-19" y="15" xml:space="preserve">Branch GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_58">
-        <rect x="493.9232" y="301.6405" width="107.45294" height="9.990005" fill="white"/>
-        <rect x="493.9232" y="301.6405" width="107.45294" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_59">
-        <text transform="translate(358.9232 277.276)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">Partial Image Coverage</tspan>
-        </text>
-      </g>
-      <g id="Graphic_60">
-        <rect x="354.1732" y="301.6405" width="107.45294" height="9.990005" fill="white"/>
-        <rect x="354.1732" y="301.6405" width="107.45294" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/05-btmgc-parent.svg

@@ -1,187 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-235 426 864 366" width="864" height="366">
-  <defs/>
-  <g id="05-btmgc-parent" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>05-btmgc-parent</title>
-    <rect fill="white" x="-235" y="426" width="864" height="366"/>
-    <g id="05-btmgc-parent_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_23">
-        <rect x="-83" y="510.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-83" y="510.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-78 516.178)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="51.714" y="11" xml:space="preserve">Append C@0x30</tspan>
-        </text>
-      </g>
-      <g id="Graphic_22">
-        <rect x="136.5" y="510.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="136.5" y="510.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_21">
-        <rect x="355.5" y="510.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="355.5" y="510.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-100.448" y1="459.224" x2="626.77344" y2="459.224" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-230 450.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_18">
-        <rect x="-82.75" y="426.748" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-82.75" y="426.748" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-77.75 432.776)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.602" y="11" xml:space="preserve">Append F@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <rect x="136.75" y="426.748" width="203.5" height="26.391998" fill="white"/>
-        <rect x="136.75" y="426.748" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_16">
-        <rect x="355.75" y="426.748" width="203.5" height="26.391998" fill="white"/>
-        <rect x="355.75" y="426.748" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_15">
-        <rect x="-82.75" y="464.645" width="113.75" height="34" fill="white"/>
-        <rect x="-82.75" y="464.645" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-77.75 467.309)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.505" y="11" xml:space="preserve">Append E@0x50</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="6.947" y="25.336" xml:space="preserve">Append D@0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_14">
-        <rect x="41.5" y="464.645" width="203.5" height="34" fill="white"/>
-        <rect x="41.5" y="464.645" width="203.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_13">
-        <rect x="255.5" y="464.645" width="303.5" height="34" fill="white"/>
-        <rect x="255.5" y="464.645" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_12">
-        <rect x="-83" y="548.047" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-83" y="548.047" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-78 554.075)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="26.796" y="11" xml:space="preserve">A@0x10, Append B@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_11">
-        <rect x="136.5" y="548.047" width="203.5" height="26.391998" fill="white"/>
-        <rect x="136.5" y="548.047" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_10">
-        <rect x="355.5" y="548.047" width="203.5" height="26.391998" fill="white"/>
-        <rect x="355.5" y="548.047" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_24">
-        <line x1="-104" y1="542" x2="610.5" y2="542" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <text transform="translate(-139.604 534.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <text transform="translate(-139.604 452.556)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_30">
-        <line x1="-100.448" y1="481.145" x2="614.052" y2="481.145" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <text transform="translate(-139.604 473.449)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Line_48">
-        <line x1="-99.448" y1="701.513" x2="627.77344" y2="701.513" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_47">
-        <text transform="translate(-229 692.789)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_46">
-        <rect x="-81.75" y="670.496" width="113.75" height="26.391998" fill="white"/>
-        <rect x="-81.75" y="670.496" width="113.75" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-76.75 676.524)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.727" y="11" xml:space="preserve">Append F@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_43">
-        <rect x="-81.75" y="708.393" width="113.75" height="34" fill="white"/>
-        <rect x="-81.75" y="708.393" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-76.75 718.225)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.505" y="11" xml:space="preserve">Append E@0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_37">
-        <line x1="-101" y1="777.2665" x2="613.5" y2="777.2665" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_36">
-        <text transform="translate(-138.604 769.7665)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_33">
-        <text transform="translate(-138.604 694.845)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_32">
-        <line x1="-99.448" y1="755.089" x2="615.052" y2="755.089" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_31">
-        <text transform="translate(-138.604 747.393)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_40">
-        <rect x="-82" y="770.909" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-82" y="770.909" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-77 770.7945)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="70.836" y="11" xml:space="preserve">AB@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_39">
-        <rect x="137.5" y="770.909" width="203.5" height="14.107002" fill="white"/>
-        <rect x="137.5" y="770.909" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_38">
-        <rect x="356.5" y="770.909" width="203.5" height="14.107002" fill="white"/>
-        <rect x="356.5" y="770.909" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_54">
-        <rect x="-81.75" y="748.5355" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-81.75" y="748.5355" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-76.75 748.421)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="62.28" y="11" xml:space="preserve">ABCD@0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_53">
-        <rect x="137.75" y="748.5355" width="203.5" height="14.107002" fill="white"/>
-        <rect x="137.75" y="748.5355" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_52">
-        <rect x="356.75" y="748.5355" width="203.5" height="14.107002" fill="white"/>
-        <rect x="356.75" y="748.5355" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_57">
-        <path d="M 211.32422 585 L 265.17578 585 L 265.17578 611.332 L 287.84375 611.332 L 238.25 633.117 L 188.65625 611.332 L 211.32422 611.332 Z" fill="white"/>
-        <path d="M 211.32422 585 L 265.17578 585 L 265.17578 611.332 L 287.84375 611.332 L 238.25 633.117 L 188.65625 611.332 L 211.32422 611.332 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_60">
-        <rect x="359" y="692.858" width="203.5" height="14.107002" fill="white"/>
-        <rect x="359" y="692.858" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_59">
-        <rect x="41.5" y="693.858" width="303" height="14.107002" fill="white"/>
-        <rect x="41.5" y="693.858" width="303" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/06-btmgc-child.svg

@@ -1,184 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-413 471 931 354" width="931" height="354">
-  <defs/>
-  <g id="06-btmgc-child" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>06-btmgc-child</title>
-    <rect fill="white" x="-413" y="471" width="931" height="354"/>
-    <g id="06-btmgc-child_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_47">
-        <rect x="-412" y="594.402" width="928" height="28.447998" fill="white"/>
-        <rect x="-412" y="594.402" width="928" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_46">
-        <rect x="-205" y="555.552" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-205" y="555.552" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-200 561.58)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.158" y="11" xml:space="preserve">Append P@0x30</tspan>
-        </text>
-      </g>
-      <g id="Graphic_45">
-        <rect x="14.5" y="555.552" width="203.5" height="26.391998" fill="white"/>
-        <rect x="14.5" y="555.552" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_44">
-        <rect x="233.5" y="555.552" width="203.5" height="26.391998" fill="white"/>
-        <rect x="233.5" y="555.552" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_43">
-        <line x1="-222.448" y1="504.724" x2="504.77344" y2="504.724" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_42">
-        <text transform="translate(-352 496)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_41">
-        <rect x="-204.75" y="472.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-204.75" y="472.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199.75 478.178)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.158" y="11" xml:space="preserve">Append S@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_40">
-        <rect x="14.75" y="472.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="14.75" y="472.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_39">
-        <rect x="233.75" y="472.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="233.75" y="472.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_38">
-        <rect x="-204.75" y="510.047" width="113.75" height="34" fill="white"/>
-        <rect x="-204.75" y="510.047" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199.75 512.711)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.061" y="11" xml:space="preserve">Append R@0x50</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="6.611" y="25.336" xml:space="preserve">Append Q@0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_37">
-        <rect x="-80.5" y="510.047" width="203.5" height="34" fill="white"/>
-        <rect x="-80.5" y="510.047" width="203.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_36">
-        <rect x="133.5" y="510.047" width="303.5" height="34" fill="white"/>
-        <rect x="133.5" y="510.047" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_33">
-        <text transform="translate(-261.604 498.056)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_30">
-        <line x1="-224" y1="607.9115" x2="490.5" y2="607.9115" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <text transform="translate(-261.604 600.4115)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <rect x="-205" y="601.554" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-205" y="601.554" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-200 601.4395)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="70.836" y="11" xml:space="preserve">AB@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_27">
-        <rect x="14.5" y="601.554" width="203.5" height="14.107002" fill="white"/>
-        <rect x="14.5" y="601.554" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <rect x="233.5" y="601.554" width="203.5" height="14.107002" fill="white"/>
-        <rect x="233.5" y="601.554" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <text transform="translate(-407 599.1875)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">Ancestor Branch</tspan>
-        </text>
-      </g>
-      <g id="Graphic_24">
-        <rect x="-411" y="795.46" width="928" height="28.447998" fill="white"/>
-        <rect x="-411" y="795.46" width="928" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-221.448" y1="755.528" x2="505.77344" y2="755.528" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-351 746.804)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_18">
-        <rect x="-203.75" y="723.579" width="203.25" height="26.391998" fill="white"/>
-        <rect x="-203.75" y="723.579" width="203.25" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-198.75 729.607)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.033" y="11" xml:space="preserve">Append S@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_10">
-        <text transform="translate(-260.604 748.86)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_7">
-        <line x1="-223" y1="808.9695" x2="491.5" y2="808.9695" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_6">
-        <text transform="translate(-260.604 801.4695)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_5">
-        <rect x="-204" y="802.612" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-204" y="802.612" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199 802.4975)" fill="#b1001c">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="#b1001c" x="70.836" y="11" xml:space="preserve">AB</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" y="11" xml:space="preserve">@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_4">
-        <rect x="15.5" y="802.612" width="203.5" height="14.107002" fill="white"/>
-        <rect x="15.5" y="802.612" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_3">
-        <rect x="234.5" y="802.612" width="203.5" height="14.107002" fill="white"/>
-        <rect x="234.5" y="802.612" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_2">
-        <text transform="translate(-406 800.2455)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">Ancestor Branch</tspan>
-        </text>
-      </g>
-      <g id="Graphic_48">
-        <path d="M 89.32422 639.081 L 143.17578 639.081 L 143.17578 665.413 L 165.84375 665.413 L 116.25 687.198 L 66.65625 665.413 L 89.32422 665.413 Z" fill="white"/>
-        <path d="M 89.32422 639.081 L 143.17578 639.081 L 143.17578 665.413 L 165.84375 665.413 L 116.25 687.198 L 66.65625 665.413 L 89.32422 665.413 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_49">
-        <rect x="-204" y="762.428" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-204" y="762.428" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199 768.456)" fill="#b1001c">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="#b1001c" x="58.278" y="11" xml:space="preserve">AB</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" y="11" xml:space="preserve">PQR@0x50</tspan>
-        </text>
-      </g>
-      <g id="Graphic_59">
-        <rect x="14.5" y="723.579" width="203.5" height="26.391998" fill="white"/>
-        <rect x="14.5" y="723.579" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_58">
-        <rect x="233.5" y="723.579" width="203.5" height="26.391998" fill="white"/>
-        <rect x="233.5" y="723.579" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_63">
-        <rect x="9" y="762.085" width="203.5" height="26.391998" fill="white"/>
-        <rect x="9" y="762.085" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_62">
-        <rect x="225" y="762.085" width="213" height="26.391998" fill="white"/>
-        <rect x="225" y="762.085" width="213" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/07-btmgc-analysis-1.svg

@@ -1,180 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-556 476 923 411" width="923" height="411">
-  <defs/>
-  <g id="07-btmgc-analysis-1" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>07-btmgc-analysis-1</title>
-    <rect fill="white" x="-556" y="476" width="923" height="411"/>
-    <g id="07-btmgc-analysis-1_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_85">
-        <rect x="-404" y="609.062" width="203.5" height="17.5" fill="white"/>
-        <rect x="-404" y="609.062" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_84">
-        <rect x="-184.5" y="609.062" width="203.5" height="17.5" fill="white"/>
-        <rect x="-184.5" y="609.062" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_83">
-        <rect x="34.5" y="609.062" width="203.5" height="17.5" fill="white"/>
-        <rect x="34.5" y="609.062" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_82">
-        <rect x="-404" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-404" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_81">
-        <rect x="-263" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-263" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_80">
-        <rect x="-122" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-122" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_79">
-        <rect x="19" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="19" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_78">
-        <rect x="160" y="479.922" width="78" height="77.5" fill="white"/>
-        <rect x="160" y="479.922" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_77">
-        <line x1="-452.5" y1="518.172" x2="251" y2="518.172" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_76">
-        <text transform="translate(-551 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_75">
-        <text transform="translate(-531.59 602.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.39" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="29132252e-19" y="28.447998" xml:space="preserve">at earlier LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_74">
-        <text transform="translate(-516.08 481)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_73">
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" fill="white"/>
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_72">
-        <rect x="-403.8" y="827.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="-403.8" y="827.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_71">
-        <rect x="-184.3" y="827.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="-184.3" y="827.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_70">
-        <rect x="34.7" y="827.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="34.7" y="827.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_69">
-        <rect x="-403.8" y="761" width="127" height="52.974" fill="white"/>
-        <rect x="-403.8" y="761" width="127" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_68">
-        <rect x="-262.8" y="761" width="310.5" height="52.974" fill="white"/>
-        <rect x="-262.8" y="761" width="310.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_67">
-        <rect x="64.7" y="761" width="173.5" height="52.974" fill="white"/>
-        <rect x="64.7" y="761" width="173.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_66">
-        <line x1="-452.3" y1="835.724" x2="251.2" y2="835.724" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_65">
-        <text transform="translate(-515.88 766)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_64">
-        <text transform="translate(-523 820.86)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="3488765e-18" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="4.01" y="28.447998" xml:space="preserve">at GC LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_63">
-        <rect x="-394.93164" y="491.422" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(-389.93164 492.49)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_62">
-        <rect x="-394.93164" y="526.922" width="624.3633" height="17.5" fill="#c0ffff"/>
-        <text transform="translate(-389.93164 527.99)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.89414" y="12" xml:space="preserve">Deltas below GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_61">
-        <rect x="-394.73164" y="778.737" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(-389.73164 779.805)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_60">
-        <rect x="-333.8" y="858.474" width="485.5" height="28.447998" fill="#c0ffff"/>
-        <text transform="translate(-328.8 865.016)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="63.095" y="12" xml:space="preserve">Deltas and image below GC Horizon gets garbage-collected</tspan>
-        </text>
-      </g>
-      <g id="Graphic_86">
-        <text transform="translate(263 499.724)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="32" fill="black" x="0" y="30" xml:space="preserve">size=A</tspan>
-        </text>
-      </g>
-      <g id="Line_87">
-        <line x1="260.87012" y1="479.068" x2="360.71387" y2="479.068" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_88">
-        <line x1="260.87012" y1="561" x2="360.71387" y2="561" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_89">
-        <rect x="-403.8" y="569" width="161.8" height="28.447998" fill="white"/>
-        <rect x="-403.8" y="569" width="161.8" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_90">
-        <rect x="-229.5" y="569.018" width="277.2" height="28.447998" fill="white"/>
-        <rect x="-229.5" y="569.018" width="277.2" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_91">
-        <rect x="64.7" y="569.018" width="173.5" height="28.447998" fill="white"/>
-        <rect x="64.7" y="569.018" width="173.5" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_92">
-        <line x1="262" y1="602" x2="361.84375" y2="602" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_93">
-        <line x1="263" y1="625.562" x2="362.84375" y2="625.562" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_94">
-        <text transform="translate(264.53787 562.276)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="32" fill="black" x="14210855e-21" y="30" xml:space="preserve">size=B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_95">
-        <text transform="translate(285.12 599.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="24" fill="black" x="0" y="23" xml:space="preserve">size=C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_98">
-        <text transform="translate(264.53787 773.772)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="8881784e-19" y="25" xml:space="preserve">A</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-      <g id="Graphic_97">
-        <text transform="translate(265.87013 815.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="6536993e-19" y="25" xml:space="preserve">B</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/08-optimization.svg

@@ -1,158 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-235 406 586 424" width="586" height="424">
-  <defs/>
-  <g id="08-optimization" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>08-optimization</title>
-    <rect fill="white" x="-235" y="406" width="586" height="424"/>
-    <g id="08-optimization_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_22">
-        <rect x="-100.448" y="509.902" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="509.902" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_21">
-        <rect x="118.552" y="509.902" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="509.902" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-101.79572" y1="420.322" x2="349.5" y2="420.322" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-230 411.598)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <rect x="-100.198" y="426.5" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.198" y="426.5" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_16">
-        <rect x="118.802" y="426.5" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.802" y="426.5" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_14">
-        <rect x="-100.198" y="464.397" width="108.25" height="34" fill="white"/>
-        <rect x="-100.198" y="464.397" width="108.25" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_13">
-        <rect x="18.552" y="464.397" width="303.5" height="34" fill="white"/>
-        <rect x="18.552" y="464.397" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_11">
-        <rect x="-100.448" y="547.799" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="547.799" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_10">
-        <rect x="118.552" y="547.799" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="547.799" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_24">
-        <line x1="-104" y1="542" x2="339.4011" y2="542" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <text transform="translate(-139.604 534.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Line_27">
-        <line x1="-101.79572" y1="459.098" x2="341.6054" y2="459.098" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <text transform="translate(-139.604 451.402)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <text transform="translate(-139.604 413.654)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x60</tspan>
-        </text>
-      </g>
-      <g id="Line_30">
-        <line x1="-101.79572" y1="481.145" x2="341.6054" y2="481.145" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <text transform="translate(-139.604 473.449)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_77">
-        <rect x="-100.448" y="765.19595" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="765.19595" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_76">
-        <rect x="118.552" y="765.19595" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="765.19595" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_75">
-        <line x1="-101.79572" y1="637.317" x2="349.5" y2="637.317" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_74">
-        <text transform="translate(-230 628.593)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_73">
-        <rect x="-100.198" y="681.794" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.198" y="681.794" width="203.5" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_72">
-        <rect x="118.802" y="681.794" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.802" y="681.794" width="203.5" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_71">
-        <rect x="-100.198" y="719.69096" width="108.25" height="34" fill="white"/>
-        <rect x="-100.198" y="719.69096" width="108.25" height="34" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_70">
-        <rect x="18.552" y="719.69096" width="303.5" height="34" fill="white"/>
-        <rect x="18.552" y="719.69096" width="303.5" height="34" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_69">
-        <rect x="-100.448" y="803.09295" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="803.09295" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_68">
-        <rect x="118.552" y="803.09295" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="803.09295" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_67">
-        <line x1="-104" y1="797.294" x2="339.4011" y2="797.294" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_66">
-        <text transform="translate(-139.604 789.794)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_63">
-        <text transform="translate(-139.604 630.649)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x70</tspan>
-        </text>
-      </g>
-      <g id="Line_62">
-        <line x1="-101.79572" y1="736.439" x2="341.6054" y2="736.439" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_61">
-        <text transform="translate(-139.604 728.743)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_79">
-        <rect x="-100.198" y="644.393" width="168.198" height="26.391998" fill="white"/>
-        <rect x="-100.198" y="644.393" width="168.198" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_78">
-        <rect x="80" y="644.393" width="242.302" height="26.391998" fill="white"/>
-        <rect x="80" y="644.393" width="242.302" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_81">
-        <line x1="-101.79572" y1="714.139" x2="341.6054" y2="714.139" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_80">
-        <text transform="translate(-139.604 706.443)" fill="#a5a5a5">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="#a5a5a5" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/09-btmgc-analysis-2.svg

@@ -1,184 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-562 479 876 429" width="876" height="429">
-  <defs/>
-  <g id="09-btmgc-analysis-2" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>09-btmgc-analysis-2</title>
-    <rect fill="white" x="-562" y="479" width="876" height="429"/>
-    <g id="09-btmgc-analysis-2_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_85">
-        <rect x="-404" y="622.386" width="203.5" height="17.5" fill="white"/>
-        <rect x="-404" y="622.386" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-399 621.912)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_84">
-        <rect x="-184.5" y="622.386" width="203.5" height="17.5" fill="white"/>
-        <rect x="-184.5" y="622.386" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-179.5 621.912)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_83">
-        <rect x="34.5" y="622.386" width="203.5" height="17.5" fill="white"/>
-        <rect x="34.5" y="622.386" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(39.5 621.912)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_82">
-        <rect x="-404" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-404" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-399 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_81">
-        <rect x="-263" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-263" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-258 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_80">
-        <rect x="-122" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-122" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-117 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_79">
-        <rect x="19" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="19" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(24 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_78">
-        <rect x="160" y="479.922" width="78" height="77.5" fill="white"/>
-        <rect x="160" y="479.922" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(165 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="28.816" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Line_77">
-        <line x1="-452.5" y1="518.172" x2="251" y2="518.172" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_76">
-        <text transform="translate(-551 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_73">
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" fill="white"/>
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_89">
-        <rect x="-403.8" y="582.324" width="161.8" height="28.447998" fill="white"/>
-        <rect x="-403.8" y="582.324" width="161.8" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-398.8 587.324)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="70.42" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_90">
-        <rect x="-229.5" y="582.342" width="277.2" height="28.447998" fill="white"/>
-        <rect x="-229.5" y="582.342" width="277.2" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-224.5 587.342)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="128.12" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_91">
-        <rect x="64.7" y="582.342" width="173.5" height="28.447998" fill="white"/>
-        <rect x="64.7" y="582.342" width="173.5" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(69.7 587.342)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="76.27" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_97">
-        <rect x="-403.8" y="564.842" width="490.8" height="12.157997" fill="white"/>
-        <rect x="-403.8" y="564.842" width="490.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-398.8 561.697)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="234.624" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_109">
-        <rect x="28.6" y="889.964" width="203.5" height="17.5" fill="white"/>
-        <rect x="28.6" y="889.964" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(33.6 889.49)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_108">
-        <rect x="-409.9" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="-409.9" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-404.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_107">
-        <rect x="-268.9" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="-268.9" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-263.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_106">
-        <rect x="-127.9" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="-127.9" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-122.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_105">
-        <rect x="13.1" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="13.1" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(18.1 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_104">
-        <rect x="154.1" y="747.5" width="78" height="77.5" fill="white"/>
-        <rect x="154.1" y="747.5" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(159.1 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="28.816" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Line_103">
-        <line x1="-458.4" y1="785.75" x2="245.1" y2="785.75" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_102">
-        <text transform="translate(-556.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_99">
-        <rect x="58.8" y="849.92" width="173.5" height="28.447998" fill="white"/>
-        <rect x="58.8" y="849.92" width="173.5" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(63.8 854.92)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="76.27" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_98">
-        <rect x="-409.7" y="832.42" width="490.8" height="12.157997" fill="white"/>
-        <rect x="-409.7" y="832.42" width="490.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-404.7 829.275)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="234.624" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_112">
-        <text transform="translate(273 797.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="6536993e-19" y="25" xml:space="preserve">B</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-      <g id="Graphic_113">
-        <text transform="translate(273 833.974)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="42277293e-20" y="25" xml:space="preserve">C</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/10-btmgc-analysis-3.svg

@@ -1,81 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-12 920 809 269" width="809" height="269">
-  <defs/>
-  <g id="10-btmgc-analysis-3" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>10-btmgc-analysis-3</title>
-    <rect fill="white" x="-12" y="920" width="809" height="269"/>
-    <g id="10-btmgc-analysis-3_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_13">
-        <rect x="433.7" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="433.7" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(438.7 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_12">
-        <rect x="503.7654" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="503.7654" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(508.7654 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_11">
-        <rect x="574.8318" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="574.8318" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(579.8318 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_10">
-        <rect x="645.3977" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="645.3977" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(650.3977 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Line_8">
-        <line x1="92" y1="934.276" x2="795.5" y2="934.276" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <text transform="translate(-6.500003 925.552)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_2">
-        <rect x="113.2" y="1033.92" width="321.3" height="12.157997" fill="white"/>
-        <rect x="113.2" y="1033.92" width="321.3" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(118.2 1030.775)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="150.762" y="15" xml:space="preserve">X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" fill="white"/>
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_27">
-        <line x1="93" y1="1164.224" x2="796.5" y2="1164.224" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <text transform="translate(-5.5000034 1155.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_25">
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" fill="white"/>
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(119 1170.355)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="306.564" y="15" xml:space="preserve">2X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_33">
-        <rect x="715.96355" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="715.96355" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(720.96355 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/11-btmgc-analysis-4.svg

@@ -1,81 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-12 920 809 269" width="809" height="269">
-  <defs/>
-  <g id="11-btmgc-analysis-4" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>11-btmgc-analysis-4</title>
-    <rect fill="white" x="-12" y="920" width="809" height="269"/>
-    <g id="11-btmgc-analysis-4_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_13">
-        <rect x="113" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="113" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(118 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_12">
-        <rect x="253" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="253" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(258 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_11">
-        <rect x="395" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="395" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(400 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_10">
-        <rect x="536" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="536" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(541 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_9">
-        <rect x="677" y="949" width="78" height="77.5" fill="white"/>
-        <rect x="677" y="949" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(682 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="14.584" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Line_8">
-        <line x1="92" y1="934.276" x2="795.5" y2="934.276" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <text transform="translate(-6.500003 925.552)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_2">
-        <rect x="113.2" y="1033.92" width="641.8" height="12.157997" fill="white"/>
-        <rect x="113.2" y="1033.92" width="641.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(118.2 1030.775)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="310.268" y="15" xml:space="preserve">D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" fill="white"/>
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_27">
-        <line x1="93" y1="1164.224" x2="796.5" y2="1164.224" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <text transform="translate(-5.5000034 1155.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_25">
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" fill="white"/>
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(119 1170.355)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="310.268" y="15" xml:space="preserve">D</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/13-job-split.svg

@@ -1,176 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg version="1.1" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" viewBox="210 271 870 514" width="870" height="514">
-  <defs/>
-  <g id="gc-compaction-split" stroke-dasharray="none" fill-opacity="1" stroke="none" fill="none" stroke-opacity="1">
-    <title>gc-compaction-split</title>
-    <rect fill="white" x="210" y="271" width="870" height="514"/>
-    <g id="gc-compaction-split_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_12">
-        <rect x="241" y="272" width="213" height="50.5" fill="white"/>
-        <rect x="241" y="272" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_11">
-        <rect x="468.72266" y="272" width="213" height="50.5" fill="white"/>
-        <rect x="468.72266" y="272" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_10">
-        <rect x="695.72266" y="272" width="213" height="50.5" fill="white"/>
-        <rect x="695.72266" y="272" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_9">
-        <rect x="241" y="337.3711" width="303.5" height="50.5" fill="white"/>
-        <rect x="241" y="337.3711" width="303.5" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_8">
-        <rect x="556.2617" y="337.3711" width="352.46094" height="50.5" fill="white"/>
-        <rect x="556.2617" y="337.3711" width="352.46094" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <rect x="241" y="402.7422" width="667.72266" height="50.5" fill="white"/>
-        <rect x="241" y="402.7422" width="667.72266" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_6">
-        <line x1="211" y1="355.5" x2="947.4961" y2="355.5" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_5">
-        <text transform="translate(952.4961 346.776)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">branch point</tspan>
-        </text>
-      </g>
-      <g id="Line_4">
-        <line x1="212" y1="438.5182" x2="948.4961" y2="438.5182" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_3">
-        <text transform="translate(953.4961 429.7942)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">last branch point</tspan>
-        </text>
-      </g>
-      <g id="Graphic_13">
-        <rect x="241" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(246 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 1</tspan>
-        </text>
-      </g>
-      <g id="Graphic_57">
-        <rect x="359" y="647.96484" width="551.72266" height="50.5" fill="white"/>
-        <rect x="359" y="647.96484" width="551.72266" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_54">
-        <rect x="359" y="517.22266" width="96" height="50.5" fill="white"/>
-        <rect x="359" y="517.22266" width="96" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_53">
-        <rect x="469.72266" y="517.22266" width="213" height="50.5" fill="white"/>
-        <rect x="469.72266" y="517.22266" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_52">
-        <rect x="696.72266" y="517.22266" width="213" height="50.5" fill="white"/>
-        <rect x="696.72266" y="517.22266" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_51">
-        <rect x="359" y="582.59375" width="186.5" height="50.5" fill="white"/>
-        <rect x="359" y="582.59375" width="186.5" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_50">
-        <rect x="557.2617" y="582.59375" width="352.46094" height="50.5" fill="white"/>
-        <rect x="557.2617" y="582.59375" width="352.46094" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_49">
-        <line x1="212" y1="600.72266" x2="948.4961" y2="600.72266" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_48">
-        <text transform="translate(953.4961 591.99866)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">branch point</tspan>
-        </text>
-      </g>
-      <g id="Line_47">
-        <line x1="213" y1="683.74084" x2="949.4961" y2="683.74084" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_46">
-        <text transform="translate(954.4961 675.01685)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">last branch point</tspan>
-        </text>
-      </g>
-      <g id="Graphic_63">
-        <rect x="376.72525" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(381.72525 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 2</tspan>
-        </text>
-      </g>
-      <g id="Graphic_64">
-        <rect x="511.39405" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(516.39405 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 3</tspan>
-        </text>
-      </g>
-      <g id="Graphic_65">
-        <rect x="646.06285" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(651.06285 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 4</tspan>
-        </text>
-      </g>
-      <g id="Graphic_66">
-        <rect x="780.73165" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(785.73165 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 5</tspan>
-        </text>
-      </g>
-      <g id="Graphic_56">
-        <rect x="243.5" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="243.5" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_55">
-        <rect x="243.5" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="243.5" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_68">
-        <rect x="379.22525" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="379.22525" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_67">
-        <rect x="379.22525" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="379.22525" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_70">
-        <rect x="514.22525" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="514.22525" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_69">
-        <rect x="514.22525" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="514.22525" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_72">
-        <rect x="649.22525" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="649.22525" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_71">
-        <rect x="649.22525" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="649.22525" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_74">
-        <rect x="785.23165" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="785.23165" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_73">
-        <rect x="785.23165" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="785.23165" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_78">
-        <rect x="241" y="731.3359" width="125.49101" height="27.26953" fill="#ccc"/>
-        <rect x="241" y="731.3359" width="125.49101" height="27.26953" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(246 735.7467)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="17.297502" y="15" xml:space="preserve">Delta Layer</tspan>
-        </text>
-      </g>
-      <g id="Graphic_79">
-        <rect x="241" y="766.759" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="241" y="766.759" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(246 766.285)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="13.737502" y="15" xml:space="preserve">Image Layer</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

libs/neon-shmem/Cargo.toml

@@ -1,13 +0,0 @@
-[package]
-name = "neon-shmem"
-version = "0.1.0"
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-thiserror.workspace = true
-nix.workspace=true
-workspace_hack = { version = "0.1", path = "../../workspace_hack" }
-
-[target.'cfg(target_os = "macos")'.dependencies]
-tempfile = "3.14.0"

libs/neon-shmem/src/lib.rs

@@ -1,418 +0,0 @@
-//! Shared memory utilities for neon communicator
-
-use std::num::NonZeroUsize;
-use std::os::fd::{AsFd, BorrowedFd, OwnedFd};
-use std::ptr::NonNull;
-use std::sync::atomic::{AtomicUsize, Ordering};
-
-use nix::errno::Errno;
-use nix::sys::mman::MapFlags;
-use nix::sys::mman::ProtFlags;
-use nix::sys::mman::mmap as nix_mmap;
-use nix::sys::mman::munmap as nix_munmap;
-use nix::unistd::ftruncate as nix_ftruncate;
-
-/// ShmemHandle represents a shared memory area that can be shared by processes over fork().
-/// Unlike shared memory allocated by Postgres, this area is resizable, up to 'max_size' that's
-/// specified at creation.
-///
-/// The area is backed by an anonymous file created with memfd_create(). The full address space for
-/// 'max_size' is reserved up-front with mmap(), but whenever you call [`ShmemHandle::set_size`],
-/// the underlying file is resized. Do not access the area beyond the current size. Currently, that
-/// will cause the file to be expanded, but we might use mprotect() etc. to enforce that in the
-/// future.
-pub struct ShmemHandle {
-    /// memfd file descriptor
-    fd: OwnedFd,
-
-    max_size: usize,
-
-    // Pointer to the beginning of the shared memory area. The header is stored there.
-    shared_ptr: NonNull<SharedStruct>,
-
-    // Pointer to the beginning of the user data
-    pub data_ptr: NonNull<u8>,
-}
-
-/// This is stored at the beginning in the shared memory area.
-struct SharedStruct {
-    max_size: usize,
-
-    /// Current size of the backing file. The high-order bit is used for the RESIZE_IN_PROGRESS flag
-    current_size: AtomicUsize,
-}
-
-const RESIZE_IN_PROGRESS: usize = 1 << 63;
-
-const HEADER_SIZE: usize = std::mem::size_of::<SharedStruct>();
-
-/// Error type returned by the ShmemHandle functions.
-#[derive(thiserror::Error, Debug)]
-#[error("{msg}: {errno}")]
-pub struct Error {
-    pub msg: String,
-    pub errno: Errno,
-}
-
-impl Error {
-    fn new(msg: &str, errno: Errno) -> Error {
-        Error {
-            msg: msg.to_string(),
-            errno,
-        }
-    }
-}
-
-impl ShmemHandle {
-    /// Create a new shared memory area. To communicate between processes, the processes need to be
-    /// fork()'d after calling this, so that the ShmemHandle is inherited by all processes.
-    ///
-    /// If the ShmemHandle is dropped, the memory is unmapped from the current process. Other
-    /// processes can continue using it, however.
-    pub fn new(name: &str, initial_size: usize, max_size: usize) -> Result<ShmemHandle, Error> {
-        // create the backing anonymous file.
-        let fd = create_backing_file(name)?;
-
-        Self::new_with_fd(fd, initial_size, max_size)
-    }
-
-    fn new_with_fd(
-        fd: OwnedFd,
-        initial_size: usize,
-        max_size: usize,
-    ) -> Result<ShmemHandle, Error> {
-        // We reserve the high-order bit for the RESIZE_IN_PROGRESS flag, and the actual size
-        // is a little larger than this because of the SharedStruct header. Make the upper limit
-        // somewhat smaller than that, because with anything close to that, you'll run out of
-        // memory anyway.
-        if max_size >= 1 << 48 {
-            panic!("max size {} too large", max_size);
-        }
-        if initial_size > max_size {
-            panic!("initial size {initial_size} larger than max size {max_size}");
-        }
-
-        // The actual initial / max size is the one given by the caller, plus the size of
-        // 'SharedStruct'.
-        let initial_size = HEADER_SIZE + initial_size;
-        let max_size = NonZeroUsize::new(HEADER_SIZE + max_size).unwrap();
-
-        // Reserve address space for it with mmap
-        //
-        // TODO: Use MAP_HUGETLB if possible
-        let start_ptr = unsafe {
-            nix_mmap(
-                None,
-                max_size,
-                ProtFlags::PROT_READ | ProtFlags::PROT_WRITE,
-                MapFlags::MAP_SHARED,
-                &fd,
-                0,
-            )
-        }
-        .map_err(|e| Error::new("mmap failed: {e}", e))?;
-
-        // Reserve space for the initial size
-        enlarge_file(fd.as_fd(), initial_size as u64)?;
-
-        // Initialize the header
-        let shared: NonNull<SharedStruct> = start_ptr.cast();
-        unsafe {
-            shared.write(SharedStruct {
-                max_size: max_size.into(),
-                current_size: AtomicUsize::new(initial_size),
-            })
-        };
-
-        // The user data begins after the header
-        let data_ptr = unsafe { start_ptr.cast().add(HEADER_SIZE) };
-
-        Ok(ShmemHandle {
-            fd,
-            max_size: max_size.into(),
-            shared_ptr: shared,
-            data_ptr,
-        })
-    }
-
-    // return reference to the header
-    fn shared(&self) -> &SharedStruct {
-        unsafe { self.shared_ptr.as_ref() }
-    }
-
-    /// Resize the shared memory area. 'new_size' must not be larger than the 'max_size' specified
-    /// when creating the area.
-    ///
-    /// This may only be called from one process/thread concurrently. We detect that case
-    /// and return an Error.
-    pub fn set_size(&self, new_size: usize) -> Result<(), Error> {
-        let new_size = new_size + HEADER_SIZE;
-        let shared = self.shared();
-
-        if new_size > self.max_size {
-            panic!(
-                "new size ({} is greater than max size ({})",
-                new_size, self.max_size
-            );
-        }
-        assert_eq!(self.max_size, shared.max_size);
-
-        // Lock the area by setting the bit in 'current_size'
-        //
-        // Ordering::Relaxed would probably be sufficient here, as we don't access any other memory
-        // and the posix_fallocate/ftruncate call is surely a synchronization point anyway. But
-        // since this is not performance-critical, better safe than sorry .
-        let mut old_size = shared.current_size.load(Ordering::Acquire);
-        loop {
-            if (old_size & RESIZE_IN_PROGRESS) != 0 {
-                return Err(Error::new(
-                    "concurrent resize detected",
-                    Errno::UnknownErrno,
-                ));
-            }
-            match shared.current_size.compare_exchange(
-                old_size,
-                new_size,
-                Ordering::Acquire,
-                Ordering::Relaxed,
-            ) {
-                Ok(_) => break,
-                Err(x) => old_size = x,
-            }
-        }
-
-        // Ok, we got the lock.
-        //
-        // NB: If anything goes wrong, we *must* clear the bit!
-        let result = {
-            use std::cmp::Ordering::{Equal, Greater, Less};
-            match new_size.cmp(&old_size) {
-                Less => nix_ftruncate(&self.fd, new_size as i64).map_err(|e| {
-                    Error::new("could not shrink shmem segment, ftruncate failed: {e}", e)
-                }),
-                Equal => Ok(()),
-                Greater => enlarge_file(self.fd.as_fd(), new_size as u64),
-            }
-        };
-
-        // Unlock
-        shared.current_size.store(
-            if result.is_ok() { new_size } else { old_size },
-            Ordering::Release,
-        );
-
-        result
-    }
-
-    /// Returns the current user-visible size of the shared memory segment.
-    ///
-    /// NOTE: a concurrent set_size() call can change the size at any time. It is the caller's
-    /// responsibility not to access the area beyond the current size.
-    pub fn current_size(&self) -> usize {
-        let total_current_size =
-            self.shared().current_size.load(Ordering::Relaxed) & !RESIZE_IN_PROGRESS;
-        total_current_size - HEADER_SIZE
-    }
-}
-
-impl Drop for ShmemHandle {
-    fn drop(&mut self) {
-        // SAFETY: The pointer was obtained from mmap() with the given size.
-        // We unmap the entire region.
-        let _ = unsafe { nix_munmap(self.shared_ptr.cast(), self.max_size) };
-        // The fd is dropped automatically by OwnedFd.
-    }
-}
-
-/// Create a "backing file" for the shared memory area. On Linux, use memfd_create(), to create an
-/// anonymous in-memory file. One macos, fall back to a regular file. That's good enough for
-/// development and testing, but in production we want the file to stay in memory.
-///
-/// disable 'unused_variables' warnings, because in the macos path, 'name' is unused.
-#[allow(unused_variables)]
-fn create_backing_file(name: &str) -> Result<OwnedFd, Error> {
-    #[cfg(not(target_os = "macos"))]
-    {
-        nix::sys::memfd::memfd_create(name, nix::sys::memfd::MFdFlags::empty())
-            .map_err(|e| Error::new("memfd_create failed: {e}", e))
-    }
-    #[cfg(target_os = "macos")]
-    {
-        let file = tempfile::tempfile().map_err(|e| {
-            Error::new(
-                "could not create temporary file to back shmem area: {e}",
-                nix::errno::Errno::from_raw(e.raw_os_error().unwrap_or(0)),
-            )
-        })?;
-        Ok(OwnedFd::from(file))
-    }
-}
-
-fn enlarge_file(fd: BorrowedFd, size: u64) -> Result<(), Error> {
-    // Use posix_fallocate() to enlarge the file. It reserves the space correctly, so that
-    // we don't get a segfault later when trying to actually use it.
-    #[cfg(not(target_os = "macos"))]
-    {
-        nix::fcntl::posix_fallocate(fd, 0, size as i64).map_err(|e| {
-            Error::new(
-                "could not grow shmem segment, posix_fallocate failed: {e}",
-                e,
-            )
-        })
-    }
-    // As a fallback on macos, which doesn't have posix_fallocate, use plain 'fallocate'
-    #[cfg(target_os = "macos")]
-    {
-        nix::unistd::ftruncate(fd, size as i64)
-            .map_err(|e| Error::new("could not grow shmem segment, ftruncate failed: {e}", e))
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    use nix::unistd::ForkResult;
-    use std::ops::Range;
-
-    /// check that all bytes in given range have the expected value.
-    fn assert_range(ptr: *const u8, expected: u8, range: Range<usize>) {
-        for i in range {
-            let b = unsafe { *(ptr.add(i)) };
-            assert_eq!(expected, b, "unexpected byte at offset {}", i);
-        }
-    }
-
-    /// Write 'b' to all bytes in the given range
-    fn write_range(ptr: *mut u8, b: u8, range: Range<usize>) {
-        unsafe { std::ptr::write_bytes(ptr.add(range.start), b, range.end - range.start) };
-    }
-
-    // simple single-process test of growing and shrinking
-    #[test]
-    fn test_shmem_resize() -> Result<(), Error> {
-        let max_size = 1024 * 1024;
-        let init_struct = ShmemHandle::new("test_shmem_resize", 0, max_size)?;
-
-        assert_eq!(init_struct.current_size(), 0);
-
-        // Initial grow
-        let size1 = 10000;
-        init_struct.set_size(size1).unwrap();
-        assert_eq!(init_struct.current_size(), size1);
-
-        // Write some data
-        let data_ptr = init_struct.data_ptr.as_ptr();
-        write_range(data_ptr, 0xAA, 0..size1);
-        assert_range(data_ptr, 0xAA, 0..size1);
-
-        // Shrink
-        let size2 = 5000;
-        init_struct.set_size(size2).unwrap();
-        assert_eq!(init_struct.current_size(), size2);
-
-        // Grow again
-        let size3 = 20000;
-        init_struct.set_size(size3).unwrap();
-        assert_eq!(init_struct.current_size(), size3);
-
-        // Try to read it. The area that was shrunk and grown again should read as all zeros now
-        assert_range(data_ptr, 0xAA, 0..5000);
-        assert_range(data_ptr, 0, 5000..size1);
-
-        // Try to grow beyond max_size
-        //let size4 = max_size + 1;
-        //assert!(init_struct.set_size(size4).is_err());
-
-        // Dropping init_struct should unmap the memory
-        drop(init_struct);
-
-        Ok(())
-    }
-
-    /// This is used in tests to coordinate between test processes. It's like std::sync::Barrier,
-    /// but is stored in the shared memory area and works across processes. It's implemented by
-    /// polling, because e.g. standard rust mutexes are not guaranteed to work across processes.
-    struct SimpleBarrier {
-        num_procs: usize,
-        count: AtomicUsize,
-    }
-
-    impl SimpleBarrier {
-        unsafe fn init(ptr: *mut SimpleBarrier, num_procs: usize) {
-            unsafe {
-                *ptr = SimpleBarrier {
-                    num_procs,
-                    count: AtomicUsize::new(0),
-                }
-            }
-        }
-
-        pub fn wait(&self) {
-            let old = self.count.fetch_add(1, Ordering::Relaxed);
-
-            let generation = old / self.num_procs;
-
-            let mut current = old + 1;
-            while current < (generation + 1) * self.num_procs {
-                std::thread::sleep(std::time::Duration::from_millis(10));
-                current = self.count.load(Ordering::Relaxed);
-            }
-        }
-    }
-
-    #[test]
-    fn test_multi_process() {
-        // Initialize
-        let max_size = 1_000_000_000_000;
-        let init_struct = ShmemHandle::new("test_multi_process", 0, max_size).unwrap();
-        let ptr = init_struct.data_ptr.as_ptr();
-
-        // Store the SimpleBarrier in the first 1k of the area.
-        init_struct.set_size(10000).unwrap();
-        let barrier_ptr: *mut SimpleBarrier = unsafe {
-            ptr.add(ptr.align_offset(std::mem::align_of::<SimpleBarrier>()))
-                .cast()
-        };
-        unsafe { SimpleBarrier::init(barrier_ptr, 2) };
-        let barrier = unsafe { barrier_ptr.as_ref().unwrap() };
-
-        // Fork another test process. The code after this runs in both processes concurrently.
-        let fork_result = unsafe { nix::unistd::fork().unwrap() };
-
-        // In the parent, fill bytes between 1000..2000. In the child, between 2000..3000
-        if fork_result.is_parent() {
-            write_range(ptr, 0xAA, 1000..2000);
-        } else {
-            write_range(ptr, 0xBB, 2000..3000);
-        }
-        barrier.wait();
-        // Verify the contents. (in both processes)
-        assert_range(ptr, 0xAA, 1000..2000);
-        assert_range(ptr, 0xBB, 2000..3000);
-
-        // Grow, from the child this time
-        let size = 10_000_000;
-        if !fork_result.is_parent() {
-            init_struct.set_size(size).unwrap();
-        }
-        barrier.wait();
-
-        // make some writes at the end
-        if fork_result.is_parent() {
-            write_range(ptr, 0xAA, (size - 10)..size);
-        } else {
-            write_range(ptr, 0xBB, (size - 20)..(size - 10));
-        }
-        barrier.wait();
-
-        // Verify the contents. (This runs in both processes)
-        assert_range(ptr, 0, (size - 1000)..(size - 20));
-        assert_range(ptr, 0xBB, (size - 20)..(size - 10));
-        assert_range(ptr, 0xAA, (size - 10)..size);
-
-        if let ForkResult::Parent { child } = fork_result {
-            nix::sys::wait::waitpid(child, None).unwrap();
-        }
-    }
-}

libs/pageserver_api/src/config.rs

@@ -305,7 +305,6 @@ impl From<OtelExporterProtocol> for tracing_utils::Protocol {
 pub struct TimelineImportConfig {
     pub import_job_concurrency: NonZeroUsize,
     pub import_job_soft_size_limit: NonZeroUsize,
-    pub import_job_checkpoint_threshold: NonZeroUsize,
 }
 
 pub mod statvfs {
@@ -640,15 +639,23 @@ impl Default for ConfigToml {
             tenant_config: TenantConfigToml::default(),
             no_sync: None,
             wal_receiver_protocol: DEFAULT_WAL_RECEIVER_PROTOCOL,
-            page_service_pipelining: PageServicePipeliningConfig::Pipelined(
-                PageServicePipeliningConfigPipelined {
+            page_service_pipelining: if !cfg!(test) {
+                PageServicePipeliningConfig::Serial
+            } else {
+                // Do not turn this into the default until scattered reads have been
+                // validated and rolled-out fully.
+                PageServicePipeliningConfig::Pipelined(PageServicePipeliningConfigPipelined {
                     max_batch_size: NonZeroUsize::new(32).unwrap(),
                     execution: PageServiceProtocolPipelinedExecutionStrategy::ConcurrentFutures,
                     batching: PageServiceProtocolPipelinedBatchingStrategy::ScatteredLsn,
-                },
-            ),
-            get_vectored_concurrent_io: GetVectoredConcurrentIo::SidecarTask,
-            enable_read_path_debugging: if cfg!(feature = "testing") {
+                })
+            },
+            get_vectored_concurrent_io: if !cfg!(test) {
+                GetVectoredConcurrentIo::Sequential
+            } else {
+                GetVectoredConcurrentIo::SidecarTask
+            },
+            enable_read_path_debugging: if cfg!(test) || cfg!(feature = "testing") {
                 Some(true)
             } else {
                 None
@@ -662,7 +669,6 @@ impl Default for ConfigToml {
             timeline_import_config: TimelineImportConfig {
                 import_job_concurrency: NonZeroUsize::new(128).unwrap(),
                 import_job_soft_size_limit: NonZeroUsize::new(1024 * 1024 * 1024).unwrap(),
-                import_job_checkpoint_threshold: NonZeroUsize::new(128).unwrap(),
             },
         }
     }

libs/pageserver_api/src/key.rs

@@ -910,11 +910,6 @@ impl Key {
         self.field1 == 0x00 && self.field4 != 0 && self.field6 != 0xffffffff
     }
 
-    #[inline(always)]
-    pub fn is_rel_block_of_rel(&self, rel: Oid) -> bool {
-        self.is_rel_block_key() && self.field4 == rel
-    }
-
     #[inline(always)]
     pub fn is_rel_dir_key(&self) -> bool {
         self.field1 == 0x00

libs/pageserver_api/src/models.rs

@@ -336,30 +336,14 @@ impl TimelineCreateRequest {
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
 pub enum ShardImportStatus {
-    InProgress(Option<ShardImportProgress>),
+    InProgress,
     Done,
     Error(String),
 }
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub enum ShardImportProgress {
-    V1(ShardImportProgressV1),
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub struct ShardImportProgressV1 {
-    /// Total number of jobs in the import plan
-    pub jobs: usize,
-    /// Number of jobs completed
-    pub completed: usize,
-    /// Hash of the plan
-    pub import_plan_hash: u64,
-}
-
 impl ShardImportStatus {
     pub fn is_terminal(&self) -> bool {
         match self {
-            ShardImportStatus::InProgress(_) => false,
+            ShardImportStatus::InProgress => false,
             ShardImportStatus::Done | ShardImportStatus::Error(_) => true,
         }
     }
@@ -1819,6 +1803,7 @@ pub struct TopTenantShardsResponse {
 }
 
 pub mod virtual_file {
+    use std::sync::LazyLock;
 
     #[derive(
         Copy,
@@ -1866,7 +1851,15 @@ pub mod virtual_file {
 
     impl IoMode {
         pub fn preferred() -> Self {
-            IoMode::DirectRw
+            // The default behavior when running Rust unit tests without any further
+            // flags is to use the newest behavior (DirectRw).
+            // The CI uses the environment variable to unit tests for all different modes.
+            // NB: the Python regression & perf tests have their own defaults management
+            // that writes pageserver.toml; they do not use this variable.
+            static ENV_OVERRIDE: LazyLock<Option<IoMode>> = LazyLock::new(|| {
+                utils::env::var_serde_json_string("NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IO_MODE")
+            });
+            ENV_OVERRIDE.unwrap_or(IoMode::DirectRw)
         }
     }
 

libs/pageserver_api/src/upcall_api.rs

@@ -4,7 +4,6 @@
 //! See docs/rfcs/025-generation-numbers.md
 
 use serde::{Deserialize, Serialize};
-use utils::generation::Generation;
 use utils::id::{NodeId, TimelineId};
 
 use crate::controller_api::NodeRegisterRequest;
@@ -64,17 +63,9 @@ pub struct ValidateResponseTenant {
     pub valid: bool,
 }
 
-#[derive(Serialize, Deserialize)]
-pub struct TimelineImportStatusRequest {
-    pub tenant_shard_id: TenantShardId,
-    pub timeline_id: TimelineId,
-    pub generation: Generation,
-}
-
 #[derive(Serialize, Deserialize)]
 pub struct PutTimelineImportStatusRequest {
     pub tenant_shard_id: TenantShardId,
     pub timeline_id: TimelineId,
     pub status: ShardImportStatus,
-    pub generation: Generation,
 }

libs/utils/src/crashsafe.rs

@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 use std::fs::{self, File};
 use std::io::{self, Write};
-use std::os::fd::AsFd;
+use std::os::fd::AsRawFd;
 
 use camino::{Utf8Path, Utf8PathBuf};
 
@@ -210,13 +210,13 @@ pub fn overwrite(
 
 /// Syncs the filesystem for the given file descriptor.
 #[cfg_attr(target_os = "macos", allow(unused_variables))]
-pub fn syncfs(fd: impl AsFd) -> anyhow::Result<()> {
+pub fn syncfs(fd: impl AsRawFd) -> anyhow::Result<()> {
     // Linux guarantees durability for syncfs.
     // POSIX doesn't have syncfs, and further does not actually guarantee durability of sync().
     #[cfg(target_os = "linux")]
     {
         use anyhow::Context;
-        nix::unistd::syncfs(fd).context("syncfs")?;
+        nix::unistd::syncfs(fd.as_raw_fd()).context("syncfs")?;
     }
     #[cfg(target_os = "macos")]
     {

libs/utils/src/fs_ext/rename_noreplace.rs

@@ -11,9 +11,9 @@ pub fn rename_noreplace<P1: ?Sized + NixPath, P2: ?Sized + NixPath>(
         #[cfg(all(target_os = "linux", target_env = "gnu"))]
         {
             nix::fcntl::renameat2(
-                nix::fcntl::AT_FDCWD,
+                None,
                 src,
-                nix::fcntl::AT_FDCWD,
+                None,
                 dst,
                 nix::fcntl::RenameFlags::RENAME_NOREPLACE,
             )

libs/utils/src/lock_file.rs

@@ -1,6 +1,6 @@
 //! A module to create and read lock files.
 //!
-//! File locking is done using [`nix::fcntl::Flock`] exclusive locks.
+//! File locking is done using [`fcntl::flock`] exclusive locks.
 //! The only consumer of this module is currently
 //! [`pid_file`](crate::pid_file). See the module-level comment
 //! there for potential pitfalls with lock files that are used
@@ -9,25 +9,26 @@
 use std::fs;
 use std::io::{Read, Write};
 use std::ops::Deref;
+use std::os::unix::prelude::AsRawFd;
 
 use anyhow::Context;
 use camino::{Utf8Path, Utf8PathBuf};
 use nix::errno::Errno::EAGAIN;
-use nix::fcntl::{Flock, FlockArg};
+use nix::fcntl;
 
 use crate::crashsafe;
 
-/// A handle to an open and flocked, but not-yet-written lock file.
+/// A handle to an open and unlocked, but not-yet-written lock file.
 /// Returned by [`create_exclusive`].
 #[must_use]
 pub struct UnwrittenLockFile {
     path: Utf8PathBuf,
-    file: Flock<fs::File>,
+    file: fs::File,
 }
 
 /// Returned by [`UnwrittenLockFile::write_content`].
 #[must_use]
-pub struct LockFileGuard(Flock<fs::File>);
+pub struct LockFileGuard(fs::File);
 
 impl Deref for LockFileGuard {
     type Target = fs::File;
@@ -66,14 +67,17 @@ pub fn create_exclusive(lock_file_path: &Utf8Path) -> anyhow::Result<UnwrittenLo
         .open(lock_file_path)
         .context("open lock file")?;
 
-    let res = Flock::lock(lock_file, FlockArg::LockExclusiveNonblock);
+    let res = fcntl::flock(
+        lock_file.as_raw_fd(),
+        fcntl::FlockArg::LockExclusiveNonblock,
+    );
     match res {
-        Ok(lock_file) => Ok(UnwrittenLockFile {
+        Ok(()) => Ok(UnwrittenLockFile {
             path: lock_file_path.to_owned(),
             file: lock_file,
         }),
-        Err((_, EAGAIN)) => anyhow::bail!("file is already locked"),
-        Err((_, e)) => Err(e).context("flock error"),
+        Err(EAGAIN) => anyhow::bail!("file is already locked"),
+        Err(e) => Err(e).context("flock error"),
     }
 }
 
@@ -101,37 +105,32 @@ pub enum LockFileRead {
 /// Check the [`LockFileRead`] variants for details.
 pub fn read_and_hold_lock_file(path: &Utf8Path) -> anyhow::Result<LockFileRead> {
     let res = fs::OpenOptions::new().read(true).open(path);
-    let lock_file = match res {
+    let mut lock_file = match res {
         Ok(f) => f,
         Err(e) => match e.kind() {
             std::io::ErrorKind::NotFound => return Ok(LockFileRead::NotExist),
             _ => return Err(e).context("open lock file"),
         },
     };
-    let res = Flock::lock(lock_file, FlockArg::LockExclusiveNonblock);
+    let res = fcntl::flock(
+        lock_file.as_raw_fd(),
+        fcntl::FlockArg::LockExclusiveNonblock,
+    );
     // We need the content regardless of lock success / failure.
     // But, read it after flock so that, if it succeeded, the content is consistent.
+    let mut content = String::new();
+    lock_file
+        .read_to_string(&mut content)
+        .context("read lock file")?;
     match res {
-        Ok(mut locked_file) => {
-            let mut content = String::new();
-            locked_file
-                .read_to_string(&mut content)
-                .context("read lock file")?;
-            Ok(LockFileRead::NotHeldByAnyProcess(
-                LockFileGuard(locked_file),
-                content,
-            ))
-        }
-        Err((mut not_locked_file, EAGAIN)) => {
-            let mut content = String::new();
-            not_locked_file
-                .read_to_string(&mut content)
-                .context("read lock file")?;
-            Ok(LockFileRead::LockedByOtherProcess {
-                not_locked_file,
-                content,
-            })
-        }
-        Err((_, e)) => Err(e).context("flock error"),
+        Ok(()) => Ok(LockFileRead::NotHeldByAnyProcess(
+            LockFileGuard(lock_file),
+            content,
+        )),
+        Err(EAGAIN) => Ok(LockFileRead::LockedByOtherProcess {
+            not_locked_file: lock_file,
+            content,
+        }),
+        Err(e) => Err(e).context("flock error"),
     }
 }

libs/utils/src/tracing_span_assert.rs

@@ -127,12 +127,12 @@ macro_rules! __check_fields_present {
 
             match check_fields_present0($extractors) {
                 Ok(FoundEverything) => Ok(()),
-                Ok(Unconfigured) if cfg!(feature = "testing") => {
+                Ok(Unconfigured) if cfg!(test) => {
                     // allow unconfigured in tests
                     Ok(())
                 },
                 Ok(Unconfigured) => {
-                    panic!(r#"utils::tracing_span_assert: outside of #[cfg(feature = "testing")] expected tracing to be configured with tracing_error::ErrorLayer"#)
+                    panic!("utils::tracing_span_assert: outside of #[cfg(test)] expected tracing to be configured with tracing_error::ErrorLayer")
                 },
                 Err(missing) => Err(missing)
             }

pageserver/Cargo.toml

@@ -96,7 +96,6 @@ strum.workspace = true
 strum_macros.workspace = true
 wal_decoder.workspace = true
 smallvec.workspace = true
-twox-hash.workspace = true
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs.workspace = true

pageserver/pagebench/src/cmd/getpage_latest_lsn.rs

@@ -65,9 +65,6 @@ pub(crate) struct Args {
     #[clap(long, default_value = "1")]
     queue_depth: NonZeroUsize,
 
-    #[clap(long)]
-    only_relnode: Option<u32>,
-
     targets: Option<Vec<TenantTimelineId>>,
 }
 
@@ -209,12 +206,7 @@ async fn main_impl(
                     for r in partitioning.keys.ranges.iter() {
                         let mut i = r.start;
                         while i != r.end {
-                            let mut include = true;
-                            include &= i.is_rel_block_key();
-                            if let Some(only_relnode) = args.only_relnode {
-                                include &= i.is_rel_block_of_rel(only_relnode);
-                            }
-                            if include {
+                            if i.is_rel_block_key() {
                                 filtered.add_key(i);
                             }
                             i = i.next();

pageserver/src/controller_upcall_client.rs

@@ -7,7 +7,7 @@ use pageserver_api::models::ShardImportStatus;
 use pageserver_api::shard::TenantShardId;
 use pageserver_api::upcall_api::{
     PutTimelineImportStatusRequest, ReAttachRequest, ReAttachResponse, ReAttachResponseTenant,
-    TimelineImportStatusRequest, ValidateRequest, ValidateRequestTenant, ValidateResponse,
+    ValidateRequest, ValidateRequestTenant, ValidateResponse,
 };
 use reqwest::Certificate;
 use serde::Serialize;
@@ -51,15 +51,13 @@ pub trait StorageControllerUpcallApi {
         &self,
         tenant_shard_id: TenantShardId,
         timeline_id: TimelineId,
-        generation: Generation,
         status: ShardImportStatus,
     ) -> impl Future<Output = Result<(), RetryForeverError>> + Send;
     fn get_timeline_import_status(
         &self,
         tenant_shard_id: TenantShardId,
         timeline_id: TimelineId,
-        generation: Generation,
-    ) -> impl Future<Output = Result<ShardImportStatus, RetryForeverError>> + Send;
+    ) -> impl Future<Output = Result<Option<ShardImportStatus>, RetryForeverError>> + Send;
 }
 
 impl StorageControllerUpcallClient {
@@ -104,7 +102,6 @@ impl StorageControllerUpcallClient {
         &self,
         url: &url::Url,
         request: R,
-        method: reqwest::Method,
     ) -> Result<T, RetryForeverError>
     where
         R: Serialize,
@@ -114,7 +111,7 @@ impl StorageControllerUpcallClient {
             || async {
                 let response = self
                     .http_client
-                    .request(method.clone(), url.clone())
+                    .post(url.clone())
                     .json(&request)
                     .send()
                     .await?;
@@ -223,9 +220,7 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
             register: register.clone(),
         };
 
-        let response: ReAttachResponse = self
-            .retry_http_forever(&url, request, reqwest::Method::POST)
-            .await?;
+        let response: ReAttachResponse = self.retry_http_forever(&url, request).await?;
         tracing::info!(
             "Received re-attach response with {} tenants (node {}, register: {:?})",
             response.tenants.len(),
@@ -278,9 +273,7 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
                 return Err(RetryForeverError::ShuttingDown);
             }
 
-            let response: ValidateResponse = self
-                .retry_http_forever(&url, request, reqwest::Method::POST)
-                .await?;
+            let response: ValidateResponse = self.retry_http_forever(&url, request).await?;
             for rt in response.tenants {
                 result.insert(rt.id, rt.valid);
             }
@@ -299,7 +292,6 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
         &self,
         tenant_shard_id: TenantShardId,
         timeline_id: TimelineId,
-        generation: Generation,
         status: ShardImportStatus,
     ) -> Result<(), RetryForeverError> {
         let url = self
@@ -310,12 +302,10 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
         let request = PutTimelineImportStatusRequest {
             tenant_shard_id,
             timeline_id,
-            generation,
             status,
         };
 
-        self.retry_http_forever(&url, request, reqwest::Method::POST)
-            .await
+        self.retry_http_forever(&url, request).await
     }
 
     #[tracing::instrument(skip_all)] // so that warning logs from retry_http_forever have context
@@ -323,22 +313,33 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
         &self,
         tenant_shard_id: TenantShardId,
         timeline_id: TimelineId,
-        generation: Generation,
-    ) -> Result<ShardImportStatus, RetryForeverError> {
+    ) -> Result<Option<ShardImportStatus>, RetryForeverError> {
         let url = self
             .base_url
-            .join("timeline_import_status")
+            .join(format!("timeline_import_status/{}/{}", tenant_shard_id, timeline_id).as_str())
             .expect("Failed to build path");
 
-        let request = TimelineImportStatusRequest {
-            tenant_shard_id,
-            timeline_id,
-            generation,
-        };
+        Ok(backoff::retry(
+            || async {
+                let response = self.http_client.get(url.clone()).send().await?;
 
-        let response: ShardImportStatus = self
-            .retry_http_forever(&url, request, reqwest::Method::GET)
-            .await?;
-        Ok(response)
+                if let Err(err) = response.error_for_status_ref() {
+                    if matches!(err.status(), Some(reqwest::StatusCode::NOT_FOUND)) {
+                        return Ok(None);
+                    } else {
+                        return Err(err);
+                    }
+                }
+                response.json::<ShardImportStatus>().await.map(Some)
+            },
+            |_| false,
+            3,
+            u32::MAX,
+            "storage controller upcall",
+            &self.cancel,
+        )
+        .await
+        .ok_or(RetryForeverError::ShuttingDown)?
+        .expect("We retry forever, this should never be reached"))
     }
 }

pageserver/src/deletion_queue.rs

@@ -793,7 +793,6 @@ mod test {
             &self,
             _tenant_shard_id: TenantShardId,
             _timeline_id: TimelineId,
-            _generation: Generation,
             _status: pageserver_api::models::ShardImportStatus,
         ) -> Result<(), RetryForeverError> {
             unimplemented!()
@@ -803,8 +802,7 @@ mod test {
             &self,
             _tenant_shard_id: TenantShardId,
             _timeline_id: TimelineId,
-            _generation: Generation,
-        ) -> Result<ShardImportStatus, RetryForeverError> {
+        ) -> Result<Option<ShardImportStatus>, RetryForeverError> {
             unimplemented!()
         }
     }

pageserver/src/pgdatadir_mapping.rs

@@ -50,7 +50,9 @@ use crate::span::{
     debug_assert_current_span_has_tenant_and_timeline_id_no_shard_id,
 };
 use crate::tenant::storage_layer::IoConcurrency;
-use crate::tenant::timeline::{GetVectoredError, VersionedKeySpaceQuery};
+use crate::tenant::timeline::{
+    GetVectoredError, MissingKeyError, RelSizeCacheEntry, VersionedKeySpaceQuery,
+};
 
 /// Max delta records appended to the AUX_FILES_KEY (for aux v1). The write path will write a full image once this threshold is reached.
 pub const MAX_AUX_FILE_DELTAS: usize = 1024;
@@ -470,8 +472,26 @@ impl Timeline {
             ));
         }
 
-        if let Some(nblocks) = self.get_cached_rel_size(&tag, version.get_lsn()) {
-            return Ok(nblocks);
+        if let Some(entry) = self.get_cached_rel_size(&tag, version.get_lsn()) {
+            match entry {
+                RelSizeCacheEntry::Present(nblocks) => {
+                    return Ok(nblocks);
+                }
+                RelSizeCacheEntry::Truncated => {
+                    let key = rel_size_to_key(tag);
+                    return Err(PageReconstructError::MissingKey(Box::new(
+                        MissingKeyError {
+                            keyspace: KeySpace::single(key..key.next()),
+                            shard: self.get_shard_identity().number,
+                            query: None,
+                            original_hwm_lsn: version.get_lsn(),
+                            ancestor_lsn: None,
+                            read_path: None,
+                            backtrace: None,
+                        },
+                    )));
+                }
+            }
         }
 
         if (tag.forknum == FSM_FORKNUM || tag.forknum == VISIBILITYMAP_FORKNUM)
@@ -510,8 +530,15 @@ impl Timeline {
         }
 
         // first try to lookup relation in cache
-        if let Some(_nblocks) = self.get_cached_rel_size(&tag, version.get_lsn()) {
-            return Ok(true);
+        if let Some(entry) = self.get_cached_rel_size(&tag, version.get_lsn()) {
+            match entry {
+                RelSizeCacheEntry::Present(_) => {
+                    return Ok(true);
+                }
+                RelSizeCacheEntry::Truncated => {
+                    return Ok(false);
+                }
+            }
         }
         // then check if the database was already initialized.
         // get_rel_exists can be called before dbdir is created.
@@ -1330,12 +1357,12 @@ impl Timeline {
     }
 
     /// Get cached size of relation if it not updated after specified LSN
-    pub fn get_cached_rel_size(&self, tag: &RelTag, lsn: Lsn) -> Option<BlockNumber> {
+    pub fn get_cached_rel_size(&self, tag: &RelTag, lsn: Lsn) -> Option<RelSizeCacheEntry> {
         let rel_size_cache = self.rel_size_cache.read().unwrap();
-        if let Some((cached_lsn, nblocks)) = rel_size_cache.map.get(tag) {
+        if let Some((cached_lsn, entry)) = rel_size_cache.map.get(tag) {
             if lsn >= *cached_lsn {
                 RELSIZE_CACHE_HITS.inc();
-                return Some(*nblocks);
+                return Some(*entry);
             }
             RELSIZE_CACHE_MISSES_OLD.inc();
         }
@@ -1359,11 +1386,11 @@ impl Timeline {
             hash_map::Entry::Occupied(mut entry) => {
                 let cached_lsn = entry.get_mut();
                 if lsn >= cached_lsn.0 {
-                    *cached_lsn = (lsn, nblocks);
+                    *cached_lsn = (lsn, RelSizeCacheEntry::Present(nblocks));
                 }
             }
             hash_map::Entry::Vacant(entry) => {
-                entry.insert((lsn, nblocks));
+                entry.insert((lsn, RelSizeCacheEntry::Present(nblocks)));
                 RELSIZE_CACHE_ENTRIES.inc();
             }
         }
@@ -1372,15 +1399,23 @@ impl Timeline {
     /// Store cached relation size
     pub fn set_cached_rel_size(&self, tag: RelTag, lsn: Lsn, nblocks: BlockNumber) {
         let mut rel_size_cache = self.rel_size_cache.write().unwrap();
-        if rel_size_cache.map.insert(tag, (lsn, nblocks)).is_none() {
+        if rel_size_cache
+            .map
+            .insert(tag, (lsn, RelSizeCacheEntry::Present(nblocks)))
+            .is_none()
+        {
             RELSIZE_CACHE_ENTRIES.inc();
         }
     }
 
     /// Remove cached relation size
-    pub fn remove_cached_rel_size(&self, tag: &RelTag) {
+    pub fn remove_cached_rel_size(&self, tag: RelTag, lsn: Lsn) {
         let mut rel_size_cache = self.rel_size_cache.write().unwrap();
-        if rel_size_cache.map.remove(tag).is_some() {
+        if rel_size_cache
+            .map
+            .insert(tag, (lsn, RelSizeCacheEntry::Truncated))
+            .is_some()
+        {
             RELSIZE_CACHE_ENTRIES.dec();
         }
     }
@@ -1585,7 +1620,9 @@ impl DatadirModification<'_> {
         //       check the cache too. This is because eagerly checking the cache results in
         //       less work overall and 10% better performance. It's more work on cache miss
         //       but cache miss is rare.
-        if let Some(nblocks) = self.tline.get_cached_rel_size(&rel, self.get_lsn()) {
+        if let Some(RelSizeCacheEntry::Present(nblocks)) =
+            self.tline.get_cached_rel_size(&rel, self.get_lsn())
+        {
             Ok(nblocks)
         } else if !self
             .tline
@@ -2172,7 +2209,7 @@ impl DatadirModification<'_> {
                     self.pending_nblocks -= old_size as i64;
 
                     // Remove entry from relation size cache
-                    self.tline.remove_cached_rel_size(&rel_tag);
+                    self.tline.remove_cached_rel_size(rel_tag, self.lsn);
 
                     // Delete size entry, as well as all blocks; this is currently a no-op because we haven't implemented tombstones in storage.
                     self.delete(rel_key_range(rel_tag));

pageserver/src/tenant/secondary/downloader.rs

@@ -668,9 +668,7 @@ impl From<DownloadError> for UpdateError {
 
 impl From<std::io::Error> for UpdateError {
     fn from(value: std::io::Error) -> Self {
-        if let Some(nix::errno::Errno::ENOSPC) =
-            value.raw_os_error().map(nix::errno::Errno::from_raw)
-        {
+        if let Some(nix::errno::Errno::ENOSPC) = value.raw_os_error().map(nix::errno::from_i32) {
             UpdateError::NoSpace
         } else if value
             .get_ref()

pageserver/src/tenant/timeline.rs

@@ -204,7 +204,13 @@ pub struct TimelineResources {
 /// value can be used to also update the cache, see [`Timeline::update_cached_rel_size`].
 pub(crate) struct RelSizeCache {
     pub(crate) complete_as_of: Lsn,
-    pub(crate) map: HashMap<RelTag, (Lsn, BlockNumber)>,
+    pub(crate) map: HashMap<RelTag, (Lsn, RelSizeCacheEntry)>,
+}
+
+#[derive(Debug, Copy, Clone)]
+pub enum RelSizeCacheEntry {
+    Present(BlockNumber),
+    Truncated,
 }
 
 pub struct Timeline {
@@ -690,15 +696,15 @@ impl std::fmt::Display for ReadPath {
 
 #[derive(thiserror::Error)]
 pub struct MissingKeyError {
-    keyspace: KeySpace,
-    shard: ShardNumber,
-    query: Option<VersionedKeySpaceQuery>,
+    pub keyspace: KeySpace,
+    pub shard: ShardNumber,
+    pub query: Option<VersionedKeySpaceQuery>,
     // This is largest request LSN from the get page request batch
-    original_hwm_lsn: Lsn,
-    ancestor_lsn: Option<Lsn>,
+    pub original_hwm_lsn: Lsn,
+    pub ancestor_lsn: Option<Lsn>,
     /// Debug information about the read path if there's an error
-    read_path: Option<ReadPath>,
-    backtrace: Option<std::backtrace::Backtrace>,
+    pub read_path: Option<ReadPath>,
+    pub backtrace: Option<std::backtrace::Backtrace>,
 }
 
 impl MissingKeyError {

pageserver/src/tenant/timeline/import_pgdata.rs

@@ -1,7 +1,6 @@
 use std::sync::Arc;
 
 use anyhow::{Context, bail};
-use importbucket_client::{ControlFile, RemoteStorageWrapper};
 use pageserver_api::models::ShardImportStatus;
 use remote_storage::RemotePath;
 use tokio::task::JoinHandle;
@@ -49,50 +48,121 @@ pub async fn doit(
     let storcon_client = StorageControllerUpcallClient::new(timeline.conf, &cancel);
 
     let shard_status = storcon_client
-        .get_timeline_import_status(
-            timeline.tenant_shard_id,
-            timeline.timeline_id,
-            timeline.generation,
-        )
+        .get_timeline_import_status(timeline.tenant_shard_id, timeline.timeline_id)
         .await
         .map_err(|_err| anyhow::anyhow!("Shut down while getting timeline import status"))?;
 
     info!(?shard_status, "peeking shard status");
     match shard_status {
-        ShardImportStatus::InProgress(maybe_progress) => {
+        None | Some(ShardImportStatus::InProgress) => {
+            // TODO: checkpoint the progress into the IndexPart instead of restarting
+            // from the beginning.
+
+            //
+            // Wipe the slate clean - the flow does not allow resuming.
+            // We can implement resuming in the future by checkpointing the progress into the IndexPart.
+            //
+            info!("wipe the slate clean");
+            {
+                // TODO: do we need to hold GC lock for this?
+                let mut guard = timeline.layers.write().await;
+                assert!(
+                    guard.layer_map()?.open_layer.is_none(),
+                    "while importing, there should be no in-memory layer" // this just seems like a good place to assert it
+                );
+                let all_layers_keys = guard.all_persistent_layers();
+                let all_layers: Vec<_> = all_layers_keys
+                    .iter()
+                    .map(|key| guard.get_from_key(key))
+                    .collect();
+                let open = guard.open_mut().context("open_mut")?;
+
+                timeline.remote_client.schedule_gc_update(&all_layers)?;
+                open.finish_gc_timeline(&all_layers);
+            }
+
+            //
+            // Wait for pgdata to finish uploading
+            //
+            info!("wait for pgdata to reach status 'done'");
             let storage =
                 importbucket_client::new(timeline.conf, &location, cancel.clone()).await?;
-
-            let control_file_res = if maybe_progress.is_none() {
-                // Only prepare the import once when there's no progress.
-                prepare_import(timeline, storage.clone(), &cancel).await
-            } else {
-                storage.get_control_file().await
-            };
-
-            let control_file = match control_file_res {
-                Ok(cf) => cf,
-                Err(err) => {
-                    return Err(
-                        terminate_flow_with_error(timeline, err, &storcon_client, &cancel).await,
-                    );
+            let status_prefix = RemotePath::from_string("status").unwrap();
+            let pgdata_status_key = status_prefix.join("pgdata");
+            loop {
+                let res = async {
+                    let pgdata_status: Option<importbucket_format::PgdataStatus> = storage
+                        .get_json(&pgdata_status_key)
+                        .await
+                        .context("get pgdata status")?;
+                    info!(?pgdata_status, "peeking pgdata status");
+                    if pgdata_status.map(|st| st.done).unwrap_or(false) {
+                        Ok(())
+                    } else {
+                        Err(anyhow::anyhow!("pgdata not done yet"))
+                    }
+                }
+                .await;
+                match res {
+                    Ok(_) => break,
+                    Err(err) => {
+                        info!(?err, "indefinitely waiting for pgdata to finish");
+                        if tokio::time::timeout(
+                            std::time::Duration::from_secs(10),
+                            cancel.cancelled(),
+                        )
+                        .await
+                        .is_ok()
+                        {
+                            bail!("cancelled while waiting for pgdata");
+                        }
+                    }
                 }
-            };
-
-            let res = flow::run(
-                timeline.clone(),
-                control_file,
-                storage.clone(),
-                maybe_progress,
-                ctx,
-            )
-            .await;
-            if let Err(err) = res {
-                return Err(
-                    terminate_flow_with_error(timeline, err, &storcon_client, &cancel).await,
-                );
             }
 
+            //
+            // Do the import
+            //
+            info!("do the import");
+            let control_file = storage.get_control_file().await?;
+            let base_lsn = control_file.base_lsn();
+
+            info!("update TimelineMetadata based on LSNs from control file");
+            {
+                let pg_version = control_file.pg_version();
+                let _ctx: &RequestContext = ctx;
+                async move {
+                    // FIXME: The 'disk_consistent_lsn' should be the LSN at the *end* of the
+                    // checkpoint record, and prev_record_lsn should point to its beginning.
+                    // We should read the real end of the record from the WAL, but here we
+                    // just fake it.
+                    let disk_consistent_lsn = Lsn(base_lsn.0 + 8);
+                    let prev_record_lsn = base_lsn;
+                    let metadata = TimelineMetadata::new(
+                        disk_consistent_lsn,
+                        Some(prev_record_lsn),
+                        None,     // no ancestor
+                        Lsn(0),   // no ancestor lsn
+                        base_lsn, // latest_gc_cutoff_lsn
+                        base_lsn, // initdb_lsn
+                        pg_version,
+                    );
+
+                    let _start_lsn = disk_consistent_lsn + 1;
+
+                    timeline
+                        .remote_client
+                        .schedule_index_upload_for_full_metadata_update(&metadata)?;
+
+                    timeline.remote_client.wait_completion().await?;
+
+                    anyhow::Ok(())
+                }
+            }
+            .await?;
+
+            flow::run(timeline.clone(), control_file, storage.clone(), ctx).await?;
+
             // Communicate that shard is done.
             // Ensure at-least-once delivery of the upcall to storage controller
             // before we mark the task as done and never come here again.
@@ -105,7 +175,7 @@ pub async fn doit(
                 .put_timeline_import_status(
                     timeline.tenant_shard_id,
                     timeline.timeline_id,
-                    timeline.generation,
+                    // TODO(vlad): What about import errors?
                     ShardImportStatus::Done,
                 )
                 .await
@@ -113,151 +183,16 @@ pub async fn doit(
                     anyhow::anyhow!("Shut down while putting timeline import status")
                 })?;
         }
-        ShardImportStatus::Error(err) => {
+        Some(ShardImportStatus::Error(err)) => {
             info!(
                 "shard status indicates that the shard is done (error), skipping import {}",
                 err
             );
         }
-        ShardImportStatus::Done => {
+        Some(ShardImportStatus::Done) => {
             info!("shard status indicates that the shard is done (success), skipping import");
         }
     }
 
     Ok(())
 }
-
-async fn prepare_import(
-    timeline: &Arc<Timeline>,
-    storage: RemoteStorageWrapper,
-    cancel: &CancellationToken,
-) -> anyhow::Result<ControlFile> {
-    // Wipe the slate clean before starting the import as a precaution.
-    // This method is only called when there's no recorded checkpoint for the import
-    // in the storage controller.
-    //
-    // Note that this is split-brain safe (two imports for same timeline shards running in
-    // different generations) because we go through the usual deletion path, including deletion queue.
-    info!("wipe the slate clean");
-    {
-        // TODO: do we need to hold GC lock for this?
-        let mut guard = timeline.layers.write().await;
-        assert!(
-            guard.layer_map()?.open_layer.is_none(),
-            "while importing, there should be no in-memory layer" // this just seems like a good place to assert it
-        );
-        let all_layers_keys = guard.all_persistent_layers();
-        let all_layers: Vec<_> = all_layers_keys
-            .iter()
-            .map(|key| guard.get_from_key(key))
-            .collect();
-        let open = guard.open_mut().context("open_mut")?;
-
-        timeline.remote_client.schedule_gc_update(&all_layers)?;
-        open.finish_gc_timeline(&all_layers);
-    }
-
-    //
-    // Wait for pgdata to finish uploading
-    //
-    info!("wait for pgdata to reach status 'done'");
-    let status_prefix = RemotePath::from_string("status").unwrap();
-    let pgdata_status_key = status_prefix.join("pgdata");
-    loop {
-        let res = async {
-            let pgdata_status: Option<importbucket_format::PgdataStatus> = storage
-                .get_json(&pgdata_status_key)
-                .await
-                .context("get pgdata status")?;
-            info!(?pgdata_status, "peeking pgdata status");
-            if pgdata_status.map(|st| st.done).unwrap_or(false) {
-                Ok(())
-            } else {
-                Err(anyhow::anyhow!("pgdata not done yet"))
-            }
-        }
-        .await;
-        match res {
-            Ok(_) => break,
-            Err(err) => {
-                info!(?err, "indefinitely waiting for pgdata to finish");
-                if tokio::time::timeout(std::time::Duration::from_secs(10), cancel.cancelled())
-                    .await
-                    .is_ok()
-                {
-                    bail!("cancelled while waiting for pgdata");
-                }
-            }
-        }
-    }
-
-    let control_file = storage.get_control_file().await?;
-    let base_lsn = control_file.base_lsn();
-
-    info!("update TimelineMetadata based on LSNs from control file");
-    {
-        let pg_version = control_file.pg_version();
-        async move {
-            // FIXME: The 'disk_consistent_lsn' should be the LSN at the *end* of the
-            // checkpoint record, and prev_record_lsn should point to its beginning.
-            // We should read the real end of the record from the WAL, but here we
-            // just fake it.
-            let disk_consistent_lsn = Lsn(base_lsn.0 + 8);
-            let prev_record_lsn = base_lsn;
-            let metadata = TimelineMetadata::new(
-                disk_consistent_lsn,
-                Some(prev_record_lsn),
-                None,     // no ancestor
-                Lsn(0),   // no ancestor lsn
-                base_lsn, // latest_gc_cutoff_lsn
-                base_lsn, // initdb_lsn
-                pg_version,
-            );
-
-            let _start_lsn = disk_consistent_lsn + 1;
-
-            timeline
-                .remote_client
-                .schedule_index_upload_for_full_metadata_update(&metadata)?;
-
-            timeline.remote_client.wait_completion().await?;
-
-            anyhow::Ok(())
-        }
-    }
-    .await?;
-
-    Ok(control_file)
-}
-
-async fn terminate_flow_with_error(
-    timeline: &Arc<Timeline>,
-    error: anyhow::Error,
-    storcon_client: &StorageControllerUpcallClient,
-    cancel: &CancellationToken,
-) -> anyhow::Error {
-    // The import task is a aborted on tenant shutdown, so in principle, it should
-    // never be cancelled. To be on the safe side, check the cancellation tokens
-    // before marking the import as failed.
-    if !(cancel.is_cancelled() || timeline.cancel.is_cancelled()) {
-        let notify_res = storcon_client
-            .put_timeline_import_status(
-                timeline.tenant_shard_id,
-                timeline.timeline_id,
-                timeline.generation,
-                ShardImportStatus::Error(format!("{error:#}")),
-            )
-            .await;
-
-        if let Err(_notify_error) = notify_res {
-            // The [`StorageControllerUpcallClient::put_timeline_import_status`] retries
-            // forever internally, so errors returned by it can only be due to cancellation.
-            info!("failed to notify storcon about permanent import error");
-        }
-
-        // Will be logged by [`Tenant::create_timeline_import_pgdata_task`]
-        error
-    } else {
-        anyhow::anyhow!("Import task cancelled")
-    }
-}

pageserver/src/tenant/timeline/import_pgdata/flow.rs

@@ -29,11 +29,10 @@
 //! - version-specific CheckPointData (=> pgv abstraction, already exists for regular walingest)
 
 use std::collections::HashSet;
-use std::hash::{Hash, Hasher};
 use std::ops::Range;
 use std::sync::Arc;
 
-use anyhow::ensure;
+use anyhow::{bail, ensure};
 use bytes::Bytes;
 use futures::stream::FuturesOrdered;
 use itertools::Itertools;
@@ -44,7 +43,6 @@ use pageserver_api::key::{
     slru_segment_size_to_key,
 };
 use pageserver_api::keyspace::{contiguous_range_len, is_contiguous_range, singleton_range};
-use pageserver_api::models::{ShardImportProgress, ShardImportProgressV1, ShardImportStatus};
 use pageserver_api::reltag::{RelTag, SlruKind};
 use pageserver_api::shard::ShardIdentity;
 use postgres_ffi::relfile_utils::parse_relfilename;
@@ -61,36 +59,16 @@ use super::Timeline;
 use super::importbucket_client::{ControlFile, RemoteStorageWrapper};
 use crate::assert_u64_eq_usize::UsizeIsU64;
 use crate::context::{DownloadBehavior, RequestContext};
-use crate::controller_upcall_client::{StorageControllerUpcallApi, StorageControllerUpcallClient};
 use crate::pgdatadir_mapping::{
     DbDirectory, RelDirectory, SlruSegmentDirectory, TwoPhaseDirectory,
 };
 use crate::task_mgr::TaskKind;
-use crate::tenant::storage_layer::{AsLayerDesc, ImageLayerWriter, Layer};
+use crate::tenant::storage_layer::{ImageLayerWriter, Layer};
 
 pub async fn run(
     timeline: Arc<Timeline>,
     control_file: ControlFile,
     storage: RemoteStorageWrapper,
-    import_progress: Option<ShardImportProgress>,
-    ctx: &RequestContext,
-) -> anyhow::Result<()> {
-    // Match how we run the import based on the progress version.
-    // If there's no import progress, it means that this is a new import
-    // and we can use whichever version we want.
-    match import_progress {
-        Some(ShardImportProgress::V1(progress)) => {
-            run_v1(timeline, control_file, storage, Some(progress), ctx).await
-        }
-        None => run_v1(timeline, control_file, storage, None, ctx).await,
-    }
-}
-
-async fn run_v1(
-    timeline: Arc<Timeline>,
-    control_file: ControlFile,
-    storage: RemoteStorageWrapper,
-    import_progress: Option<ShardImportProgressV1>,
     ctx: &RequestContext,
 ) -> anyhow::Result<()> {
     let planner = Planner {
@@ -103,31 +81,9 @@ async fn run_v1(
     let import_config = &timeline.conf.timeline_import_config;
     let plan = planner.plan(import_config).await?;
 
-    // Hash the plan and compare with the hash of the plan we got back from the storage controller.
-    // If the two match, it means that the planning stage had the same output.
-    //
-    // This is not intended to be a cryptographically secure hash.
-    const SEED: u64 = 42;
-    let mut hasher = twox_hash::XxHash64::with_seed(SEED);
-    plan.hash(&mut hasher);
-    let plan_hash = hasher.finish();
-
-    if let Some(progress) = &import_progress {
-        if plan_hash != progress.import_plan_hash {
-            anyhow::bail!("Import plan does not match storcon metadata");
-        }
-
-        // Handle collisions on jobs of unequal length
-        if progress.jobs != plan.jobs.len() {
-            anyhow::bail!("Import plan job length does not match storcon metadata")
-        }
-    }
-
     pausable_failpoint!("import-timeline-pre-execute-pausable");
 
-    let start_from_job_idx = import_progress.map(|progress| progress.completed);
-    plan.execute(timeline, start_from_job_idx, plan_hash, import_config, ctx)
-        .await
+    plan.execute(timeline, import_config, ctx).await
 }
 
 struct Planner {
@@ -137,11 +93,8 @@ struct Planner {
     tasks: Vec<AnyImportTask>,
 }
 
-#[derive(Hash)]
 struct Plan {
     jobs: Vec<ChunkProcessingJob>,
-    // Included here such that it ends up in the hash for the plan
-    shard: ShardIdentity,
 }
 
 impl Planner {
@@ -245,10 +198,7 @@ impl Planner {
             pgdata_lsn,
         ));
 
-        Ok(Plan {
-            jobs,
-            shard: self.shard,
-        })
+        Ok(Plan { jobs })
     }
 
     #[instrument(level = tracing::Level::DEBUG, skip_all, fields(dboid=%db.dboid, tablespace=%db.spcnode, path=%db.path))]
@@ -377,45 +327,25 @@ impl Plan {
     async fn execute(
         self,
         timeline: Arc<Timeline>,
-        start_after_job_idx: Option<usize>,
-        import_plan_hash: u64,
         import_config: &TimelineImportConfig,
         ctx: &RequestContext,
     ) -> anyhow::Result<()> {
-        let storcon_client = StorageControllerUpcallClient::new(timeline.conf, &timeline.cancel);
-
         let mut work = FuturesOrdered::new();
         let semaphore = Arc::new(Semaphore::new(import_config.import_job_concurrency.into()));
 
         let jobs_in_plan = self.jobs.len();
 
-        let mut jobs = self
-            .jobs
-            .into_iter()
-            .enumerate()
-            .map(|(idx, job)| (idx + 1, job))
-            .filter(|(idx, _job)| {
-                // Filter out any jobs that have been done already
-                if let Some(start_after) = start_after_job_idx {
-                    *idx > start_after
-                } else {
-                    true
-                }
-            })
-            .peekable();
-
-        let mut last_completed_job_idx = start_after_job_idx.unwrap_or(0);
-        let checkpoint_every: usize = import_config.import_job_checkpoint_threshold.into();
+        let mut jobs = self.jobs.into_iter().enumerate().peekable();
+        let mut results = Vec::new();
 
         // Run import jobs concurrently up to the limit specified by the pageserver configuration.
         // Note that we process completed futures in the oreder of insertion. This will be the
         // building block for resuming imports across pageserver restarts or tenant migrations.
-        while last_completed_job_idx < jobs_in_plan {
+        while results.len() < jobs_in_plan {
             tokio::select! {
                 permit = semaphore.clone().acquire_owned(), if jobs.peek().is_some() => {
                     let permit = permit.expect("never closed");
                     let (job_idx, job) = jobs.next().expect("we peeked");
-
                     let job_timeline = timeline.clone();
                     let ctx = ctx.detached_child(TaskKind::ImportPgdata, DownloadBehavior::Error);
 
@@ -427,35 +357,13 @@ impl Plan {
                 },
                 maybe_complete_job_idx = work.next() => {
                     match maybe_complete_job_idx {
-                        Some(Ok((job_idx, res))) => {
-                            assert!(last_completed_job_idx.checked_add(1).unwrap() == job_idx);
-
-                            res?;
-                            last_completed_job_idx = job_idx;
-
-                            if last_completed_job_idx % checkpoint_every == 0 {
-                                let progress = ShardImportProgressV1 {
-                                    jobs: jobs_in_plan,
-                                    completed: last_completed_job_idx,
-                                    import_plan_hash,
-                                };
-
-                                storcon_client.put_timeline_import_status(
-                                    timeline.tenant_shard_id,
-                                    timeline.timeline_id,
-                                    timeline.generation,
-                                    ShardImportStatus::InProgress(Some(ShardImportProgress::V1(progress)))
-                                )
-                                .await
-                                .map_err(|_err| {
-                                    anyhow::anyhow!("Shut down while putting timeline import status")
-                                })?;
-                            }
+                        Some(Ok((_job_idx, res))) => {
+                            results.push(res);
                         },
                         Some(Err(_)) => {
-                            anyhow::bail!(
-                                "import job panicked or cancelled"
-                            );
+                            results.push(Err(anyhow::anyhow!(
+                                "parallel job panicked or cancelled, check pageserver logs"
+                            )));
                         }
                         None => {}
                     }
@@ -463,7 +371,17 @@ impl Plan {
             }
         }
 
-        Ok(())
+        if results.iter().all(|r| r.is_ok()) {
+            Ok(())
+        } else {
+            let mut msg = String::new();
+            for result in results {
+                if let Err(err) = result {
+                    msg.push_str(&format!("{err:?}\n\n"));
+                }
+            }
+            bail!("Some parallel jobs failed:\n\n{msg}");
+        }
     }
 }
 
@@ -635,15 +553,6 @@ struct ImportSingleKeyTask {
     buf: Bytes,
 }
 
-impl Hash for ImportSingleKeyTask {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let ImportSingleKeyTask { key, buf } = self;
-
-        key.hash(state);
-        buf.hash(state);
-    }
-}
-
 impl ImportSingleKeyTask {
     fn new(key: Key, buf: Bytes) -> Self {
         ImportSingleKeyTask { key, buf }
@@ -672,20 +581,6 @@ struct ImportRelBlocksTask {
     storage: RemoteStorageWrapper,
 }
 
-impl Hash for ImportRelBlocksTask {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let ImportRelBlocksTask {
-            shard_identity: _,
-            key_range,
-            path,
-            storage: _,
-        } = self;
-
-        key_range.hash(state);
-        path.hash(state);
-    }
-}
-
 impl ImportRelBlocksTask {
     fn new(
         shard_identity: ShardIdentity,
@@ -770,19 +665,6 @@ struct ImportSlruBlocksTask {
     storage: RemoteStorageWrapper,
 }
 
-impl Hash for ImportSlruBlocksTask {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let ImportSlruBlocksTask {
-            key_range,
-            path,
-            storage: _,
-        } = self;
-
-        key_range.hash(state);
-        path.hash(state);
-    }
-}
-
 impl ImportSlruBlocksTask {
     fn new(key_range: Range<Key>, path: &RemotePath, storage: RemoteStorageWrapper) -> Self {
         ImportSlruBlocksTask {
@@ -825,7 +707,6 @@ impl ImportTask for ImportSlruBlocksTask {
     }
 }
 
-#[derive(Hash)]
 enum AnyImportTask {
     SingleKey(ImportSingleKeyTask),
     RelBlocks(ImportRelBlocksTask),
@@ -872,7 +753,6 @@ impl From<ImportSlruBlocksTask> for AnyImportTask {
     }
 }
 
-#[derive(Hash)]
 struct ChunkProcessingJob {
     range: Range<Key>,
     tasks: Vec<AnyImportTask>,
@@ -910,51 +790,17 @@ impl ChunkProcessingJob {
 
         let resident_layer = if nimages > 0 {
             let (desc, path) = writer.finish(ctx).await?;
-
-            {
-                let guard = timeline.layers.read().await;
-                let existing_layer = guard.try_get_from_key(&desc.key());
-                if let Some(layer) = existing_layer {
-                    if layer.metadata().generation != timeline.generation {
-                        return Err(anyhow::anyhow!(
-                            "Import attempted to rewrite layer file in the same generation: {}",
-                            layer.local_path()
-                        ));
-                    }
-                }
-            }
-
             Layer::finish_creating(timeline.conf, &timeline, desc, &path)?
         } else {
             // dropping the writer cleans up
             return Ok(());
         };
 
-        // The same import job might run multiple times since not each job is checkpointed.
-        // Hence, we must support the cases where the layer already exists. We cannot be
-        // certain that the existing layer is identical to the new one, so in that case
-        // we replace the old layer with the one we just generated.
-
+        // this is sharing the same code as create_image_layers
         let mut guard = timeline.layers.write().await;
-
-        let existing_layer = guard
-            .try_get_from_key(&resident_layer.layer_desc().key())
-            .cloned();
-        match existing_layer {
-            Some(existing) => {
-                guard.open_mut()?.rewrite_layers(
-                    &[(existing.clone(), resident_layer.clone())],
-                    &[],
-                    &timeline.metrics,
-                );
-            }
-            None => {
-                guard
-                    .open_mut()?
-                    .track_new_image_layers(&[resident_layer.clone()], &timeline.metrics);
-            }
-        }
-
+        guard
+            .open_mut()?
+            .track_new_image_layers(&[resident_layer.clone()], &timeline.metrics);
         crate::tenant::timeline::drop_wlock(guard);
 
         timeline

pageserver/src/virtual_file.rs

@@ -408,7 +408,7 @@ impl OpenFiles {
 /// error types may be elegible for retry.
 pub(crate) fn is_fatal_io_error(e: &std::io::Error) -> bool {
     use nix::errno::Errno::*;
-    match e.raw_os_error().map(nix::errno::Errno::from_raw) {
+    match e.raw_os_error().map(nix::errno::from_i32) {
         Some(EIO) => {
             // Terminate on EIO because we no longer trust the device to store
             // data safely, or to uphold persistence guarantees on fsync.

pageserver/src/virtual_file/io_engine.rs

@@ -124,7 +124,9 @@ pub(super) fn epoll_uring_error_to_std(
 ) -> std::io::Error {
     match e {
         tokio_epoll_uring::Error::Op(e) => e,
-        tokio_epoll_uring::Error::System(system) => std::io::Error::other(system),
+        tokio_epoll_uring::Error::System(system) => {
+            std::io::Error::new(std::io::ErrorKind::Other, system)
+        }
     }
 }
 

pgxn/neon/file_cache.c

@@ -936,44 +936,6 @@ lfc_prewarm_main(Datum main_arg)
 	lfc_ctl->prewarm_workers[worker_id].completed = GetCurrentTimestamp();
 }
 
-void
-lfc_invalidate(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber nblocks)
-{
-	BufferTag	tag;
-	FileCacheEntry *entry;
-	uint32		hash;
-
-	if (lfc_maybe_disabled())	/* fast exit if file cache is disabled */
-		return;
-
-	CopyNRelFileInfoToBufTag(tag, rinfo);
-	tag.forkNum = forkNum;
-
-	CriticalAssert(BufTagGetRelNumber(&tag) != InvalidRelFileNumber);
-
-	LWLockAcquire(lfc_lock, LW_EXCLUSIVE);
-	if (LFC_ENABLED())
-	{
-		for (BlockNumber blkno = 0; blkno < nblocks; blkno += lfc_blocks_per_chunk)
-		{
-			tag.blockNum = blkno;
-			hash = get_hash_value(lfc_hash, &tag);
-			entry = hash_search_with_hash_value(lfc_hash, &tag, hash, HASH_FIND, NULL);
-			if (entry != NULL)
-			{
-				for (int i = 0; i < lfc_blocks_per_chunk; i++)
-				{
-					if (GET_STATE(entry, i) == AVAILABLE)
-					{
-						lfc_ctl->used_pages -= 1;
-						SET_STATE(entry, i, UNAVAILABLE);
-					}
-				}
-			}
-		}
-	}
-	LWLockRelease(lfc_lock);
-}
 
 /*
  * Check if page is present in the cache.

pgxn/neon/file_cache.h

@@ -28,7 +28,6 @@ typedef struct FileCacheState
 extern bool lfc_store_prefetch_result;
 
 /* functions for local file cache */
-extern void lfc_invalidate(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber nblocks);
 extern void lfc_writev(NRelFileInfo rinfo, ForkNumber forkNum,
 					   BlockNumber blkno, const void *const *buffers,
 					   BlockNumber nblocks);

pgxn/neon/pagestore_smgr.c

@@ -919,6 +919,9 @@ neon_extend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno,
 		case RELPERSISTENCE_TEMP:
 		case RELPERSISTENCE_UNLOGGED:
 			mdextend(reln, forkNum, blkno, buffer, skipFsync);
+			/* Update LFC in case of unlogged index build */
+			if (reln == unlogged_build_rel && unlogged_build_phase == UNLOGGED_BUILD_PHASE_2)
+				lfc_write(InfoFromSMgrRel(reln), forkNum, blkno, buffer);
 			return;
 
 		default:
@@ -1007,6 +1010,14 @@ neon_zeroextend(SMgrRelation reln, ForkNumber forkNum, BlockNumber blocknum,
 		case RELPERSISTENCE_TEMP:
 		case RELPERSISTENCE_UNLOGGED:
 			mdzeroextend(reln, forkNum, blocknum, nblocks, skipFsync);
+			/* Update LFC in case of unlogged index build */
+			if (reln == unlogged_build_rel && unlogged_build_phase == UNLOGGED_BUILD_PHASE_2)
+			{
+				for (int i = 0; i < nblocks; i++)
+				{
+					lfc_write(InfoFromSMgrRel(reln), forkNum, blocknum + i, buffer.data);
+				}
+			}
 			return;
 
 		default:
@@ -1606,6 +1617,9 @@ neon_write(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, const vo
 			#else
 			mdwrite(reln, forknum, blocknum, buffer, skipFsync);
 			#endif
+			/* Update LFC in case of unlogged index build */
+			if (reln == unlogged_build_rel && unlogged_build_phase == UNLOGGED_BUILD_PHASE_2)
+				lfc_write(InfoFromSMgrRel(reln), forknum, blocknum, buffer);
 			return;
 		default:
 			neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence);
@@ -1671,6 +1685,9 @@ neon_writev(SMgrRelation reln, ForkNumber forknum, BlockNumber blkno,
 		case RELPERSISTENCE_TEMP:
 		case RELPERSISTENCE_UNLOGGED:
 			mdwritev(reln, forknum, blkno, buffers, nblocks, skipFsync);
+			/* Update LFC in case of unlogged index build */
+			if (reln == unlogged_build_rel && unlogged_build_phase == UNLOGGED_BUILD_PHASE_2)
+				lfc_writev(InfoFromSMgrRel(reln), forknum, blkno, buffers, nblocks);
 			return;
 		default:
 			neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence);
@@ -2066,8 +2083,6 @@ neon_end_unlogged_build(SMgrRelation reln)
 				 forknum);
 
 			forget_cached_relsize(InfoFromNInfoB(rinfob), forknum);
-			lfc_invalidate(InfoFromNInfoB(rinfob), forknum, nblocks);
-
 			mdclose(reln, forknum);
 #ifndef DEBUG_COMPARE_LOCAL
 			/* use isRedo == true, so that we drop it immediately */

proxy/Cargo.toml

@@ -73,7 +73,6 @@ rustc-hash.workspace = true
 rustls.workspace = true
 rustls-native-certs.workspace = true
 rustls-pemfile.workspace = true
-ryu = "1"
 scopeguard.workspace = true
 serde.workspace = true
 serde_json.workspace = true

proxy/src/binary/pg_sni_router.rs

@@ -394,7 +394,6 @@ async fn handle_client(
     }
 }
 
-#[allow(clippy::large_enum_variant)]
 enum Connection {
     Raw(tokio::net::TcpStream),
     Tls(tokio_rustls::client::TlsStream<tokio::net::TcpStream>),

proxy/src/binary/proxy.rs

@@ -43,12 +43,11 @@ project_build_tag!(BUILD_TAG);
 use clap::{Parser, ValueEnum};
 
 #[derive(Clone, Debug, ValueEnum)]
-#[clap(rename_all = "kebab-case")]
 enum AuthBackendType {
-    #[clap(alias("cplane-v1"))]
-    ControlPlane,
+    #[value(name("cplane-v1"), alias("control-plane"))]
+    ControlPlaneV1,
 
-    #[clap(alias("link"))]
+    #[value(name("link"), alias("control-redirect"))]
     ConsoleRedirect,
 
     #[cfg(any(test, feature = "testing"))]
@@ -708,7 +707,7 @@ fn build_auth_backend(
     args: &ProxyCliArgs,
 ) -> anyhow::Result<Either<&'static auth::Backend<'static, ()>, &'static ConsoleRedirectBackend>> {
     match &args.auth_backend {
-        AuthBackendType::ControlPlane => {
+        AuthBackendType::ControlPlaneV1 => {
             let wake_compute_cache_config: CacheOptions = args.wake_compute_cache.parse()?;
             let project_info_cache_config: ProjectInfoCacheOptions =
                 args.project_info_cache.parse()?;
@@ -863,7 +862,7 @@ async fn configure_redis(
         ("irsa", _) => match (&args.redis_host, args.redis_port) {
             (Some(host), Some(port)) => Some(
                 ConnectionWithCredentialsProvider::new_with_credentials_provider(
-                    host.clone(),
+                    host.to_string(),
                     port,
                     elasticache::CredentialsProvider::new(
                         args.aws_region.clone(),

proxy/src/context/mod.rs

@@ -78,7 +78,7 @@ struct RequestContextInner {
 
 #[derive(Clone, Debug)]
 pub(crate) enum AuthMethod {
-    // aka link
+    // aka passwordless, fka link
     ConsoleRedirect,
     ScramSha256,
     ScramSha256Plus,

proxy/src/logging/json.rs

@@ -1,356 +0,0 @@
-//! Vendoring of serde_json's string escaping code.
-//!
-//! <https://github.com/serde-rs/json/blob/c1826ebcccb1a520389c6b78ad3da15db279220d/src/ser.rs#L1514-L1552>
-//! <https://github.com/serde-rs/json/blob/c1826ebcccb1a520389c6b78ad3da15db279220d/src/ser.rs#L2081-L2157>
-//! Licensed by David Tolnay under MIT or Apache-2.0.
-//!
-//! With modifications by Conrad Ludgate on behalf of Neon.
-
-use std::fmt::{self, Write};
-
-use serde_json::ser::CharEscape;
-
-#[must_use]
-pub struct ValueSer<'buf> {
-    buf: &'buf mut Vec<u8>,
-}
-
-impl<'buf> ValueSer<'buf> {
-    pub fn new(buf: &'buf mut Vec<u8>) -> Self {
-        Self { buf }
-    }
-
-    #[inline]
-    pub fn serialize(self, value: &SerializedValue) {
-        self.buf.extend_from_slice(&value.0);
-    }
-
-    #[inline]
-    pub fn str(self, s: &str) {
-        format_escaped_str(self.buf, s);
-    }
-
-    #[inline]
-    pub fn str_args(self, s: fmt::Arguments) {
-        format_escaped_display(self.buf, s);
-    }
-
-    #[inline]
-    pub fn bytes_hex(self, s: &[u8]) {
-        self.str_args(format_args!("{s:x?}"));
-    }
-
-    #[inline]
-    pub fn int(self, x: impl itoa::Integer) {
-        write_int(x, self.buf);
-    }
-
-    #[inline]
-    pub fn float(self, x: impl ryu::Float) {
-        write_float(x, self.buf);
-    }
-
-    #[inline]
-    pub fn bool(self, x: bool) {
-        let bool = if x { "true" } else { "false" };
-        self.buf.extend_from_slice(bool.as_bytes());
-    }
-
-    #[inline]
-    pub fn map(self) -> MapSer<'buf> {
-        MapSer::new(self.buf)
-    }
-
-    #[inline]
-    #[expect(unused)]
-    pub fn list(self) -> ListSer<'buf> {
-        ListSer::new(self.buf)
-    }
-}
-
-pub struct MapSer<'buf> {
-    buf: &'buf mut Vec<u8>,
-    first: bool,
-}
-
-impl<'buf> MapSer<'buf> {
-    #[inline]
-    fn new(buf: &'buf mut Vec<u8>) -> Self {
-        buf.push(b'{');
-        Self { buf, first: true }
-    }
-
-    #[inline]
-    pub fn entry(&mut self, key: Escaped) -> ValueSer {
-        self.entry_inner(|b| key.write(b))
-    }
-
-    #[inline]
-    pub fn entry_escape(&mut self, key: &str) -> ValueSer {
-        self.entry_inner(|b| format_escaped_str(b, key))
-    }
-
-    #[inline]
-    pub fn entry_escape_args(&mut self, key: fmt::Arguments) -> ValueSer {
-        self.entry_inner(|b| format_escaped_display(b, key))
-    }
-
-    #[inline]
-    fn entry_inner(&mut self, f: impl FnOnce(&mut Vec<u8>)) -> ValueSer {
-        if !self.first {
-            self.buf.push(b',');
-        }
-        self.first = false;
-
-        f(self.buf);
-
-        self.buf.push(b':');
-        ValueSer { buf: self.buf }
-    }
-}
-
-impl Drop for MapSer<'_> {
-    fn drop(&mut self) {
-        self.buf.push(b'}');
-    }
-}
-
-pub struct ListSer<'buf> {
-    buf: &'buf mut Vec<u8>,
-    first: bool,
-}
-
-impl<'buf> ListSer<'buf> {
-    #[inline]
-    fn new(buf: &'buf mut Vec<u8>) -> Self {
-        buf.push(b'[');
-        Self { buf, first: true }
-    }
-
-    #[expect(unused)]
-    #[inline]
-    fn entry(&mut self) -> ValueSer {
-        if !self.first {
-            self.buf.push(b',');
-        }
-        self.first = false;
-        ValueSer { buf: self.buf }
-    }
-}
-
-impl Drop for ListSer<'_> {
-    fn drop(&mut self) {
-        self.buf.push(b']');
-    }
-}
-
-#[derive(Clone)]
-pub struct SerializedValue(Box<[u8]>);
-
-impl SerializedValue {
-    #[inline]
-    pub fn str(s: &str) -> Self {
-        let mut v = vec![];
-        v.reserve_exact(2 + s.len());
-        format_escaped_str(&mut v, s);
-        Self(v.into_boxed_slice())
-    }
-
-    #[inline]
-    pub fn str_args(s: fmt::Arguments) -> Self {
-        if let Some(s) = s.as_str() {
-            return Self::str(s);
-        }
-
-        let mut v = vec![];
-        format_escaped_display(&mut v, s);
-        Self(v.into_boxed_slice())
-    }
-
-    #[inline]
-    pub fn bytes_hex(s: &[u8]) -> Self {
-        Self::str_args(format_args!("{s:x?}"))
-    }
-
-    #[inline]
-    pub fn int(x: impl itoa::Integer) -> Self {
-        Self(itoa::Buffer::new().format(x).as_bytes().into())
-    }
-
-    #[inline]
-    pub fn float(x: impl ryu::Float) -> Self {
-        Self(ryu::Buffer::new().format(x).as_bytes().into())
-    }
-
-    #[inline]
-    pub fn bool(x: bool) -> Self {
-        let bool = if x { "true" } else { "false" };
-        Self(bool.as_bytes().into())
-    }
-}
-
-/// Represents a string that didn't need escaping because it's already valid json string.
-#[derive(Clone, Copy)]
-pub struct Escaped(&'static str);
-
-impl Escaped {
-    pub const fn new(s: &'static str) -> Self {
-        let mut i = 0;
-        while i < s.len() {
-            let escape = ESCAPE[s.as_bytes()[i] as usize];
-            i += 1;
-            assert!(escape == 0, "const json string should not need escaping");
-        }
-
-        Self(s)
-    }
-
-    pub fn as_str(self) -> &'static str {
-        self.0
-    }
-
-    fn write(self, buf: &mut Vec<u8>) {
-        buf.push(b'"');
-        buf.extend_from_slice(self.0.as_bytes());
-        buf.push(b'"');
-    }
-}
-
-fn write_int(x: impl itoa::Integer, b: &mut Vec<u8>) {
-    b.extend_from_slice(itoa::Buffer::new().format(x).as_bytes());
-}
-
-fn write_float(x: impl ryu::Float, b: &mut Vec<u8>) {
-    b.extend_from_slice(ryu::Buffer::new().format(x).as_bytes());
-}
-
-#[inline]
-fn char_escape_from_escape_table(escape: u8, byte: u8) -> CharEscape {
-    match escape {
-        self::BB => CharEscape::Backspace,
-        self::TT => CharEscape::Tab,
-        self::NN => CharEscape::LineFeed,
-        self::FF => CharEscape::FormFeed,
-        self::RR => CharEscape::CarriageReturn,
-        self::QU => CharEscape::Quote,
-        self::BS => CharEscape::ReverseSolidus,
-        self::UU => CharEscape::AsciiControl(byte),
-        _ => unreachable!(),
-    }
-}
-
-fn format_escaped_str(writer: &mut Vec<u8>, value: &str) {
-    writer.push(b'"');
-    let rest = format_escaped_str_contents(writer, value);
-    writer.extend_from_slice(rest);
-    writer.push(b'"');
-}
-
-fn format_escaped_display(writer: &mut Vec<u8>, args: fmt::Arguments) {
-    writer.push(b'"');
-
-    if let Some(s) = args.as_str() {
-        let rest = format_escaped_str_contents(writer, s);
-        writer.extend_from_slice(rest);
-    } else {
-        Collect { buf: writer }
-            .write_fmt(args)
-            .expect("formatting should not error");
-    }
-
-    writer.push(b'"');
-}
-
-struct Collect<'buf> {
-    buf: &'buf mut Vec<u8>,
-}
-
-impl fmt::Write for Collect<'_> {
-    fn write_str(&mut self, s: &str) -> fmt::Result {
-        let last = format_escaped_str_contents(self.buf, s);
-        self.buf.extend(last);
-        Ok(())
-    }
-}
-
-// writes any escape sequences, and returns the suffix still needed to be written.
-fn format_escaped_str_contents<'a>(writer: &mut Vec<u8>, value: &'a str) -> &'a [u8] {
-    let bytes = value.as_bytes();
-
-    let mut start = 0;
-
-    for (i, &byte) in bytes.iter().enumerate() {
-        let escape = ESCAPE[byte as usize];
-        if escape == 0 {
-            continue;
-        }
-
-        writer.extend_from_slice(&bytes[start..i]);
-
-        let char_escape = char_escape_from_escape_table(escape, byte);
-        write_char_escape(writer, char_escape);
-
-        start = i + 1;
-    }
-
-    &bytes[start..]
-}
-
-const BB: u8 = b'b'; // \x08
-const TT: u8 = b't'; // \x09
-const NN: u8 = b'n'; // \x0A
-const FF: u8 = b'f'; // \x0C
-const RR: u8 = b'r'; // \x0D
-const QU: u8 = b'"'; // \x22
-const BS: u8 = b'\\'; // \x5C
-const UU: u8 = b'u'; // \x00...\x1F except the ones above
-const __: u8 = 0;
-
-// Lookup table of escape sequences. A value of b'x' at index i means that byte
-// i is escaped as "\x" in JSON. A value of 0 means that byte i is not escaped.
-static ESCAPE: [u8; 256] = [
-    //   1   2   3   4   5   6   7   8   9   A   B   C   D   E   F
-    UU, UU, UU, UU, UU, UU, UU, UU, BB, TT, NN, UU, FF, RR, UU, UU, // 0
-    UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, // 1
-    __, __, QU, __, __, __, __, __, __, __, __, __, __, __, __, __, // 2
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 3
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 4
-    __, __, __, __, __, __, __, __, __, __, __, __, BS, __, __, __, // 5
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 6
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 7
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 8
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 9
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // A
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // B
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // C
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // D
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // E
-    __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // F
-];
-
-fn write_char_escape(writer: &mut Vec<u8>, char_escape: CharEscape) {
-    let s = match char_escape {
-        CharEscape::Quote => b"\\\"",
-        CharEscape::ReverseSolidus => b"\\\\",
-        CharEscape::Solidus => b"\\/",
-        CharEscape::Backspace => b"\\b",
-        CharEscape::FormFeed => b"\\f",
-        CharEscape::LineFeed => b"\\n",
-        CharEscape::CarriageReturn => b"\\r",
-        CharEscape::Tab => b"\\t",
-        CharEscape::AsciiControl(byte) => {
-            static HEX_DIGITS: [u8; 16] = *b"0123456789abcdef";
-            let bytes = &[
-                b'\\',
-                b'u',
-                b'0',
-                b'0',
-                HEX_DIGITS[(byte >> 4) as usize],
-                HEX_DIGITS[(byte & 0xF) as usize],
-            ];
-            return writer.extend_from_slice(bytes);
-        }
-    };
-
-    writer.extend_from_slice(s);
-}

proxy/src/proxy/copy_bidirectional.rs

@@ -67,6 +67,7 @@ where
     }
 }
 
+#[tracing::instrument(skip_all)]
 pub async fn copy_bidirectional_client_compute<Client, Compute>(
     client: &mut Client,
     compute: &mut Compute,

proxy/src/proxy/passthrough.rs

@@ -13,6 +13,7 @@ use crate::stream::Stream;
 use crate::usage_metrics::{Ids, MetricCounterRecorder, USAGE_METRICS};
 
 /// Forward bytes in both directions (client <-> compute).
+#[tracing::instrument(skip_all)]
 pub(crate) async fn proxy_pass(
     client: impl AsyncRead + AsyncWrite + Unpin,
     compute: impl AsyncRead + AsyncWrite + Unpin,

rust-toolchain.toml

@@ -1,5 +1,5 @@
 [toolchain]
-channel = "1.87.0"
+channel = "1.86.0"
 profile = "default"
 # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy.
 # https://rust-lang.github.io/rustup/concepts/profiles.html

safekeeper/src/bin/safekeeper.rs

@@ -22,10 +22,9 @@ use safekeeper::defaults::{
     DEFAULT_PARTIAL_BACKUP_TIMEOUT, DEFAULT_PG_LISTEN_ADDR, DEFAULT_SSL_CERT_FILE,
     DEFAULT_SSL_CERT_RELOAD_PERIOD, DEFAULT_SSL_KEY_FILE,
 };
-use safekeeper::wal_backup::WalBackup;
 use safekeeper::{
     BACKGROUND_RUNTIME, BROKER_RUNTIME, GlobalTimelines, HTTP_RUNTIME, SafeKeeperConf,
-    WAL_SERVICE_RUNTIME, broker, control_file, http, wal_service,
+    WAL_SERVICE_RUNTIME, broker, control_file, http, wal_backup, wal_service,
 };
 use sd_notify::NotifyState;
 use storage_broker::{DEFAULT_ENDPOINT, Uri};
@@ -485,15 +484,15 @@ async fn start_safekeeper(conf: Arc<SafeKeeperConf>) -> Result<()> {
         None => None,
     };
 
-    let wal_backup = Arc::new(WalBackup::new(&conf).await?);
-
-    let global_timelines = Arc::new(GlobalTimelines::new(conf.clone(), wal_backup.clone()));
+    let global_timelines = Arc::new(GlobalTimelines::new(conf.clone()));
 
     // Register metrics collector for active timelines. It's important to do this
     // after daemonizing, otherwise process collector will be upset.
     let timeline_collector = safekeeper::metrics::TimelineCollector::new(global_timelines.clone());
     metrics::register_internal(Box::new(timeline_collector))?;
 
+    wal_backup::init_remote_storage(&conf).await;
+
     // Keep handles to main tasks to die if any of them disappears.
     let mut tasks_handles: FuturesUnordered<BoxFuture<(String, JoinTaskRes)>> =
         FuturesUnordered::new();

safekeeper/src/copy_timeline.rs

@@ -3,7 +3,6 @@ use std::sync::Arc;
 use anyhow::{Result, bail};
 use camino::Utf8PathBuf;
 use postgres_ffi::{MAX_SEND_SIZE, WAL_SEGMENT_SIZE};
-use remote_storage::GenericRemoteStorage;
 use safekeeper_api::membership::Configuration;
 use tokio::fs::OpenOptions;
 use tokio::io::{AsyncSeekExt, AsyncWriteExt};
@@ -31,7 +30,6 @@ pub struct Request {
 pub async fn handle_request(
     request: Request,
     global_timelines: Arc<GlobalTimelines>,
-    storage: Arc<GenericRemoteStorage>,
 ) -> Result<()> {
     // TODO: request.until_lsn MUST be a valid LSN, and we cannot check it :(
     //   if LSN will point to the middle of a WAL record, timeline will be in "broken" state
@@ -129,7 +127,6 @@ pub async fn handle_request(
     assert!(first_ondisk_segment >= first_segment);
 
     copy_s3_segments(
-        &storage,
         wal_seg_size,
         &request.source_ttid,
         &request.destination_ttid,

safekeeper/src/http/routes.rs

@@ -258,7 +258,6 @@ async fn timeline_snapshot_handler(request: Request<Body>) -> Result<Response<Bo
 
     let global_timelines = get_global_timelines(&request);
     let tli = global_timelines.get(ttid).map_err(ApiError::from)?;
-    let storage = global_timelines.get_wal_backup().get_storage();
 
     // To stream the body use wrap_stream which wants Stream of Result<Bytes>,
     // so create the chan and write to it in another task.
@@ -270,7 +269,6 @@ async fn timeline_snapshot_handler(request: Request<Body>) -> Result<Response<Bo
         conf.my_id,
         destination,
         tx,
-        storage,
     ));
 
     let rx_stream = ReceiverStream::new(rx);
@@ -392,18 +390,12 @@ async fn timeline_copy_handler(mut request: Request<Body>) -> Result<Response<Bo
     );
 
     let global_timelines = get_global_timelines(&request);
-    let wal_backup = global_timelines.get_wal_backup();
-    let storage = wal_backup
-        .get_storage()
-        .ok_or(ApiError::BadRequest(anyhow::anyhow!(
-            "Remote Storage is not configured"
-        )))?;
 
     copy_timeline::handle_request(copy_timeline::Request{
         source_ttid,
         until_lsn: request_data.until_lsn,
         destination_ttid: TenantTimelineId::new(source_ttid.tenant_id, request_data.target_timeline_id),
-    }, global_timelines, storage)
+    }, global_timelines)
         .instrument(info_span!("copy_timeline", from=%source_ttid, to=%request_data.target_timeline_id, until_lsn=%request_data.until_lsn))
         .await
         .map_err(ApiError::InternalServerError)?;

safekeeper/src/lib.rs

@@ -125,6 +125,12 @@ pub struct SafeKeeperConf {
     pub enable_tls_wal_service_api: bool,
 }
 
+impl SafeKeeperConf {
+    pub fn is_wal_backup_enabled(&self) -> bool {
+        self.remote_storage.is_some() && self.wal_backup_enabled
+    }
+}
+
 impl SafeKeeperConf {
     pub fn dummy() -> Self {
         SafeKeeperConf {

safekeeper/src/pull_timeline.rs

@@ -9,7 +9,6 @@ use chrono::{DateTime, Utc};
 use futures::{SinkExt, StreamExt, TryStreamExt};
 use http_utils::error::ApiError;
 use postgres_ffi::{PG_TLI, XLogFileName, XLogSegNo};
-use remote_storage::GenericRemoteStorage;
 use reqwest::Certificate;
 use safekeeper_api::Term;
 use safekeeper_api::models::{PullTimelineRequest, PullTimelineResponse, TimelineStatus};
@@ -44,7 +43,6 @@ pub async fn stream_snapshot(
     source: NodeId,
     destination: NodeId,
     tx: mpsc::Sender<Result<Bytes>>,
-    storage: Option<Arc<GenericRemoteStorage>>,
 ) {
     match tli.try_wal_residence_guard().await {
         Err(e) => {
@@ -55,32 +53,10 @@ pub async fn stream_snapshot(
         Ok(maybe_resident_tli) => {
             if let Err(e) = match maybe_resident_tli {
                 Some(resident_tli) => {
-                    stream_snapshot_resident_guts(
-                        resident_tli,
-                        source,
-                        destination,
-                        tx.clone(),
-                        storage,
-                    )
-                    .await
-                }
-                None => {
-                    if let Some(storage) = storage {
-                        stream_snapshot_offloaded_guts(
-                            tli,
-                            source,
-                            destination,
-                            tx.clone(),
-                            &storage,
-                        )
+                    stream_snapshot_resident_guts(resident_tli, source, destination, tx.clone())
                         .await
-                    } else {
-                        tx.send(Err(anyhow!("remote storage not configured")))
-                            .await
-                            .ok();
-                        return;
-                    }
                 }
+                None => stream_snapshot_offloaded_guts(tli, source, destination, tx.clone()).await,
             } {
                 // Error type/contents don't matter as they won't can't reach the client
                 // (hyper likely doesn't do anything with it), but http stream will be
@@ -147,12 +123,10 @@ pub(crate) async fn stream_snapshot_offloaded_guts(
     source: NodeId,
     destination: NodeId,
     tx: mpsc::Sender<Result<Bytes>>,
-    storage: &GenericRemoteStorage,
 ) -> Result<()> {
     let mut ar = prepare_tar_stream(tx);
 
-    tli.snapshot_offloaded(&mut ar, source, destination, storage)
-        .await?;
+    tli.snapshot_offloaded(&mut ar, source, destination).await?;
 
     ar.finish().await?;
 
@@ -165,13 +139,10 @@ pub async fn stream_snapshot_resident_guts(
     source: NodeId,
     destination: NodeId,
     tx: mpsc::Sender<Result<Bytes>>,
-    storage: Option<Arc<GenericRemoteStorage>>,
 ) -> Result<()> {
     let mut ar = prepare_tar_stream(tx);
 
-    let bctx = tli
-        .start_snapshot(&mut ar, source, destination, storage)
-        .await?;
+    let bctx = tli.start_snapshot(&mut ar, source, destination).await?;
     pausable_failpoint!("sk-snapshot-after-list-pausable");
 
     let tli_dir = tli.get_timeline_dir();
@@ -211,7 +182,6 @@ impl Timeline {
         ar: &mut tokio_tar::Builder<W>,
         source: NodeId,
         destination: NodeId,
-        storage: &GenericRemoteStorage,
     ) -> Result<()> {
         // Take initial copy of control file, then release state lock
         let mut control_file = {
@@ -246,7 +216,6 @@ impl Timeline {
         // can fail if the timeline was un-evicted and modified in the background.
         let remote_timeline_path = &self.remote_path;
         wal_backup::copy_partial_segment(
-            storage,
             &replace.previous.remote_path(remote_timeline_path),
             &replace.current.remote_path(remote_timeline_path),
         )
@@ -293,7 +262,6 @@ impl WalResidentTimeline {
         ar: &mut tokio_tar::Builder<W>,
         source: NodeId,
         destination: NodeId,
-        storage: Option<Arc<GenericRemoteStorage>>,
     ) -> Result<SnapshotContext> {
         let mut shared_state = self.write_shared_state().await;
         let wal_seg_size = shared_state.get_wal_seg_size();
@@ -315,7 +283,6 @@ impl WalResidentTimeline {
 
             let remote_timeline_path = &self.tli.remote_path;
             wal_backup::copy_partial_segment(
-                &*storage.context("remote storage not configured")?,
                 &replace.previous.remote_path(remote_timeline_path),
                 &replace.current.remote_path(remote_timeline_path),
             )

safekeeper/src/test_utils.rs

@@ -18,7 +18,7 @@ use crate::send_wal::EndWatch;
 use crate::state::{TimelinePersistentState, TimelineState};
 use crate::timeline::{SharedState, StateSK, Timeline, get_timeline_dir};
 use crate::timelines_set::TimelinesSet;
-use crate::wal_backup::{WalBackup, remote_timeline_path};
+use crate::wal_backup::remote_timeline_path;
 use crate::{SafeKeeperConf, control_file, receive_wal, wal_storage};
 
 /// A Safekeeper testing or benchmarking environment. Uses a tempdir for storage, removed on drop.
@@ -101,22 +101,18 @@ impl Env {
         let safekeeper = self.make_safekeeper(node_id, ttid, start_lsn).await?;
         let shared_state = SharedState::new(StateSK::Loaded(safekeeper));
 
-        let wal_backup = Arc::new(WalBackup::new(&conf).await?);
-
         let timeline = Timeline::new(
             ttid,
             &timeline_dir,
             &remote_path,
             shared_state,
             conf.clone(),
-            wal_backup.clone(),
         );
         timeline.bootstrap(
             &mut timeline.write_shared_state().await,
             &conf,
             Arc::new(TimelinesSet::default()), // ignored for now
             RateLimiter::new(0, 0),
-            wal_backup,
         );
         Ok(timeline)
     }

safekeeper/src/timeline.rs

@@ -35,8 +35,7 @@ use crate::state::{EvictionState, TimelineMemState, TimelinePersistentState, Tim
 use crate::timeline_guard::ResidenceGuard;
 use crate::timeline_manager::{AtomicStatus, ManagerCtl};
 use crate::timelines_set::TimelinesSet;
-use crate::wal_backup;
-use crate::wal_backup::{WalBackup, remote_timeline_path};
+use crate::wal_backup::{self, remote_timeline_path};
 use crate::wal_backup_partial::PartialRemoteSegment;
 use crate::wal_storage::{Storage as wal_storage_iface, WalReader};
 use crate::{SafeKeeperConf, control_file, debug_dump, timeline_manager, wal_storage};
@@ -453,8 +452,6 @@ pub struct Timeline {
     manager_ctl: ManagerCtl,
     conf: Arc<SafeKeeperConf>,
 
-    pub(crate) wal_backup: Arc<WalBackup>,
-
     remote_deletion: std::sync::Mutex<Option<RemoteDeletionReceiver>>,
 
     /// Hold this gate from code that depends on the Timeline's non-shut-down state.  While holding
@@ -479,7 +476,6 @@ impl Timeline {
         remote_path: &RemotePath,
         shared_state: SharedState,
         conf: Arc<SafeKeeperConf>,
-        wal_backup: Arc<WalBackup>,
     ) -> Arc<Self> {
         let (commit_lsn_watch_tx, commit_lsn_watch_rx) =
             watch::channel(shared_state.sk.state().commit_lsn);
@@ -513,7 +509,6 @@ impl Timeline {
             wal_backup_active: AtomicBool::new(false),
             last_removed_segno: AtomicU64::new(0),
             mgr_status: AtomicStatus::new(),
-            wal_backup,
         })
     }
 
@@ -521,7 +516,6 @@ impl Timeline {
     pub fn load_timeline(
         conf: Arc<SafeKeeperConf>,
         ttid: TenantTimelineId,
-        wal_backup: Arc<WalBackup>,
     ) -> Result<Arc<Timeline>> {
         let _enter = info_span!("load_timeline", timeline = %ttid.timeline_id).entered();
 
@@ -535,7 +529,6 @@ impl Timeline {
             &remote_path,
             shared_state,
             conf,
-            wal_backup,
         ))
     }
 
@@ -546,7 +539,6 @@ impl Timeline {
         conf: &SafeKeeperConf,
         broker_active_set: Arc<TimelinesSet>,
         partial_backup_rate_limiter: RateLimiter,
-        wal_backup: Arc<WalBackup>,
     ) {
         let (tx, rx) = self.manager_ctl.bootstrap_manager();
 
@@ -569,7 +561,6 @@ impl Timeline {
                     tx,
                     rx,
                     partial_backup_rate_limiter,
-                    wal_backup,
                 )
                 .await
             }
@@ -615,10 +606,9 @@ impl Timeline {
         // it is cancelled, so WAL storage won't be opened again.
         shared_state.sk.close_wal_store();
 
-        if !only_local {
+        if !only_local && self.conf.is_wal_backup_enabled() {
             self.remote_delete().await?;
         }
-
         let dir_existed = delete_dir(&self.timeline_dir).await?;
         Ok(dir_existed)
     }
@@ -685,20 +675,11 @@ impl Timeline {
         guard: &mut std::sync::MutexGuard<Option<RemoteDeletionReceiver>>,
     ) -> RemoteDeletionReceiver {
         tracing::info!("starting remote deletion");
-        let storage = self.wal_backup.get_storage().clone();
         let (result_tx, result_rx) = tokio::sync::watch::channel(None);
         let ttid = self.ttid;
         tokio::task::spawn(
             async move {
-                let r = if let Some(storage) = storage {
-                    wal_backup::delete_timeline(&storage, &ttid).await
-                } else {
-                    tracing::info!(
-                        "skipping remote deletion because no remote storage is configured; this effectively leaks the objects in remote storage"
-                    );
-                    Ok(())
-                };
-
+                let r = wal_backup::delete_timeline(&ttid).await;
                 if let Err(e) = &r {
                     // Log error here in case nobody ever listens for our result (e.g. dropped API request)
                     tracing::error!("remote deletion failed: {e}");
@@ -1065,13 +1046,14 @@ impl WalResidentTimeline {
 
     pub async fn get_walreader(&self, start_lsn: Lsn) -> Result<WalReader> {
         let (_, persisted_state) = self.get_state().await;
+        let enable_remote_read = self.conf.is_wal_backup_enabled();
 
         WalReader::new(
             &self.ttid,
             self.timeline_dir.clone(),
             &persisted_state,
             start_lsn,
-            self.wal_backup.clone(),
+            enable_remote_read,
         )
     }
 

safekeeper/src/timeline_eviction.rs

@@ -6,7 +6,7 @@
 
 use anyhow::Context;
 use camino::Utf8PathBuf;
-use remote_storage::{GenericRemoteStorage, RemotePath};
+use remote_storage::RemotePath;
 use tokio::fs::File;
 use tokio::io::{AsyncRead, AsyncWriteExt};
 use tracing::{debug, info, instrument, warn};
@@ -68,10 +68,6 @@ impl Manager {
     #[instrument(name = "evict_timeline", skip_all)]
     pub(crate) async fn evict_timeline(&mut self) -> bool {
         assert!(!self.is_offloaded);
-        let Some(storage) = self.wal_backup.get_storage() else {
-            warn!("no remote storage configured, skipping uneviction");
-            return false;
-        };
         let partial_backup_uploaded = match &self.partial_backup_uploaded {
             Some(p) => p.clone(),
             None => {
@@ -91,7 +87,7 @@ impl Manager {
                 .inc();
         });
 
-        if let Err(e) = do_eviction(self, &partial_backup_uploaded, &storage).await {
+        if let Err(e) = do_eviction(self, &partial_backup_uploaded).await {
             warn!("failed to evict timeline: {:?}", e);
             return false;
         }
@@ -106,10 +102,6 @@ impl Manager {
     #[instrument(name = "unevict_timeline", skip_all)]
     pub(crate) async fn unevict_timeline(&mut self) {
         assert!(self.is_offloaded);
-        let Some(storage) = self.wal_backup.get_storage() else {
-            warn!("no remote storage configured, skipping uneviction");
-            return;
-        };
         let partial_backup_uploaded = match &self.partial_backup_uploaded {
             Some(p) => p.clone(),
             None => {
@@ -129,7 +121,7 @@ impl Manager {
                 .inc();
         });
 
-        if let Err(e) = do_uneviction(self, &partial_backup_uploaded, &storage).await {
+        if let Err(e) = do_uneviction(self, &partial_backup_uploaded).await {
             warn!("failed to unevict timeline: {:?}", e);
             return;
         }
@@ -145,12 +137,8 @@ impl Manager {
 /// Ensure that content matches the remote partial backup, if local segment exists.
 /// Then change state in control file and in-memory. If `delete_offloaded_wal` is set,
 /// delete the local segment.
-async fn do_eviction(
-    mgr: &mut Manager,
-    partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
-) -> anyhow::Result<()> {
-    compare_local_segment_with_remote(mgr, partial, storage).await?;
+async fn do_eviction(mgr: &mut Manager, partial: &PartialRemoteSegment) -> anyhow::Result<()> {
+    compare_local_segment_with_remote(mgr, partial).await?;
 
     mgr.tli.switch_to_offloaded(partial).await?;
     // switch manager state as soon as possible
@@ -165,16 +153,12 @@ async fn do_eviction(
 
 /// Ensure that content matches the remote partial backup, if local segment exists.
 /// Then download segment to local disk and change state in control file and in-memory.
-async fn do_uneviction(
-    mgr: &mut Manager,
-    partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
-) -> anyhow::Result<()> {
+async fn do_uneviction(mgr: &mut Manager, partial: &PartialRemoteSegment) -> anyhow::Result<()> {
     // if the local segment is present, validate it
-    compare_local_segment_with_remote(mgr, partial, storage).await?;
+    compare_local_segment_with_remote(mgr, partial).await?;
 
     // atomically download the partial segment
-    redownload_partial_segment(mgr, partial, storage).await?;
+    redownload_partial_segment(mgr, partial).await?;
 
     mgr.tli.switch_to_present().await?;
     // switch manager state as soon as possible
@@ -197,7 +181,6 @@ async fn delete_local_segment(mgr: &Manager, partial: &PartialRemoteSegment) ->
 async fn redownload_partial_segment(
     mgr: &Manager,
     partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
 ) -> anyhow::Result<()> {
     let tmp_file = mgr.tli.timeline_dir().join("remote_partial.tmp");
     let remote_segfile = remote_segment_path(mgr, partial);
@@ -207,7 +190,7 @@ async fn redownload_partial_segment(
         remote_segfile, tmp_file
     );
 
-    let mut reader = wal_backup::read_object(storage, &remote_segfile, 0).await?;
+    let mut reader = wal_backup::read_object(&remote_segfile, 0).await?;
     let mut file = File::create(&tmp_file).await?;
 
     let actual_len = tokio::io::copy(&mut reader, &mut file).await?;
@@ -251,16 +234,13 @@ async fn redownload_partial_segment(
 async fn compare_local_segment_with_remote(
     mgr: &Manager,
     partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
 ) -> anyhow::Result<()> {
     let local_path = local_segment_path(mgr, partial);
 
     match File::open(&local_path).await {
-        Ok(mut local_file) => {
-            do_validation(mgr, &mut local_file, mgr.wal_seg_size, partial, storage)
-                .await
-                .context("validation failed")
-        }
+        Ok(mut local_file) => do_validation(mgr, &mut local_file, mgr.wal_seg_size, partial)
+            .await
+            .context("validation failed"),
         Err(_) => {
             info!(
                 "local WAL file {} is not present, skipping validation",
@@ -278,7 +258,6 @@ async fn do_validation(
     file: &mut File,
     wal_seg_size: usize,
     partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
 ) -> anyhow::Result<()> {
     let local_size = file.metadata().await?.len() as usize;
     if local_size != wal_seg_size {
@@ -291,7 +270,7 @@ async fn do_validation(
 
     let remote_segfile = remote_segment_path(mgr, partial);
     let mut remote_reader: std::pin::Pin<Box<dyn AsyncRead + Send + Sync>> =
-        wal_backup::read_object(storage, &remote_segfile, 0).await?;
+        wal_backup::read_object(&remote_segfile, 0).await?;
 
     // remote segment should have bytes excatly up to `flush_lsn`
     let expected_remote_size = partial.flush_lsn.segment_offset(mgr.wal_seg_size);

safekeeper/src/timeline_manager.rs

@@ -35,7 +35,7 @@ use crate::state::TimelineState;
 use crate::timeline::{ManagerTimeline, ReadGuardSharedState, StateSK, WalResidentTimeline};
 use crate::timeline_guard::{AccessService, GuardId, ResidenceGuard};
 use crate::timelines_set::{TimelineSetGuard, TimelinesSet};
-use crate::wal_backup::{self, WalBackup, WalBackupTaskHandle};
+use crate::wal_backup::{self, WalBackupTaskHandle};
 use crate::wal_backup_partial::{self, PartialBackup, PartialRemoteSegment};
 
 pub(crate) struct StateSnapshot {
@@ -200,7 +200,6 @@ pub(crate) struct Manager {
     pub(crate) conf: SafeKeeperConf,
     pub(crate) wal_seg_size: usize,
     pub(crate) walsenders: Arc<WalSenders>,
-    pub(crate) wal_backup: Arc<WalBackup>,
 
     // current state
     pub(crate) state_version_rx: tokio::sync::watch::Receiver<usize>,
@@ -239,7 +238,6 @@ pub async fn main_task(
     manager_tx: tokio::sync::mpsc::UnboundedSender<ManagerCtlMessage>,
     mut manager_rx: tokio::sync::mpsc::UnboundedReceiver<ManagerCtlMessage>,
     global_rate_limiter: RateLimiter,
-    wal_backup: Arc<WalBackup>,
 ) {
     tli.set_status(Status::Started);
 
@@ -258,7 +256,6 @@ pub async fn main_task(
         broker_active_set,
         manager_tx,
         global_rate_limiter,
-        wal_backup,
     )
     .await;
 
@@ -374,7 +371,7 @@ pub async fn main_task(
     mgr.tli_broker_active.set(false);
 
     // shutdown background tasks
-    if let Some(storage) = mgr.wal_backup.get_storage() {
+    if mgr.conf.is_wal_backup_enabled() {
         if let Some(backup_task) = mgr.backup_task.take() {
             // If we fell through here, then the timeline is shutting down. This is important
             // because otherwise joining on the wal_backup handle might hang.
@@ -382,7 +379,7 @@ pub async fn main_task(
 
             backup_task.join().await;
         }
-        wal_backup::update_task(&mut mgr, storage, false, &last_state).await;
+        wal_backup::update_task(&mut mgr, false, &last_state).await;
     }
 
     if let Some(recovery_task) = &mut mgr.recovery_task {
@@ -418,13 +415,11 @@ impl Manager {
         broker_active_set: Arc<TimelinesSet>,
         manager_tx: tokio::sync::mpsc::UnboundedSender<ManagerCtlMessage>,
         global_rate_limiter: RateLimiter,
-        wal_backup: Arc<WalBackup>,
     ) -> Manager {
         let (is_offloaded, partial_backup_uploaded) = tli.bootstrap_mgr().await;
         Manager {
             wal_seg_size: tli.get_wal_seg_size().await,
             walsenders: tli.get_walsenders().clone(),
-            wal_backup,
             state_version_rx: tli.get_state_version_rx(),
             num_computes_rx: tli.get_walreceivers().get_num_rx(),
             tli_broker_active: broker_active_set.guard(tli.clone()),
@@ -482,8 +477,8 @@ impl Manager {
         let is_wal_backup_required =
             wal_backup::is_wal_backup_required(self.wal_seg_size, num_computes, state);
 
-        if let Some(storage) = self.wal_backup.get_storage() {
-            wal_backup::update_task(self, storage, is_wal_backup_required, state).await;
+        if self.conf.is_wal_backup_enabled() {
+            wal_backup::update_task(self, is_wal_backup_required, state).await;
         }
 
         // update the state in Arc<Timeline>
@@ -629,9 +624,9 @@ impl Manager {
     /// Spawns partial WAL backup task if needed.
     async fn update_partial_backup(&mut self, state: &StateSnapshot) {
         // check if WAL backup is enabled and should be started
-        let Some(storage) = self.wal_backup.get_storage() else {
+        if !self.conf.is_wal_backup_enabled() {
             return;
-        };
+        }
 
         if self.partial_backup_task.is_some() {
             // partial backup is already running
@@ -655,7 +650,6 @@ impl Manager {
             self.conf.clone(),
             self.global_rate_limiter.clone(),
             cancel.clone(),
-            storage,
         ));
         self.partial_backup_task = Some((handle, cancel));
     }
@@ -675,10 +669,6 @@ impl Manager {
     /// Reset partial backup state and remove its remote storage data. Since it
     /// might concurrently uploading something, cancel the task first.
     async fn backup_partial_reset(&mut self) -> anyhow::Result<Vec<String>> {
-        let Some(storage) = self.wal_backup.get_storage() else {
-            anyhow::bail!("remote storage is not enabled");
-        };
-
         info!("resetting partial backup state");
         // Force unevict timeline if it is evicted before erasing partial backup
         // state. The intended use of this function is to drop corrupted remote
@@ -699,7 +689,7 @@ impl Manager {
         }
 
         let tli = self.wal_resident_timeline()?;
-        let mut partial_backup = PartialBackup::new(tli, self.conf.clone(), storage).await;
+        let mut partial_backup = PartialBackup::new(tli, self.conf.clone()).await;
         // Reset might fail e.g. when cfile is already reset but s3 removal
         // failed, so set manager state to None beforehand. In any case caller
         // is expected to retry until success.

safekeeper/src/timelines_global_map.rs

@@ -25,7 +25,6 @@ use crate::rate_limit::RateLimiter;
 use crate::state::TimelinePersistentState;
 use crate::timeline::{Timeline, TimelineError, delete_dir, get_tenant_dir, get_timeline_dir};
 use crate::timelines_set::TimelinesSet;
-use crate::wal_backup::WalBackup;
 use crate::wal_storage::Storage;
 use crate::{SafeKeeperConf, control_file, wal_storage};
 
@@ -48,24 +47,15 @@ struct GlobalTimelinesState {
     conf: Arc<SafeKeeperConf>,
     broker_active_set: Arc<TimelinesSet>,
     global_rate_limiter: RateLimiter,
-    wal_backup: Arc<WalBackup>,
 }
 
 impl GlobalTimelinesState {
     /// Get dependencies for a timeline constructor.
-    fn get_dependencies(
-        &self,
-    ) -> (
-        Arc<SafeKeeperConf>,
-        Arc<TimelinesSet>,
-        RateLimiter,
-        Arc<WalBackup>,
-    ) {
+    fn get_dependencies(&self) -> (Arc<SafeKeeperConf>, Arc<TimelinesSet>, RateLimiter) {
         (
             self.conf.clone(),
             self.broker_active_set.clone(),
             self.global_rate_limiter.clone(),
-            self.wal_backup.clone(),
         )
     }
 
@@ -94,7 +84,7 @@ pub struct GlobalTimelines {
 
 impl GlobalTimelines {
     /// Create a new instance of the global timelines map.
-    pub fn new(conf: Arc<SafeKeeperConf>, wal_backup: Arc<WalBackup>) -> Self {
+    pub fn new(conf: Arc<SafeKeeperConf>) -> Self {
         Self {
             state: Mutex::new(GlobalTimelinesState {
                 timelines: HashMap::new(),
@@ -102,7 +92,6 @@ impl GlobalTimelines {
                 conf,
                 broker_active_set: Arc::new(TimelinesSet::default()),
                 global_rate_limiter: RateLimiter::new(1, 1),
-                wal_backup,
             }),
         }
     }
@@ -158,7 +147,7 @@ impl GlobalTimelines {
     /// just lock and unlock it for each timeline -- this function is called
     /// during init when nothing else is running, so this is fine.
     async fn load_tenant_timelines(&self, tenant_id: TenantId) -> Result<()> {
-        let (conf, broker_active_set, partial_backup_rate_limiter, wal_backup) = {
+        let (conf, broker_active_set, partial_backup_rate_limiter) = {
             let state = self.state.lock().unwrap();
             state.get_dependencies()
         };
@@ -173,7 +162,7 @@ impl GlobalTimelines {
                         TimelineId::from_str(timeline_dir_entry.file_name().to_str().unwrap_or(""))
                     {
                         let ttid = TenantTimelineId::new(tenant_id, timeline_id);
-                        match Timeline::load_timeline(conf.clone(), ttid, wal_backup.clone()) {
+                        match Timeline::load_timeline(conf.clone(), ttid) {
                             Ok(tli) => {
                                 let mut shared_state = tli.write_shared_state().await;
                                 self.state
@@ -186,7 +175,6 @@ impl GlobalTimelines {
                                     &conf,
                                     broker_active_set.clone(),
                                     partial_backup_rate_limiter.clone(),
-                                    wal_backup.clone(),
                                 );
                             }
                             // If we can't load a timeline, it's most likely because of a corrupted
@@ -224,10 +212,6 @@ impl GlobalTimelines {
         self.state.lock().unwrap().broker_active_set.clone()
     }
 
-    pub fn get_wal_backup(&self) -> Arc<WalBackup> {
-        self.state.lock().unwrap().wal_backup.clone()
-    }
-
     /// Create a new timeline with the given id. If the timeline already exists, returns
     /// an existing timeline.
     pub(crate) async fn create(
@@ -238,7 +222,7 @@ impl GlobalTimelines {
         start_lsn: Lsn,
         commit_lsn: Lsn,
     ) -> Result<Arc<Timeline>> {
-        let (conf, _, _, _) = {
+        let (conf, _, _) = {
             let state = self.state.lock().unwrap();
             if let Ok(timeline) = state.get(&ttid) {
                 // Timeline already exists, return it.
@@ -283,7 +267,7 @@ impl GlobalTimelines {
         check_tombstone: bool,
     ) -> Result<Arc<Timeline>> {
         // Check for existence and mark that we're creating it.
-        let (conf, broker_active_set, partial_backup_rate_limiter, wal_backup) = {
+        let (conf, broker_active_set, partial_backup_rate_limiter) = {
             let mut state = self.state.lock().unwrap();
             match state.timelines.get(&ttid) {
                 Some(GlobalMapTimeline::CreationInProgress) => {
@@ -312,14 +296,7 @@ impl GlobalTimelines {
         };
 
         // Do the actual move and reflect the result in the map.
-        match GlobalTimelines::install_temp_timeline(
-            ttid,
-            tmp_path,
-            conf.clone(),
-            wal_backup.clone(),
-        )
-        .await
-        {
+        match GlobalTimelines::install_temp_timeline(ttid, tmp_path, conf.clone()).await {
             Ok(timeline) => {
                 let mut timeline_shared_state = timeline.write_shared_state().await;
                 let mut state = self.state.lock().unwrap();
@@ -337,7 +314,6 @@ impl GlobalTimelines {
                     &conf,
                     broker_active_set,
                     partial_backup_rate_limiter,
-                    wal_backup,
                 );
                 drop(timeline_shared_state);
                 Ok(timeline)
@@ -360,7 +336,6 @@ impl GlobalTimelines {
         ttid: TenantTimelineId,
         tmp_path: &Utf8PathBuf,
         conf: Arc<SafeKeeperConf>,
-        wal_backup: Arc<WalBackup>,
     ) -> Result<Arc<Timeline>> {
         let tenant_path = get_tenant_dir(conf.as_ref(), &ttid.tenant_id);
         let timeline_path = get_timeline_dir(conf.as_ref(), &ttid);
@@ -402,7 +377,7 @@ impl GlobalTimelines {
         // Do the move.
         durable_rename(tmp_path, &timeline_path, !conf.no_sync).await?;
 
-        Timeline::load_timeline(conf, ttid, wal_backup)
+        Timeline::load_timeline(conf, ttid)
     }
 
     /// Get a timeline from the global map. If it's not present, it doesn't exist on disk,

safekeeper/src/wal_backup.rs

@@ -2,7 +2,6 @@ use std::cmp::min;
 use std::collections::HashSet;
 use std::num::NonZeroU32;
 use std::pin::Pin;
-use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{Context, Result};
@@ -18,7 +17,7 @@ use safekeeper_api::models::PeerInfo;
 use tokio::fs::File;
 use tokio::select;
 use tokio::sync::mpsc::{self, Receiver, Sender};
-use tokio::sync::watch;
+use tokio::sync::{OnceCell, watch};
 use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 use tracing::*;
@@ -64,12 +63,7 @@ pub(crate) fn is_wal_backup_required(
 /// Based on peer information determine which safekeeper should offload; if it
 /// is me, run (per timeline) task, if not yet. OTOH, if it is not me and task
 /// is running, kill it.
-pub(crate) async fn update_task(
-    mgr: &mut Manager,
-    storage: Arc<GenericRemoteStorage>,
-    need_backup: bool,
-    state: &StateSnapshot,
-) {
+pub(crate) async fn update_task(mgr: &mut Manager, need_backup: bool, state: &StateSnapshot) {
     let (offloader, election_dbg_str) =
         determine_offloader(&state.peers, state.backup_lsn, mgr.tli.ttid, &mgr.conf);
     let elected_me = Some(mgr.conf.my_id) == offloader;
@@ -88,12 +82,7 @@ pub(crate) async fn update_task(
                 return;
             };
 
-            let async_task = backup_task_main(
-                resident,
-                storage,
-                mgr.conf.backup_parallel_jobs,
-                shutdown_rx,
-            );
+            let async_task = backup_task_main(resident, mgr.conf.backup_parallel_jobs, shutdown_rx);
 
             let handle = if mgr.conf.current_thread_runtime {
                 tokio::spawn(async_task)
@@ -180,31 +169,33 @@ fn determine_offloader(
     }
 }
 
-pub struct WalBackup {
-    storage: Option<Arc<GenericRemoteStorage>>,
+static REMOTE_STORAGE: OnceCell<Option<GenericRemoteStorage>> = OnceCell::const_new();
+
+// Storage must be configured and initialized when this is called.
+fn get_configured_remote_storage() -> &'static GenericRemoteStorage {
+    REMOTE_STORAGE
+        .get()
+        .expect("failed to get remote storage")
+        .as_ref()
+        .unwrap()
 }
 
-impl WalBackup {
-    /// Create a new WalBackup instance.
-    pub async fn new(conf: &SafeKeeperConf) -> Result<Self> {
-        if !conf.wal_backup_enabled {
-            return Ok(Self { storage: None });
-        }
-
-        match conf.remote_storage.as_ref() {
-            Some(config) => {
-                let storage = GenericRemoteStorage::from_config(config).await?;
-                Ok(Self {
-                    storage: Some(Arc::new(storage)),
-                })
+pub async fn init_remote_storage(conf: &SafeKeeperConf) {
+    // TODO: refactor REMOTE_STORAGE to avoid using global variables, and provide
+    // dependencies to all tasks instead.
+    REMOTE_STORAGE
+        .get_or_init(|| async {
+            if let Some(conf) = conf.remote_storage.as_ref() {
+                Some(
+                    GenericRemoteStorage::from_config(conf)
+                        .await
+                        .expect("failed to create remote storage"),
+                )
+            } else {
+                None
             }
-            None => Ok(Self { storage: None }),
-        }
-    }
-
-    pub fn get_storage(&self) -> Option<Arc<GenericRemoteStorage>> {
-        self.storage.clone()
-    }
+        })
+        .await;
 }
 
 struct WalBackupTask {
@@ -213,14 +204,12 @@ struct WalBackupTask {
     wal_seg_size: usize,
     parallel_jobs: usize,
     commit_lsn_watch_rx: watch::Receiver<Lsn>,
-    storage: Arc<GenericRemoteStorage>,
 }
 
 /// Offload single timeline.
 #[instrument(name = "wal_backup", skip_all, fields(ttid = %tli.ttid))]
 async fn backup_task_main(
     tli: WalResidentTimeline,
-    storage: Arc<GenericRemoteStorage>,
     parallel_jobs: usize,
     mut shutdown_rx: Receiver<()>,
 ) {
@@ -234,7 +223,6 @@ async fn backup_task_main(
         timeline_dir: tli.get_timeline_dir(),
         timeline: tli,
         parallel_jobs,
-        storage,
     };
 
     // task is spinned up only when wal_seg_size already initialized
@@ -305,7 +293,6 @@ impl WalBackupTask {
 
             match backup_lsn_range(
                 &self.timeline,
-                self.storage.clone(),
                 &mut backup_lsn,
                 commit_lsn,
                 self.wal_seg_size,
@@ -335,7 +322,6 @@ impl WalBackupTask {
 
 async fn backup_lsn_range(
     timeline: &WalResidentTimeline,
-    storage: Arc<GenericRemoteStorage>,
     backup_lsn: &mut Lsn,
     end_lsn: Lsn,
     wal_seg_size: usize,
@@ -366,12 +352,7 @@ async fn backup_lsn_range(
     loop {
         let added_task = match iter.next() {
             Some(s) => {
-                uploads.push_back(backup_single_segment(
-                    &storage,
-                    s,
-                    timeline_dir,
-                    remote_timeline_path,
-                ));
+                uploads.push_back(backup_single_segment(s, timeline_dir, remote_timeline_path));
                 true
             }
             None => false,
@@ -407,7 +388,6 @@ async fn backup_lsn_range(
 }
 
 async fn backup_single_segment(
-    storage: &GenericRemoteStorage,
     seg: &Segment,
     timeline_dir: &Utf8Path,
     remote_timeline_path: &RemotePath,
@@ -415,13 +395,7 @@ async fn backup_single_segment(
     let segment_file_path = seg.file_path(timeline_dir)?;
     let remote_segment_path = seg.remote_path(remote_timeline_path);
 
-    let res = backup_object(
-        storage,
-        &segment_file_path,
-        &remote_segment_path,
-        seg.size(),
-    )
-    .await;
+    let res = backup_object(&segment_file_path, &remote_segment_path, seg.size()).await;
     if res.is_ok() {
         BACKED_UP_SEGMENTS.inc();
     } else {
@@ -481,11 +455,12 @@ fn get_segments(start: Lsn, end: Lsn, seg_size: usize) -> Vec<Segment> {
 }
 
 async fn backup_object(
-    storage: &GenericRemoteStorage,
     source_file: &Utf8Path,
     target_file: &RemotePath,
     size: usize,
 ) -> Result<()> {
+    let storage = get_configured_remote_storage();
+
     let file = File::open(&source_file)
         .await
         .with_context(|| format!("Failed to open file {source_file:?} for wal backup"))?;
@@ -500,11 +475,12 @@ async fn backup_object(
 }
 
 pub(crate) async fn backup_partial_segment(
-    storage: &GenericRemoteStorage,
     source_file: &Utf8Path,
     target_file: &RemotePath,
     size: usize,
 ) -> Result<()> {
+    let storage = get_configured_remote_storage();
+
     let file = File::open(&source_file)
         .await
         .with_context(|| format!("Failed to open file {source_file:?} for wal backup"))?;
@@ -528,20 +504,25 @@ pub(crate) async fn backup_partial_segment(
 }
 
 pub(crate) async fn copy_partial_segment(
-    storage: &GenericRemoteStorage,
     source: &RemotePath,
     destination: &RemotePath,
 ) -> Result<()> {
+    let storage = get_configured_remote_storage();
     let cancel = CancellationToken::new();
 
     storage.copy_object(source, destination, &cancel).await
 }
 
 pub async fn read_object(
-    storage: &GenericRemoteStorage,
     file_path: &RemotePath,
     offset: u64,
 ) -> anyhow::Result<Pin<Box<dyn tokio::io::AsyncRead + Send + Sync>>> {
+    let storage = REMOTE_STORAGE
+        .get()
+        .context("Failed to get remote storage")?
+        .as_ref()
+        .context("No remote storage configured")?;
+
     info!("segment download about to start from remote path {file_path:?} at offset {offset}");
 
     let cancel = CancellationToken::new();
@@ -566,10 +547,8 @@ pub async fn read_object(
 
 /// Delete WAL files for the given timeline. Remote storage must be configured
 /// when called.
-pub async fn delete_timeline(
-    storage: &GenericRemoteStorage,
-    ttid: &TenantTimelineId,
-) -> Result<()> {
+pub async fn delete_timeline(ttid: &TenantTimelineId) -> Result<()> {
+    let storage = get_configured_remote_storage();
     let remote_path = remote_timeline_path(ttid)?;
 
     // see DEFAULT_MAX_KEYS_PER_LIST_RESPONSE
@@ -639,14 +618,14 @@ pub async fn delete_timeline(
 }
 
 /// Used by wal_backup_partial.
-pub async fn delete_objects(storage: &GenericRemoteStorage, paths: &[RemotePath]) -> Result<()> {
+pub async fn delete_objects(paths: &[RemotePath]) -> Result<()> {
     let cancel = CancellationToken::new(); // not really used
+    let storage = get_configured_remote_storage();
     storage.delete_objects(paths, &cancel).await
 }
 
 /// Copy segments from one timeline to another. Used in copy_timeline.
 pub async fn copy_s3_segments(
-    storage: &GenericRemoteStorage,
     wal_seg_size: usize,
     src_ttid: &TenantTimelineId,
     dst_ttid: &TenantTimelineId,
@@ -655,6 +634,12 @@ pub async fn copy_s3_segments(
 ) -> Result<()> {
     const SEGMENTS_PROGRESS_REPORT_INTERVAL: u64 = 1024;
 
+    let storage = REMOTE_STORAGE
+        .get()
+        .expect("failed to get remote storage")
+        .as_ref()
+        .unwrap();
+
     let remote_dst_path = remote_timeline_path(dst_ttid)?;
 
     let cancel = CancellationToken::new();

safekeeper/src/wal_backup_partial.rs

@@ -19,11 +19,9 @@
 //! file. Code updates state in the control file before doing any S3 operations.
 //! This way control file stores information about all potentially existing
 //! remote partial segments and can clean them up after uploading a newer version.
-use std::sync::Arc;
-
 use camino::Utf8PathBuf;
 use postgres_ffi::{PG_TLI, XLogFileName, XLogSegNo};
-use remote_storage::{GenericRemoteStorage, RemotePath};
+use remote_storage::RemotePath;
 use safekeeper_api::Term;
 use serde::{Deserialize, Serialize};
 use tokio_util::sync::CancellationToken;
@@ -156,16 +154,12 @@ pub struct PartialBackup {
     conf: SafeKeeperConf,
     local_prefix: Utf8PathBuf,
     remote_timeline_path: RemotePath,
-    storage: Arc<GenericRemoteStorage>,
+
     state: State,
 }
 
 impl PartialBackup {
-    pub async fn new(
-        tli: WalResidentTimeline,
-        conf: SafeKeeperConf,
-        storage: Arc<GenericRemoteStorage>,
-    ) -> PartialBackup {
+    pub async fn new(tli: WalResidentTimeline, conf: SafeKeeperConf) -> PartialBackup {
         let (_, persistent_state) = tli.get_state().await;
         let wal_seg_size = tli.get_wal_seg_size().await;
 
@@ -179,7 +173,6 @@ impl PartialBackup {
             conf,
             local_prefix,
             remote_timeline_path,
-            storage,
         }
     }
 
@@ -247,8 +240,7 @@ impl PartialBackup {
         let remote_path = prepared.remote_path(&self.remote_timeline_path);
 
         // Upload first `backup_bytes` bytes of the segment to the remote storage.
-        wal_backup::backup_partial_segment(&self.storage, &local_path, &remote_path, backup_bytes)
-            .await?;
+        wal_backup::backup_partial_segment(&local_path, &remote_path, backup_bytes).await?;
         PARTIAL_BACKUP_UPLOADED_BYTES.inc_by(backup_bytes as u64);
 
         // We uploaded the segment, now let's verify that the data is still actual.
@@ -334,7 +326,7 @@ impl PartialBackup {
             let remote_path = self.remote_timeline_path.join(seg);
             objects_to_delete.push(remote_path);
         }
-        wal_backup::delete_objects(&self.storage, &objects_to_delete).await
+        wal_backup::delete_objects(&objects_to_delete).await
     }
 
     /// Delete all non-Uploaded segments from the remote storage. There should be only one
@@ -432,7 +424,6 @@ pub async fn main_task(
     conf: SafeKeeperConf,
     limiter: RateLimiter,
     cancel: CancellationToken,
-    storage: Arc<GenericRemoteStorage>,
 ) -> Option<PartialRemoteSegment> {
     debug!("started");
     let await_duration = conf.partial_backup_timeout;
@@ -441,7 +432,7 @@ pub async fn main_task(
     let mut commit_lsn_rx = tli.get_commit_lsn_watch_rx();
     let mut flush_lsn_rx = tli.get_term_flush_lsn_watch_rx();
 
-    let mut backup = PartialBackup::new(tli, conf, storage).await;
+    let mut backup = PartialBackup::new(tli, conf).await;
 
     debug!("state: {:?}", backup.state);
 

safekeeper/src/wal_storage.rs

@@ -21,7 +21,6 @@ use postgres_ffi::waldecoder::WalStreamDecoder;
 use postgres_ffi::{PG_TLI, XLogFileName, XLogSegNo, dispatch_pgversion};
 use pq_proto::SystemId;
 use remote_storage::RemotePath;
-use std::sync::Arc;
 use tokio::fs::{self, File, OpenOptions, remove_file};
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 use tracing::*;
@@ -33,7 +32,7 @@ use crate::metrics::{
     REMOVED_WAL_SEGMENTS, WAL_STORAGE_OPERATION_SECONDS, WalStorageMetrics, time_io_closure,
 };
 use crate::state::TimelinePersistentState;
-use crate::wal_backup::{WalBackup, read_object, remote_timeline_path};
+use crate::wal_backup::{read_object, remote_timeline_path};
 
 pub trait Storage {
     // Last written LSN.
@@ -646,7 +645,7 @@ pub struct WalReader {
     wal_segment: Option<Pin<Box<dyn AsyncRead + Send + Sync>>>,
 
     // S3 will be used to read WAL if LSN is not available locally
-    wal_backup: Arc<WalBackup>,
+    enable_remote_read: bool,
 
     // We don't have WAL locally if LSN is less than local_start_lsn
     local_start_lsn: Lsn,
@@ -665,7 +664,7 @@ impl WalReader {
         timeline_dir: Utf8PathBuf,
         state: &TimelinePersistentState,
         start_pos: Lsn,
-        wal_backup: Arc<WalBackup>,
+        enable_remote_read: bool,
     ) -> Result<Self> {
         if state.server.wal_seg_size == 0 || state.local_start_lsn == Lsn(0) {
             bail!("state uninitialized, no data to read");
@@ -694,7 +693,7 @@ impl WalReader {
             wal_seg_size: state.server.wal_seg_size as usize,
             pos: start_pos,
             wal_segment: None,
-            wal_backup,
+            enable_remote_read,
             local_start_lsn: state.local_start_lsn,
             timeline_start_lsn: state.timeline_start_lsn,
             pg_version: state.server.pg_version / 10000,
@@ -813,9 +812,9 @@ impl WalReader {
         }
 
         // Try to open remote file, if remote reads are enabled
-        if let Some(storage) = self.wal_backup.get_storage() {
+        if self.enable_remote_read {
             let remote_wal_file_path = self.remote_path.join(&wal_file_name);
-            return read_object(&storage, &remote_wal_file_path, xlogoff as u64).await;
+            return read_object(&remote_wal_file_path, xlogoff as u64).await;
         }
 
         bail!("WAL segment is not found")

storage_controller/src/http.rs

@@ -31,7 +31,7 @@ use pageserver_api::models::{
 };
 use pageserver_api::shard::TenantShardId;
 use pageserver_api::upcall_api::{
-    PutTimelineImportStatusRequest, ReAttachRequest, TimelineImportStatusRequest, ValidateRequest,
+    PutTimelineImportStatusRequest, ReAttachRequest, ValidateRequest,
 };
 use pageserver_client::{BlockUnblock, mgmt_api};
 use routerify::Middleware;
@@ -160,22 +160,22 @@ async fn handle_validate(req: Request<Body>) -> Result<Response<Body>, ApiError>
 async fn handle_get_timeline_import_status(req: Request<Body>) -> Result<Response<Body>, ApiError> {
     check_permissions(&req, Scope::GenerationsApi)?;
 
-    let mut req = match maybe_forward(req).await {
+    let tenant_shard_id: TenantShardId = parse_request_param(&req, "tenant_shard_id")?;
+    let timeline_id: TimelineId = parse_request_param(&req, "timeline_id")?;
+
+    let req = match maybe_forward(req).await {
         ForwardOutcome::Forwarded(res) => {
             return res;
         }
         ForwardOutcome::NotForwarded(req) => req,
     };
 
-    let get_req = json_request::<TimelineImportStatusRequest>(&mut req).await?;
-
     let state = get_state(&req);
-
     json_response(
         StatusCode::OK,
         state
             .service
-            .handle_timeline_shard_import_progress(get_req)
+            .handle_timeline_shard_import_progress(tenant_shard_id, timeline_id)
             .await?,
     )
 }

storage_controller/src/scheduler.rs

@@ -628,7 +628,11 @@ impl Scheduler {
             tracing::trace!(%node_id, "attached_shard_count={} shard_count={} expected={}", node.attached_shard_count, node.shard_count, expected_attached_shards_per_node);
         }
 
-        expected_attached_shards_per_node.saturating_sub(node.attached_shard_count)
+        if node.attached_shard_count < expected_attached_shards_per_node {
+            expected_attached_shards_per_node - node.attached_shard_count
+        } else {
+            0
+        }
     }
 
     pub(crate) fn expected_attached_shard_count(&self) -> usize {

storage_controller/src/service.rs

@@ -47,7 +47,7 @@ use pageserver_api::shard::{
 };
 use pageserver_api::upcall_api::{
     PutTimelineImportStatusRequest, ReAttachRequest, ReAttachResponse, ReAttachResponseTenant,
-    TimelineImportStatusRequest, ValidateRequest, ValidateResponse, ValidateResponseTenant,
+    ValidateRequest, ValidateResponse, ValidateResponseTenant,
 };
 use pageserver_client::{BlockUnblock, mgmt_api};
 use reqwest::{Certificate, StatusCode};
@@ -194,14 +194,6 @@ pub(crate) enum LeadershipStatus {
     Candidate,
 }
 
-enum ShardGenerationValidity {
-    Valid,
-    Mismatched {
-        claimed: Generation,
-        actual: Option<Generation>,
-    },
-}
-
 pub const RECONCILER_CONCURRENCY_DEFAULT: usize = 128;
 pub const PRIORITY_RECONCILER_CONCURRENCY_DEFAULT: usize = 256;
 pub const SAFEKEEPER_RECONCILER_CONCURRENCY_DEFAULT: usize = 32;
@@ -3917,36 +3909,19 @@ impl Service {
 
     pub(crate) async fn handle_timeline_shard_import_progress(
         self: &Arc<Self>,
-        req: TimelineImportStatusRequest,
+        tenant_shard_id: TenantShardId,
+        timeline_id: TimelineId,
     ) -> Result<ShardImportStatus, ApiError> {
-        let validity = self
-            .validate_shard_generation(req.tenant_shard_id, req.generation)
-            .await?;
-        match validity {
-            ShardGenerationValidity::Valid => {
-                // fallthrough
-            }
-            ShardGenerationValidity::Mismatched { claimed, actual } => {
-                tracing::info!(
-                    claimed=?claimed.into(),
-                    actual=?actual.and_then(|g| g.into()),
-                    "Rejecting import progress fetch from stale generation"
-                );
-
-                return Err(ApiError::BadRequest(anyhow::anyhow!("Invalid generation")));
-            }
-        }
-
         let maybe_import = self
             .persistence
-            .get_timeline_import(req.tenant_shard_id.tenant_id, req.timeline_id)
+            .get_timeline_import(tenant_shard_id.tenant_id, timeline_id)
             .await?;
 
         let import = maybe_import.ok_or_else(|| {
             ApiError::NotFound(
                 format!(
                     "import for {}/{} not found",
-                    req.tenant_shard_id.tenant_id, req.timeline_id
+                    tenant_shard_id.tenant_id, timeline_id
                 )
                 .into(),
             )
@@ -3955,11 +3930,11 @@ impl Service {
         import
             .shard_statuses
             .0
-            .get(&req.tenant_shard_id.to_index())
+            .get(&tenant_shard_id.to_index())
             .cloned()
             .ok_or_else(|| {
                 ApiError::NotFound(
-                    format!("shard {} not found", req.tenant_shard_id.shard_slug()).into(),
+                    format!("shard {} not found", tenant_shard_id.shard_slug()).into(),
                 )
             })
     }
@@ -3968,24 +3943,6 @@ impl Service {
         self: &Arc<Self>,
         req: PutTimelineImportStatusRequest,
     ) -> Result<(), ApiError> {
-        let validity = self
-            .validate_shard_generation(req.tenant_shard_id, req.generation)
-            .await?;
-        match validity {
-            ShardGenerationValidity::Valid => {
-                // fallthrough
-            }
-            ShardGenerationValidity::Mismatched { claimed, actual } => {
-                tracing::info!(
-                    claimed=?claimed.into(),
-                    actual=?actual.and_then(|g| g.into()),
-                    "Rejecting import progress update from stale generation"
-                );
-
-                return Err(ApiError::PreconditionFailed("Invalid generation".into()));
-            }
-        }
-
         let res = self
             .persistence
             .update_timeline_import(req.tenant_shard_id, req.timeline_id, req.status)
@@ -4020,56 +3977,6 @@ impl Service {
         Ok(())
     }
 
-    /// Check that a provided generation for some tenant shard is the most recent one.
-    ///
-    /// Validate with the in-mem state first, and, if that passes, validate with the
-    /// database state which is authoritative.
-    async fn validate_shard_generation(
-        self: &Arc<Self>,
-        tenant_shard_id: TenantShardId,
-        generation: Generation,
-    ) -> Result<ShardGenerationValidity, ApiError> {
-        {
-            let locked = self.inner.read().unwrap();
-            let tenant_shard =
-                locked
-                    .tenants
-                    .get(&tenant_shard_id)
-                    .ok_or(ApiError::InternalServerError(anyhow::anyhow!(
-                        "{} shard not found",
-                        tenant_shard_id
-                    )))?;
-
-            if tenant_shard.generation != Some(generation) {
-                return Ok(ShardGenerationValidity::Mismatched {
-                    claimed: generation,
-                    actual: tenant_shard.generation,
-                });
-            }
-        }
-
-        let mut db_generations = self
-            .persistence
-            .shard_generations(std::iter::once(&tenant_shard_id))
-            .await?;
-        let (_tid, db_generation) =
-            db_generations
-                .pop()
-                .ok_or(ApiError::InternalServerError(anyhow::anyhow!(
-                    "{} shard not found",
-                    tenant_shard_id
-                )))?;
-
-        if db_generation != Some(generation) {
-            return Ok(ShardGenerationValidity::Mismatched {
-                claimed: generation,
-                actual: db_generation,
-            });
-        }
-
-        Ok(ShardGenerationValidity::Valid)
-    }
-
     /// Finalize the import of a timeline
     ///
     /// This method should be called once all shards have reported that the import is complete.
@@ -4082,7 +3989,7 @@ impl Service {
     /// imports are stored in the database).
     #[instrument(skip_all, fields(
         tenant_id=%import.tenant_id,
-        timeline_id=%import.timeline_id,
+        shard_id=%import.timeline_id,
     ))]
     async fn finalize_timeline_import(
         self: &Arc<Self>,

storage_controller/src/timeline_import.rs

@@ -5,7 +5,7 @@ use http_utils::error::ApiError;
 use reqwest::Method;
 use serde::{Deserialize, Serialize};
 
-use pageserver_api::models::{ShardImportProgress, ShardImportStatus};
+use pageserver_api::models::ShardImportStatus;
 use tokio_util::sync::CancellationToken;
 use utils::{
     id::{TenantId, TimelineId},
@@ -28,12 +28,7 @@ impl ShardImportStatuses {
         ShardImportStatuses(
             shards
                 .into_iter()
-                .map(|ts_id| {
-                    (
-                        ts_id,
-                        ShardImportStatus::InProgress(None::<ShardImportProgress>),
-                    )
-                })
+                .map(|ts_id| (ts_id, ShardImportStatus::InProgress))
                 .collect(),
         )
     }

storage_scrubber/src/checks.rs

@@ -13,7 +13,7 @@ use pageserver::tenant::remote_timeline_client::{
 };
 use pageserver::tenant::storage_layer::LayerName;
 use pageserver_api::shard::ShardIndex;
-use remote_storage::{DownloadError, GenericRemoteStorage, ListingObject, RemotePath};
+use remote_storage::{GenericRemoteStorage, ListingObject, RemotePath};
 use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 use utils::generation::Generation;
@@ -165,34 +165,23 @@ pub(crate) async fn branch_cleanup_and_check_errors(
                                 .head_object(&path, &CancellationToken::new())
                                 .await;
 
-                            match response {
-                                Ok(_) => {}
-                                Err(DownloadError::NotFound) => {
-                                    // Object is not present.
-                                    let is_l0 =
-                                        LayerMap::is_l0(layer.key_range(), layer.is_delta());
+                            if let Err(e) = response {
+                                // Object is not present.
+                                let is_l0 = LayerMap::is_l0(layer.key_range(), layer.is_delta());
 
-                                    let msg = format!(
-                                        "index_part.json contains a layer {}{} (shard {}) that is not present in remote storage (layer_is_l0: {})",
-                                        layer,
-                                        metadata.generation.get_suffix(),
-                                        metadata.shard,
-                                        is_l0,
-                                    );
+                                let msg = format!(
+                                    "index_part.json contains a layer {}{} (shard {}) that is not present in remote storage (layer_is_l0: {}) with error: {}",
+                                    layer,
+                                    metadata.generation.get_suffix(),
+                                    metadata.shard,
+                                    is_l0,
+                                    e,
+                                );
 
-                                    if is_l0 || ignore_error {
-                                        result.warnings.push(msg);
-                                    } else {
-                                        result.errors.push(msg);
-                                    }
-                                }
-                                Err(e) => {
-                                    tracing::warn!(
-                                        "cannot check if the layer {}{} is present in remote storage (error: {})",
-                                        layer,
-                                        metadata.generation.get_suffix(),
-                                        e,
-                                    );
+                                if is_l0 || ignore_error {
+                                    result.warnings.push(msg);
+                                } else {
+                                    result.errors.push(msg);
                                 }
                             }
                         }

test_runner/bin/neon_local_create_deep_l0_stack.py

@@ -1,59 +0,0 @@
-"""
-Script to creates a stack of L0 deltas each of which should have 1 Value::Delta per page in `data`,
-in your running neon_local setup.
-
-Use this bash setup to reset your neon_local environment.
-The last line of this bash snippet will run this file here.
-```
- export NEON_REPO_DIR=$PWD/.neon
- export NEON_BIN_DIR=$PWD/target/release
- $NEON_BIN_DIR/neon_local stop
- rm -rf $NEON_REPO_DIR
- $NEON_BIN_DIR/neon_local init
- cat >>  $NEON_REPO_DIR/pageserver_1/pageserver.toml <<"EOF"
- # customizations
- virtual_file_io_mode = "direct-rw"
- page_service_pipelining={mode="pipelined", max_batch_size=32, execution="concurrent-futures"}
- get_vectored_concurrent_io={mode="sidecar-task"}
-EOF
- $NEON_BIN_DIR/neon_local start
-
- psql 'postgresql://localhost:1235/storage_controller' -c 'DELETE FROM tenant_shards'
- sed 's/.*get_vectored_concurrent_io.*/get_vectored_concurrent_io={mode="sidecar-task"}/' -i $NEON_REPO_DIR/pageserver_1/pageserver.toml
- $NEON_BIN_DIR/neon_local pageserver restart
- sleep 2
- $NEON_BIN_DIR/neon_local tenant create --set-default
- ./target/debug/neon_local endpoint stop foo
- rm -rf  $NEON_REPO_DIR/endpoints/foo
- ./target/debug/neon_local endpoint create foo
- echo 'full_page_writes=off' >>  $NEON_REPO_DIR/endpoints/foo/postgresql.conf
- ./target/debug/neon_local endpoint start foo
-
-  pushd test_runner; poetry run python3 -m bin.neon_local_create_deep_l0_stack 10; popd
-```
-"""
-
-import sys
-
-import psycopg2
-from fixtures.common_types import TenantShardId, TimelineId
-from fixtures.pageserver.http import PageserverHttpClient
-from fixtures.pageserver.makelayers.l0stack import L0StackShape, make_l0_stack_standalone
-
-ps_http = PageserverHttpClient(port=9898, is_testing_enabled_or_skip=lambda: None)
-vps_http = PageserverHttpClient(port=1234, is_testing_enabled_or_skip=lambda: None)
-
-tenants = ps_http.tenant_list()
-assert len(tenants) == 1
-tenant_shard_id = TenantShardId.parse(tenants[0]["id"])
-
-timlines = ps_http.timeline_list(tenant_shard_id)
-assert len(timlines) == 1
-timeline_id = TimelineId(timlines[0]["timeline_id"])
-
-connstr = "postgresql://cloud_admin@localhost:55432/postgres"
-conn = psycopg2.connect(connstr)
-
-shape = L0StackShape(logical_table_size_mib=50, delta_stack_height=int(sys.argv[1]))
-
-make_l0_stack_standalone(vps_http, ps_http, tenant_shard_id, timeline_id, conn, shape)

test_runner/fixtures/neon_fixtures.py

@@ -1255,12 +1255,6 @@ class NeonEnv:
                 "no_sync": True,
                 # Look for gaps in WAL received from safekeepeers
                 "validate_wal_contiguity": True,
-                # TODO(vlad): make these configurable through the builder
-                "timeline_import_config": {
-                    "import_job_concurrency": 4,
-                    "import_job_soft_size_limit": 512 * 1024,
-                    "import_job_checkpoint_threshold": 4,
-                },
             }
 
             # Batching (https://github.com/neondatabase/neon/issues/9377):
@@ -1377,11 +1371,7 @@ class NeonEnv:
             force=config.config_init_force,
         )
 
-    def start(
-        self,
-        timeout_in_seconds: int | None = None,
-        extra_ps_env_vars: dict[str, str] | None = None,
-    ):
+    def start(self, timeout_in_seconds: int | None = None):
         # Storage controller starts first, so that pageserver /re-attach calls don't
         # bounce through retries on startup
         self.storage_controller.start(timeout_in_seconds=timeout_in_seconds)
@@ -1400,10 +1390,7 @@ class NeonEnv:
             for pageserver in self.pageservers:
                 futs.append(
                     executor.submit(
-                        lambda ps=pageserver: ps.start(  # type: ignore[misc]
-                            extra_env_vars=extra_ps_env_vars or {},
-                            timeout_in_seconds=timeout_in_seconds,
-                        ),
+                        lambda ps=pageserver: ps.start(timeout_in_seconds=timeout_in_seconds)  # type: ignore[misc]
                     )
                 )
 

test_runner/fixtures/pageserver/makelayers/l0stack.py

@@ -1,148 +0,0 @@
-from dataclasses import dataclass
-
-from psycopg2.extensions import connection as PgConnection
-
-from fixtures.common_types import Lsn, TenantShardId, TimelineId
-from fixtures.log_helper import log
-from fixtures.neon_fixtures import Endpoint
-from fixtures.pageserver.http import PageserverHttpClient
-from fixtures.pageserver.utils import wait_for_last_record_lsn
-
-
-@dataclass
-class L0StackShape:
-    logical_table_size_mib: int = 50
-    delta_stack_height: int = 20
-
-
-def make_l0_stack(endpoint: Endpoint, shape: L0StackShape):
-    """
-    Creates stack of L0 deltas each of which should have 1 Value::Delta per page in table `data`.
-    """
-    env = endpoint.env
-
-    # TDOO: wait for storcon to finish any reonciles before jumping to action here?
-    description = env.storage_controller.tenant_describe(endpoint.tenant_id)
-    shards = description["shards"]
-    assert len(shards) == 1, "does not support sharding"
-    tenant_shard_id = TenantShardId.parse(shards[0]["tenant_shard_id"])
-
-    endpoint.config(["full_page_writes=off"])
-    endpoint.reconfigure()
-
-    ps = env.get_pageserver(shards[0]["node_attached"])
-
-    timeline_id = endpoint.show_timeline_id()
-
-    vps_http = env.storage_controller.pageserver_api()
-    ps_http = ps.http_client()
-    endpoint_conn = endpoint.connect()
-    make_l0_stack_standalone(vps_http, ps_http, tenant_shard_id, timeline_id, endpoint_conn, shape)
-
-
-def make_l0_stack_standalone(
-    vps_http: PageserverHttpClient,
-    ps_http: PageserverHttpClient,
-    tenant_shard_id: TenantShardId,
-    timeline_id: TimelineId,
-    endpoint_conn: PgConnection,
-    shape: L0StackShape,
-):
-    """
-    See make_l0_stack for details.
-
-    This function is a standalone version of make_l0_stack, usable from not-test code.
-    """
-
-    assert not tenant_shard_id.shard_index.is_sharded, (
-        "the current implementation only supports unsharded tenants"
-    )
-
-    tenant_id = tenant_shard_id.tenant_id
-    conn = endpoint_conn
-    desired_size = shape.logical_table_size_mib * 1024 * 1024
-
-    config = {
-        "gc_period": "0s",  # disable periodic gc
-        "checkpoint_timeout": "10 years",
-        "compaction_period": "1h",  # doesn't matter, but 0 value will kill walredo every 10s
-        "compaction_threshold": 100000,  # we just want L0s
-        "compaction_target_size": 134217728,
-        "checkpoint_distance": 268435456,
-        "image_creation_threshold": 100000,  # we just want L0s
-    }
-
-    vps_http.set_tenant_config(tenant_id, config)
-
-    conn.autocommit = True
-    cur = conn.cursor()
-
-    # Ensure full_page_writes are disabled so that all Value::Delta in
-    # pageserver are !will_init, and therefore a getpage needs to read
-    # the entire delta stack.
-    cur.execute("SHOW full_page_writes")
-    assert cur.fetchall()[0][0] == "off", "full_page_writes should be off"
-
-    # each tuple is 23 (header) + 100 bytes = 123 bytes
-    # page header si 24 bytes
-    # 8k page size
-    # (8k-24bytes) / 123 bytes = 63 tuples per page
-    # set fillfactor to 10 to have 6 tuples per page
-    cur.execute("DROP TABLE IF EXISTS data")
-    cur.execute("CREATE TABLE data(id bigint, row char(92)) with (fillfactor=10)")
-    need_pages = desired_size // 8192
-    need_rows = need_pages * 6
-    log.info(f"Need {need_pages} pages, {need_rows} rows")
-    cur.execute(f"INSERT INTO data SELECT i,'row'||i FROM generate_series(1, {need_rows}) as i")
-    # Raise fillfactor to 100% so that all updates are HOT updates.
-    # We assert they're hot updates by checking fetch_id_to_page_mapping remains the same.
-    cur.execute("ALTER TABLE data SET (fillfactor=100)")
-
-    def settle_and_flush():
-        cur.execute("SELECT pg_current_wal_flush_lsn()")
-        flush_lsn = Lsn(cur.fetchall()[0][0])
-        wait_for_last_record_lsn(ps_http, tenant_shard_id, timeline_id, flush_lsn)
-        ps_http.timeline_checkpoint(tenant_id, timeline_id)
-
-    # create an L0 for the initial data we just inserted
-    settle_and_flush()
-
-    # assert we wrote what we think we wrote
-    cur.execute("""
-        with ntuples_per_page as (
-            select (ctid::text::point)[0]::bigint pageno,count(*) ntuples from data group by pageno
-        )
-        select ntuples, count(*) npages from ntuples_per_page group by ntuples order by ntuples;
-    """)
-    rows = cur.fetchall()
-    log.info(f"initial table layout: {rows}")
-    assert len(rows) == 1
-    assert rows[0][0] == 6, f"expected 6 tuples per page, got {rows[0][0]}"
-    assert rows[0][1] == need_pages, f"expected {need_pages} pages, got {rows[0][1]}"
-
-    def fetch_id_to_page_mapping():
-        cur.execute("""
-            SELECT id,(ctid::text::point)[0]::bigint pageno FROM data ORDER BY id
-        """)
-        return cur.fetchall()
-
-    initial_mapping = fetch_id_to_page_mapping()
-
-    # every iteration updates one tuple in each page
-    delta_stack_height = shape.delta_stack_height
-    for i in range(0, delta_stack_height):
-        log.info(i)
-        cur.execute(f"UPDATE data set row = row||',u' where id % 6 = {i % 6}")
-        log.info(f"modified rows: {cur.rowcount}")
-        assert cur.rowcount == need_pages
-        settle_and_flush()
-        post_update_mapping = fetch_id_to_page_mapping()
-        assert initial_mapping == post_update_mapping, "Postgres should be doing HOT updates"
-
-    # Assert the layer count is what we expect it is
-    layer_map = vps_http.layer_map_info(tenant_id, timeline_id)
-    assert (
-        len(layer_map.delta_l0_layers()) == delta_stack_height + 1 + 1
-    )  # +1 for the initdb layer + 1 for the table creation & fill
-    assert len(layer_map.delta_l0_layers()) == len(layer_map.delta_layers())  # it's all L0s
-    assert len(layer_map.image_layers()) == 0  # no images

test_runner/performance/README.md

@@ -15,8 +15,7 @@ Some handy pytest flags for local development:
 - `-k` selects a test to run
 - `--timeout=0` disables our default timeout of 300s (see `setup.cfg`)
 - `--preserve-database-files` to skip cleanup
-- `--out-dir` to produce a JSON with the recorded test metrics.
-  There is a post-processing tool at `test_runner/performance/out_dir_to_csv.py`.
+- `--out-dir` to produce a JSON with the recorded test metrics
 
 # What performance tests do we have and how we run them
 

test_runner/performance/out_dir_to_csv.py

@@ -1,57 +0,0 @@
-# Tool to convert the JSON output from running a perf test with `--out-dir` to a CSV that
-# can be easily pasted into a spreadsheet for quick viz & analysis.
-# Check the `./README.md` in this directory for `--out-dir`.
-#
-# TODO: add the pytest.mark.parametrize to the json and make them columns here
-# https://github.com/neondatabase/neon/issues/11878
-
-import csv
-import json
-import os
-import sys
-
-
-def json_to_csv(json_file):
-    with open(json_file) as f:
-        data = json.load(f)
-
-    # Collect all possible metric names to form headers
-    all_metrics = set()
-    for result in data.get("result", []):
-        for metric in result.get("data", []):
-            all_metrics.add(metric["name"])
-
-    # Sort metrics for consistent output
-    metrics = sorted(list(all_metrics))
-
-    # Create headers
-    headers = ["suit"] + metrics
-
-    # Prepare rows
-    rows = []
-    for result in data.get("result", []):
-        row = {"suit": result["suit"]}
-
-        # Initialize all metrics to empty
-        for metric in metrics:
-            row[metric] = ""
-
-        # Fill in available metrics
-        for item in result.get("data", []):
-            row[item["name"]] = item["value"]
-
-        rows.append(row)
-
-    # Write to stdout as CSV
-    writer = csv.DictWriter(sys.stdout, fieldnames=headers)
-    writer.writeheader()
-    writer.writerows(rows)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print(f"Usage: python {os.path.basename(__file__)} <json_file>")
-        sys.exit(1)
-
-    json_file = sys.argv[1]
-    json_to_csv(json_file)

test_runner/performance/pageserver/test_page_service_batching.py

@@ -10,8 +10,7 @@ from typing import Any
 import pytest
 from fixtures.benchmark_fixture import MetricReport, NeonBenchmarker
 from fixtures.log_helper import log
-from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, PgBin
-from fixtures.pageserver.makelayers import l0stack
+from fixtures.neon_fixtures import NeonEnvBuilder, PgBin, wait_for_last_flush_lsn
 from fixtures.utils import humantime_to_ms
 
 TARGET_RUNTIME = 30
@@ -35,18 +34,28 @@ class PageServicePipeliningConfigPipelined(PageServicePipeliningConfig):
     mode: str = "pipelined"
 
 
-PS_IO_CONCURRENCY = ["sidecar-task"]
-PIPELINING_CONFIGS: list[PageServicePipeliningConfig] = []
+EXECUTION = ["concurrent-futures"]
+BATCHING = ["uniform-lsn", "scattered-lsn"]
+
+NON_BATCHABLE: list[PageServicePipeliningConfig] = [PageServicePipeliningConfigSerial()]
+for max_batch_size in [1, 32]:
+    for execution in EXECUTION:
+        for batching in BATCHING:
+            NON_BATCHABLE.append(
+                PageServicePipeliningConfigPipelined(max_batch_size, execution, batching)
+            )
+
+BATCHABLE: list[PageServicePipeliningConfig] = []
 for max_batch_size in [32]:
-    for execution in ["concurrent-futures"]:
-        for batching in ["scattered-lsn"]:
-            PIPELINING_CONFIGS.append(
+    for execution in EXECUTION:
+        for batching in BATCHING:
+            BATCHABLE.append(
                 PageServicePipeliningConfigPipelined(max_batch_size, execution, batching)
             )
 
 
 @pytest.mark.parametrize(
-    "tablesize_mib, pipelining_config, target_runtime, ps_io_concurrency, effective_io_concurrency, readhead_buffer_size, name",
+    "tablesize_mib, pipelining_config, target_runtime, effective_io_concurrency, readhead_buffer_size, name",
     [
         # batchable workloads should show throughput and CPU efficiency improvements
         *[
@@ -54,23 +63,20 @@ for max_batch_size in [32]:
                 50,
                 config,
                 TARGET_RUNTIME,
-                ps_io_concurrency,
                 100,
                 128,
                 f"batchable {dataclasses.asdict(config)}",
             )
-            for config in PIPELINING_CONFIGS
-            for ps_io_concurrency in PS_IO_CONCURRENCY
+            for config in BATCHABLE
         ],
     ],
 )
-def test_postgres_seqscan(
+def test_throughput(
     neon_env_builder: NeonEnvBuilder,
     zenbenchmark: NeonBenchmarker,
     tablesize_mib: int,
     pipelining_config: PageServicePipeliningConfig,
     target_runtime: int,
-    ps_io_concurrency: str,
     effective_io_concurrency: int,
     readhead_buffer_size: int,
     name: str,
@@ -91,10 +97,6 @@ def test_postgres_seqscan(
     If the compute provides pipeline depth (effective_io_concurrency=100), then
     pipelining configs, especially with max_batch_size>1 should yield dramatic improvements
     in all performance metrics.
-
-    We advance the LSN from a disruptor thread to simulate the effect of a workload with concurrent writes
-    in another table. The `scattered-lsn` batching mode handles this well whereas the
-    initial implementatin (`uniform-lsn`) would break the batch.
     """
 
     #
@@ -112,19 +114,7 @@ def test_postgres_seqscan(
         }
     )
     # For storing configuration as a metric, insert a fake 0 with labels with actual data
-    params.update(
-        {
-            "config": (
-                0,
-                {
-                    "labels": {
-                        "pipelining_config": dataclasses.asdict(pipelining_config),
-                        "ps_io_concurrency": ps_io_concurrency,
-                    }
-                },
-            )
-        }
-    )
+    params.update({"pipelining_config": (0, {"labels": dataclasses.asdict(pipelining_config)})})
 
     log.info("params: %s", params)
 
@@ -276,10 +266,7 @@ def test_postgres_seqscan(
         return iters
 
     env.pageserver.patch_config_toml_nonrecursive(
-        {
-            "page_service_pipelining": dataclasses.asdict(pipelining_config),
-            "get_vectored_concurrent_io": {"mode": ps_io_concurrency},
-        }
+        {"page_service_pipelining": dataclasses.asdict(pipelining_config)}
     )
 
     # set trace for log analysis below
@@ -331,63 +318,77 @@ def test_postgres_seqscan(
     )
 
 
+PRECISION_CONFIGS: list[PageServicePipeliningConfig] = [PageServicePipeliningConfigSerial()]
+for max_batch_size in [1, 32]:
+    for execution in EXECUTION:
+        for batching in BATCHING:
+            PRECISION_CONFIGS.append(
+                PageServicePipeliningConfigPipelined(max_batch_size, execution, batching)
+            )
+
+
 @pytest.mark.parametrize(
-    "pipelining_config,ps_io_concurrency,l0_stack_height,queue_depth,name",
-    [
-        (config, ps_io_concurrency, l0_stack_height, queue_depth, f"{dataclasses.asdict(config)}")
-        for config in PIPELINING_CONFIGS
-        for ps_io_concurrency in PS_IO_CONCURRENCY
-        for queue_depth in [1, 2, 32]
-        for l0_stack_height in [0, 20]
-    ],
+    "pipelining_config,name",
+    [(config, f"{dataclasses.asdict(config)}") for config in PRECISION_CONFIGS],
 )
-def test_random_reads(
+def test_latency(
     neon_env_builder: NeonEnvBuilder,
     zenbenchmark: NeonBenchmarker,
     pg_bin: PgBin,
     pipelining_config: PageServicePipeliningConfig,
-    ps_io_concurrency: str,
-    l0_stack_height: int,
-    queue_depth: int,
     name: str,
 ):
     """
-    Throw pagebench random getpage at latest lsn workload from a single client against pageserver.
+    Measure the latency impact of pipelining in an un-batchable workloads.
+
+    An ideal implementation should not increase average or tail latencies for such workloads.
+
+    We don't have support in pagebench to create queue depth yet.
+    => https://github.com/neondatabase/neon/issues/9837
     """
 
     #
     # Setup
     #
 
-    def build_snapshot_cb(neon_env_builder: NeonEnvBuilder) -> NeonEnv:
-        env = neon_env_builder.init_start()
-        endpoint = env.endpoints.create_start("main")
-        l0stack.make_l0_stack(
-            endpoint,
-            l0stack.L0StackShape(logical_table_size_mib=50, delta_stack_height=l0_stack_height),
-        )
-        return env
-
-    env = neon_env_builder.build_and_use_snapshot(
-        f"test_page_service_batching--test_pagebench-{l0_stack_height}", build_snapshot_cb
-    )
-
     def patch_ps_config(ps_config):
-        ps_config["page_service_pipelining"] = dataclasses.asdict(pipelining_config)
-        ps_config["get_vectored_concurrent_io"] = {"mode": ps_io_concurrency}
+        if pipelining_config is not None:
+            ps_config["page_service_pipelining"] = dataclasses.asdict(pipelining_config)
 
-    env.pageserver.edit_config_toml(patch_ps_config)
+    neon_env_builder.pageserver_config_override = patch_ps_config
 
-    env.start()
+    env = neon_env_builder.init_start()
+    endpoint = env.endpoints.create_start("main")
+    conn = endpoint.connect()
+    cur = conn.cursor()
 
-    lsn = env.safekeepers[0].get_commit_lsn(env.initial_tenant, env.initial_timeline)
-    ep = env.endpoints.create_start("main", lsn=lsn)
-    data_table_relnode_oid = ep.safe_psql_scalar("SELECT 'data'::regclass::oid")
-    ep.stop_and_destroy()
+    cur.execute("SET max_parallel_workers_per_gather=0")  # disable parallel backends
+    cur.execute("SET effective_io_concurrency=1")
+
+    cur.execute("CREATE EXTENSION IF NOT EXISTS neon;")
+    cur.execute("CREATE EXTENSION IF NOT EXISTS neon_test_utils;")
+
+    log.info("Filling the table")
+    cur.execute("CREATE TABLE t (data char(1000)) with (fillfactor=10)")
+    tablesize = 50 * 1024 * 1024
+    npages = tablesize // (8 * 1024)
+    cur.execute("INSERT INTO t SELECT generate_series(1, %s)", (npages,))
+    # TODO: can we force postgres to do sequential scans?
+
+    cur.close()
+    conn.close()
+
+    wait_for_last_flush_lsn(env, endpoint, env.initial_tenant, env.initial_timeline)
+
+    endpoint.stop()
 
     for sk in env.safekeepers:
         sk.stop()
 
+    #
+    # Run single-threaded pagebench (TODO: dedup with other benchmark code)
+    #
+
     env.pageserver.allowed_errors.append(
         # https://github.com/neondatabase/neon/issues/6925
         r".*query handler for.*pagestream.*failed: unexpected message: CopyFail during COPY.*"
@@ -395,8 +396,6 @@ def test_random_reads(
 
     ps_http = env.pageserver.http_client()
 
-    metrics_before = ps_http.get_metrics()
-
     cmd = [
         str(env.neon_binpath / "pagebench"),
         "get-page-latest-lsn",
@@ -406,10 +405,6 @@ def test_random_reads(
         env.pageserver.connstr(password=None),
         "--num-clients",
         "1",
-        "--queue-depth",
-        str(queue_depth),
-        "--only-relnode",
-        str(data_table_relnode_oid),
         "--runtime",
         "10s",
     ]
@@ -418,22 +413,12 @@ def test_random_reads(
     results_path = Path(basepath + ".stdout")
     log.info(f"Benchmark results at: {results_path}")
 
-    metrics_after = ps_http.get_metrics()
-
     with open(results_path) as f:
         results = json.load(f)
     log.info(f"Results:\n{json.dumps(results, sort_keys=True, indent=2)}")
 
     total = results["total"]
 
-    metric = "request_count"
-    zenbenchmark.record(
-        metric,
-        metric_value=total[metric],
-        unit="",
-        report=MetricReport.HIGHER_IS_BETTER,
-    )
-
     metric = "latency_mean"
     zenbenchmark.record(
         metric,
@@ -450,17 +435,3 @@ def test_random_reads(
             unit="ms",
             report=MetricReport.LOWER_IS_BETTER,
         )
-
-    reads_before = metrics_before.query_one(
-        "pageserver_io_operations_seconds_count", filter={"operation": "read"}
-    )
-    reads_after = metrics_after.query_one(
-        "pageserver_io_operations_seconds_count", filter={"operation": "read"}
-    )
-
-    zenbenchmark.record(
-        "virtual_file_reads",
-        metric_value=reads_after.value - reads_before.value,
-        unit="",
-        report=MetricReport.LOWER_IS_BETTER,
-    )

test_runner/regress/test_replica_start.py

@@ -27,8 +27,9 @@ from contextlib import closing
 
 import psycopg2
 import pytest
+from fixtures.common_types import Lsn
 from fixtures.log_helper import log
-from fixtures.neon_fixtures import NeonEnv, wait_for_last_flush_lsn, wait_replica_caughtup
+from fixtures.neon_fixtures import NeonEnv, PgBin, wait_for_last_flush_lsn, wait_replica_caughtup
 from fixtures.pg_version import PgVersion
 from fixtures.utils import query_scalar, skip_on_postgres, wait_until
 
@@ -695,3 +696,110 @@ def test_replica_start_with_too_many_unused_xids(neon_simple_env: NeonEnv):
     with secondary.cursor() as secondary_cur:
         secondary_cur.execute("select count(*) from t")
         assert secondary_cur.fetchone() == (n_restarts,)
+
+
+def test_ephemeral_endpoints_vacuum(neon_simple_env: NeonEnv, pg_bin: PgBin):
+    env = neon_simple_env
+    endpoint = env.endpoints.create_start("main")
+
+    sql = """
+CREATE TABLE CHAR_TBL(f1 char(4));
+CREATE TABLE FLOAT8_TBL(f1 float8);
+CREATE TABLE INT2_TBL(f1 int2);
+CREATE TABLE INT4_TBL(f1 int4);
+CREATE TABLE INT8_TBL(q1 int8, q2 int8);
+CREATE TABLE POINT_TBL(f1 point);
+CREATE TABLE TEXT_TBL (f1 text);
+CREATE TABLE VARCHAR_TBL(f1 varchar(4));
+CREATE TABLE onek (unique1		int4);
+CREATE TABLE onek2 AS SELECT * FROM onek;
+CREATE TABLE tenk1 (unique1		int4);
+CREATE TABLE tenk2 AS SELECT * FROM tenk1;
+CREATE TABLE person (name text, age int4,location point);
+CREATE TABLE emp (salary int4, manager name) INHERITS (person);
+CREATE TABLE student (gpa float8) INHERITS (person);
+CREATE TABLE stud_emp (	percent 	int4) INHERITS (emp, student);
+CREATE TABLE road (name		text,thepath 	path);
+CREATE TABLE ihighway () INHERITS (road);
+CREATE TABLE shighway(surface		text) INHERITS (road);
+CREATE TABLE BOOLTBL3 (d text, b bool, o int);
+CREATE TABLE booltbl4(isfalse bool, istrue bool, isnul bool);
+DROP TABLE BOOLTBL3;
+DROP TABLE BOOLTBL4;
+CREATE TABLE ceil_floor_round (a numeric);
+DROP TABLE ceil_floor_round;
+CREATE TABLE width_bucket_test (operand_num numeric, operand_f8 float8);
+DROP TABLE width_bucket_test;
+CREATE TABLE num_input_test (n1 numeric);
+CREATE TABLE num_variance (a numeric);
+INSERT INTO num_variance VALUES (0);
+CREATE TABLE snapshot_test (nr	integer, snap	txid_snapshot);
+CREATE TABLE guid1(guid_field UUID, text_field TEXT DEFAULT(now()));
+CREATE TABLE guid2(guid_field UUID, text_field TEXT DEFAULT(now()));
+CREATE INDEX guid1_btree ON guid1 USING BTREE (guid_field);
+CREATE INDEX guid1_hash  ON guid1 USING HASH  (guid_field);
+TRUNCATE guid1;
+DROP TABLE guid1;
+DROP TABLE guid2 CASCADE;
+CREATE TABLE numrange_test (nr NUMRANGE);
+CREATE INDEX numrange_test_btree on numrange_test(nr);
+CREATE TABLE numrange_test2(nr numrange);
+CREATE INDEX numrange_test2_hash_idx on numrange_test2 using hash (nr);
+INSERT INTO numrange_test2 VALUES('[, 5)');
+CREATE TABLE textrange_test (tr text);
+CREATE INDEX textrange_test_btree on textrange_test(tr);
+CREATE TABLE test_range_gist(ir int4range);
+CREATE INDEX test_range_gist_idx on test_range_gist using gist (ir);
+DROP INDEX test_range_gist_idx;
+CREATE INDEX test_range_gist_idx on test_range_gist using gist (ir);
+CREATE TABLE test_range_spgist(ir int4range);
+CREATE INDEX test_range_spgist_idx on test_range_spgist using spgist (ir);
+DROP INDEX test_range_spgist_idx;
+CREATE INDEX test_range_spgist_idx on test_range_spgist using spgist (ir);
+CREATE TABLE test_range_elem(i int4);
+CREATE INDEX test_range_elem_idx on test_range_elem (i);
+CREATE INDEX ON test_range_elem using spgist(int4range(i,i+10));
+DROP TABLE test_range_elem;
+CREATE TABLE test_range_excl(room int4range, speaker int4range, during tsrange, exclude using gist (room with =, during with &&), exclude using gist (speaker with =, during with &&));
+CREATE TABLE f_test(f text, i int);
+CREATE TABLE i8r_array (f1 int, f2 text);
+CREATE TYPE arrayrange as range (subtype=int4[]);
+CREATE TYPE two_ints as (a int, b int);
+DROP TYPE two_ints cascade;
+CREATE TABLE text_support_test (t text);
+CREATE TABLE TEMP_FLOAT (f1 FLOAT8);
+CREATE TABLE TEMP_INT4 (f1 INT4);
+CREATE TABLE TEMP_INT2 (f1 INT2);
+CREATE TABLE TEMP_GROUP (f1 INT4, f2 INT4, f3 FLOAT8);
+CREATE TABLE POLYGON_TBL(f1 polygon);
+CREATE TABLE quad_poly_tbl (id int, p polygon);
+INSERT INTO quad_poly_tbl SELECT (x - 1) * 100 + y, polygon(circle(point(x * 10, y * 10), 1 + (x + y) % 10)) FROM generate_series(1, 200) x, generate_series(1, 100) y;
+CREATE TABLE quad_poly_tbl_ord_seq2 AS SELECT 1 FROM quad_poly_tbl;
+CREATE TABLE quad_poly_tbl_ord_idx2 AS SELECT 1 FROM quad_poly_tbl;
+"""
+
+    with endpoint.cursor() as cur:
+        lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()"))
+        env.endpoints.create_start(branch_name="main", lsn=lsn)
+        log.info(f"lsn: {lsn}")
+
+        for line in sql.split("\n"):
+            if len(line.strip()) == 0 or line.startswith("--"):
+                continue
+            cur.execute(line)
+
+        lsn = Lsn(query_scalar(cur, "SELECT pg_current_wal_flush_lsn()"))
+        env.endpoints.create_start(branch_name="main", lsn=lsn)
+        log.info(f"lsn: {lsn}")
+
+        cur.execute("VACUUM FULL pg_class;")
+
+    for ep in env.endpoints.endpoints:
+        log.info(f"{ep.endpoint_id} / {ep.pg_port}")
+        pg_dump_command = ["pg_dumpall", "-f", f"/tmp/dump-{ep.endpoint_id}.sql"]
+        env_vars = {
+            "PGPORT": str(ep.pg_port),
+            "PGUSER": endpoint.default_options["user"],
+            "PGHOST": endpoint.default_options["host"],
+        }
+        pg_bin.run_capture(pg_dump_command, env=env_vars)

vendor/revisions.json

@@ -5,14 +5,14 @@
   ],
   "v16": [
     "16.9",
-    "15710a76b7d07912110fcbbaf0c8ad6d7e5a9fbc"
+    "bb5eee65ac753b5a66d255ec5fb4c0e33180e8fd"
   ],
   "v15": [
     "15.13",
-    "daa81cffcf063c54b29a9aabdb6604625f675ad0"
+    "052df87d338dc30687d0c96f1a4d9b6cb4882b2e"
   ],
   "v14": [
     "14.18",
-    "4cca6f8083483dda9e12eae292cf788d45bd561f"
+    "ead1e76bdcb71ef87f52f0610bd7333247f75179"
   ]
 }

workspace_hack/Cargo.toml

@@ -60,8 +60,7 @@ lazy_static = { version = "1", default-features = false, features = ["spin_no_st
 libc = { version = "0.2", features = ["extra_traits", "use_std"] }
 log = { version = "0.4", default-features = false, features = ["std"] }
 memchr = { version = "2" }
-nix-2f80eeee3b1b6c7e = { package = "nix", version = "0.26" }
-nix-fa1f6196edfd7249 = { package = "nix", version = "0.30", features = ["dir", "ioctl", "mman", "poll", "signal", "socket"] }
+nix = { version = "0.26" }
 nom = { version = "7" }
 num = { version = "0.4" }
 num-bigint = { version = "0.4" }