proxy: refactor redis batching

.github/actions/neon-project-create/action.yml

@@ -49,6 +49,10 @@ inputs:
     description: 'A JSON object with project settings'
     required: false
     default: '{}'
+  default_endpoint_settings:
+    description: 'A JSON object with the default endpoint settings'
+    required: false
+    default: '{}'
 
 outputs:
   dsn:
@@ -135,6 +139,21 @@ runs:
             -H "Accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer ${ADMIN_API_KEY}" \
             -d "{\"scheduling\": \"Essential\"}"
         fi
+        # XXX
+        # This is a workaround for the default endpoint settings, which currently do not allow some settings in the public API.
+        # https://github.com/neondatabase/cloud/issues/27108
+        if [[ -n ${DEFAULT_ENDPOINT_SETTINGS} && ${DEFAULT_ENDPOINT_SETTINGS} != "{}" ]] ; then
+          PROJECT_DATA=$(curl -X GET \
+              "https://${API_HOST}/regions/${REGION_ID}/api/v1/admin/projects/${project_id}" \
+              -H "Accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer ${ADMIN_API_KEY}" \
+              -d "{\"scheduling\": \"Essential\"}"
+          )
+          NEW_DEFAULT_ENDPOINT_SETTINGS=$(echo ${PROJECT_DATA} | jq -rc ".project.default_endpoint_settings + ${DEFAULT_ENDPOINT_SETTINGS}")
+          curl -X POST --fail \
+                "https://${API_HOST}/regions/${REGION_ID}/api/v1/admin/projects/${project_id}/default_endpoint_settings" \
+                -H "Accept: application/json" -H "Content-Type: application/json" -H "Authorization: Bearer ${ADMIN_API_KEY}" \
+                --data "${NEW_DEFAULT_ENDPOINT_SETTINGS}"
+        fi
         
 
       env:
@@ -152,3 +171,4 @@ runs:
         PSQL: ${{ inputs.psql_path }}
         LD_LIBRARY_PATH: ${{ inputs.libpq_lib_path }}
         PROJECT_SETTINGS: ${{ inputs.project_settings }}
+        DEFAULT_ENDPOINT_SETTINGS: ${{ inputs.default_endpoint_settings }}

.github/workflows/_build-and-test-locally.yml

@@ -279,14 +279,18 @@ jobs:
           # run all non-pageserver tests
           ${cov_prefix} cargo nextest run $CARGO_FLAGS $CARGO_FEATURES -E '!package(pageserver)'
 
-          # run pageserver tests
-          # (When developing new pageserver features gated by config fields, we commonly make the rust
-          # unit tests sensitive to an environment variable NEON_PAGESERVER_UNIT_TEST_FEATURENAME.
-          # Then run the nextest invocation below for all relevant combinations. Singling out the
-          # pageserver tests from non-pageserver tests cuts down the time it takes for this CI step.)
-          NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=tokio-epoll-uring  \
-          ${cov_prefix} \
-          cargo nextest run $CARGO_FLAGS $CARGO_FEATURES  -E 'package(pageserver)'
+          # run pageserver tests with different settings
+          for get_vectored_concurrent_io in sequential sidecar-task; do
+            for io_engine in std-fs tokio-epoll-uring ; do
+                for io_mode in buffered direct direct-rw ; do
+                  NEON_PAGESERVER_UNIT_TEST_GET_VECTORED_CONCURRENT_IO=$get_vectored_concurrent_io \
+                  NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IOENGINE=$io_engine \
+                  NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IO_MODE=$io_mode \
+                  ${cov_prefix} \
+                  cargo nextest run $CARGO_FLAGS $CARGO_FEATURES  -E 'package(pageserver)'
+              done
+            done
+          done
 
           # Run separate tests for real S3
           export ENABLE_REAL_S3_REMOTE_STORAGE=nonempty
@@ -401,6 +405,8 @@ jobs:
           CHECK_ONDISK_DATA_COMPATIBILITY: nonempty
           BUILD_TAG: ${{ inputs.build-tag }}
           PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring
+          PAGESERVER_GET_VECTORED_CONCURRENT_IO: sidecar-task
+          PAGESERVER_VIRTUAL_FILE_IO_MODE: direct-rw
           USE_LFC: ${{ matrix.lfc_state == 'with-lfc' && 'true' || 'false' }}
 
       # Temporary disable this step until we figure out why it's so flaky

.github/workflows/build_and_test.yml

@@ -323,6 +323,8 @@ jobs:
           PERF_TEST_RESULT_CONNSTR: "${{ secrets.PERF_TEST_RESULT_CONNSTR }}"
           TEST_RESULT_CONNSTR: "${{ secrets.REGRESS_TEST_RESULT_CONNSTR_NEW }}"
           PAGESERVER_VIRTUAL_FILE_IO_ENGINE: tokio-epoll-uring
+          PAGESERVER_GET_VECTORED_CONCURRENT_IO: sidecar-task
+          PAGESERVER_VIRTUAL_FILE_IO_MODE: direct-rw
           SYNC_BETWEEN_TESTS: true
       # XXX: no coverage data handling here, since benchmarks are run on release builds,
       # while coverage is currently collected for the debug ones

.github/workflows/cloud-extensions.yml

@@ -35,7 +35,7 @@ jobs:
       matrix:
         pg-version: [16, 17]
 
-    runs-on: us-east-2
+    runs-on: [ self-hosted, small ]
     container:
       # We use the neon-test-extensions image here as it contains the source code for the extensions.
       image: ghcr.io/neondatabase/neon-test-extensions-v${{ matrix.pg-version }}:latest
@@ -71,7 +71,20 @@ jobs:
           region_id: ${{ inputs.region_id || 'aws-us-east-2' }}
           postgres_version: ${{ matrix.pg-version }}
           project_settings: ${{ steps.project-settings.outputs.settings }}
+          # We need these settings to get the expected output results.
+          # We cannot use the environment variables e.g. PGTZ due to
+          # https://github.com/neondatabase/neon/issues/1287
+          default_endpoint_settings: >
+            {
+              "pg_settings": {
+                "DateStyle": "Postgres,MDY",
+                "TimeZone": "America/Los_Angeles",
+                "compute_query_id": "off",
+                "neon.allow_unstable_extensions": "on"
+              }
+            }
           api_key: ${{ secrets.NEON_STAGING_API_KEY }}
+          admin_api_key: ${{ secrets.NEON_STAGING_ADMIN_API_KEY }}
 
       - name: Run the regression tests
         run: /run-tests.sh -r /ext-src

.github/workflows/neon_extra_builds.yml

@@ -63,10 +63,8 @@ jobs:
 
       - name: Filter out only v-string for build matrix
         id: postgres_changes
-        env:
-          CHANGES: ${{ steps.files_changed.outputs.changes }}
         run: |
-          v_strings_only_as_json_array=$(echo ${CHANGES} | jq '.[]|select(test("v\\d+"))' | jq --slurp -c)
+          v_strings_only_as_json_array=$(echo ${{ steps.files_changed.outputs.chnages }} | jq '.[]|select(test("v\\d+"))' | jq --slurp -c)
           echo "changes=${v_strings_only_as_json_array}" | tee -a "${GITHUB_OUTPUT}"
 
   check-macos-build:

Cargo.lock

@@ -1112,12 +1112,6 @@ version = "1.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
-[[package]]
-name = "cfg_aliases"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
-
 [[package]]
 name = "cgroups-rs"
 version = "0.3.3"
@@ -1312,7 +1306,7 @@ dependencies = [
  "itertools 0.10.5",
  "jsonwebtoken",
  "metrics",
- "nix 0.30.1",
+ "nix 0.27.1",
  "notify",
  "num_cpus",
  "once_cell",
@@ -1435,7 +1429,7 @@ dependencies = [
  "humantime-serde",
  "hyper 0.14.30",
  "jsonwebtoken",
- "nix 0.30.1",
+ "nix 0.27.1",
  "once_cell",
  "pageserver_api",
  "pageserver_client",
@@ -3518,9 +3512,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.172"
+version = "0.2.169"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa"
+checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a"
 
 [[package]]
 name = "libloading"
@@ -3827,13 +3821,12 @@ dependencies = [
 
 [[package]]
 name = "nix"
-version = "0.30.1"
+version = "0.27.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
+checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053"
 dependencies = [
  "bitflags 2.8.0",
  "cfg-if",
- "cfg_aliases",
  "libc",
  "memoffset 0.9.0",
 ]
@@ -4287,7 +4280,7 @@ dependencies = [
  "jsonwebtoken",
  "md5",
  "metrics",
- "nix 0.30.1",
+ "nix 0.27.1",
  "num-traits",
  "num_cpus",
  "once_cell",
@@ -4338,7 +4331,6 @@ dependencies = [
  "toml_edit",
  "tracing",
  "tracing-utils",
- "twox-hash",
  "url",
  "utils",
  "uuid",
@@ -4363,7 +4355,7 @@ dependencies = [
  "humantime",
  "humantime-serde",
  "itertools 0.10.5",
- "nix 0.30.1",
+ "nix 0.27.1",
  "once_cell",
  "postgres_backend",
  "postgres_ffi",
@@ -4856,19 +4848,6 @@ dependencies = [
  "workspace_hack",
 ]
 
-[[package]]
-name = "posthog_client_lite"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "reqwest",
- "serde",
- "serde_json",
- "sha2",
- "thiserror 1.0.69",
- "workspace_hack",
-]
-
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -7906,7 +7885,7 @@ dependencies = [
  "humantime",
  "jsonwebtoken",
  "metrics",
- "nix 0.30.1",
+ "nix 0.27.1",
  "once_cell",
  "pem",
  "pin-project-lite",
@@ -8460,10 +8439,8 @@ dependencies = [
  "fail",
  "form_urlencoded",
  "futures-channel",
- "futures-core",
  "futures-executor",
  "futures-io",
- "futures-task",
  "futures-util",
  "generic-array",
  "getrandom 0.2.11",
@@ -8493,7 +8470,6 @@ dependencies = [
  "once_cell",
  "p256 0.13.2",
  "parquet",
- "percent-encoding",
  "prettyplease",
  "proc-macro2",
  "prost 0.13.3",

Cargo.toml

@@ -26,7 +26,6 @@ members = [
     "libs/utils",
     "libs/consumption_metrics",
     "libs/postgres_backend",
-    "libs/posthog_client_lite",
     "libs/pq_proto",
     "libs/tenant_size_model",
     "libs/metrics",
@@ -127,7 +126,7 @@ md5 = "0.7.0"
 measured = { version = "0.0.22", features=["lasso"] }
 measured-process = { version = "0.0.22" }
 memoffset = "0.9"
-nix = { version = "0.30.1", features = ["dir", "fs", "process", "socket", "signal", "poll"] }
+nix = { version = "0.27", features = ["dir", "fs", "process", "socket", "signal", "poll"] }
 # Do not update to >= 7.0.0, at least. The update will have a significant impact
 # on compute startup metrics (start_postgres_ms), >= 25% degradation.
 notify = "6.0.0"

build-tools.Dockerfile

@@ -144,7 +144,6 @@ RUN set -e \
         openssh-client \
         parallel \
         pkg-config \
-        sudo \
         unzip \
         wget \
         xz-utils \
@@ -293,7 +292,7 @@ WORKDIR /home/nonroot
 
 # Rust
 # Please keep the version of llvm (installed above) in sync with rust llvm (`rustc --version --verbose | grep LLVM`)
-ENV RUSTC_VERSION=1.87.0
+ENV RUSTC_VERSION=1.86.0
 ENV RUSTUP_HOME="/home/nonroot/.rustup"
 ENV PATH="/home/nonroot/.cargo/bin:${PATH}"
 ARG RUSTFILT_VERSION=0.2.1

compute/compute-node.Dockerfile

@@ -1117,8 +1117,8 @@ RUN wget https://github.com/microsoft/onnxruntime/archive/refs/tags/v1.18.1.tar.
     mkdir onnxruntime-src && cd onnxruntime-src && tar xzf ../onnxruntime.tar.gz --strip-components=1 -C . && \
     echo "#nothing to test here" > neon-test.sh
 
-RUN wget https://github.com/neondatabase-labs/pgrag/archive/refs/tags/v0.1.2.tar.gz -O pgrag.tar.gz &&  \
-    echo "7361654ea24f08cbb9db13c2ee1c0fe008f6114076401bb871619690dafc5225 pgrag.tar.gz" | sha256sum --check && \
+RUN wget https://github.com/neondatabase-labs/pgrag/archive/refs/tags/v0.1.1.tar.gz -O pgrag.tar.gz &&  \
+    echo "087b2ecd11ba307dc968042ef2e9e43dc04d9ba60e8306e882c407bbe1350a50 pgrag.tar.gz" | sha256sum --check && \
     mkdir pgrag-src && cd pgrag-src && tar xzf ../pgrag.tar.gz --strip-components=1 -C .
 
 FROM rust-extensions-build-pgrx14 AS pgrag-build

control_plane/src/background_process.rs

@@ -14,7 +14,7 @@
 
 use std::ffi::OsStr;
 use std::io::Write;
-use std::os::fd::AsFd;
+use std::os::unix::prelude::AsRawFd;
 use std::os::unix::process::CommandExt;
 use std::path::Path;
 use std::process::Command;
@@ -356,7 +356,7 @@ where
             let file = pid_file::claim_for_current_process(&path).expect("claim pid file");
             // Remove the FD_CLOEXEC flag on the pidfile descriptor so that the pidfile
             // remains locked after exec.
-            nix::fcntl::fcntl(file.as_fd(), FcntlArg::F_SETFD(FdFlag::empty()))
+            nix::fcntl::fcntl(file.as_raw_fd(), FcntlArg::F_SETFD(FdFlag::empty()))
                 .expect("remove FD_CLOEXEC");
             // Don't run drop(file), it would close the file before we actually exec.
             std::mem::forget(file);

control_plane/src/bin/neon_local.rs

@@ -8,6 +8,7 @@
 use std::borrow::Cow;
 use std::collections::{BTreeSet, HashMap};
 use std::fs::File;
+use std::os::fd::AsRawFd;
 use std::path::PathBuf;
 use std::process::exit;
 use std::str::FromStr;
@@ -30,7 +31,7 @@ use control_plane::safekeeper::SafekeeperNode;
 use control_plane::storage_controller::{
     NeonStorageControllerStartArgs, NeonStorageControllerStopArgs, StorageController,
 };
-use nix::fcntl::{Flock, FlockArg};
+use nix::fcntl::{FlockArg, flock};
 use pageserver_api::config::{
     DEFAULT_HTTP_LISTEN_PORT as DEFAULT_PAGESERVER_HTTP_PORT,
     DEFAULT_PG_LISTEN_PORT as DEFAULT_PAGESERVER_PG_PORT,
@@ -748,16 +749,16 @@ struct TimelineTreeEl {
 
 /// A flock-based guard over the neon_local repository directory
 struct RepoLock {
-    _file: Flock<File>,
+    _file: File,
 }
 
 impl RepoLock {
     fn new() -> Result<Self> {
         let repo_dir = File::open(local_env::base_path())?;
-        match Flock::lock(repo_dir, FlockArg::LockExclusive) {
-            Ok(f) => Ok(Self { _file: f }),
-            Err((_, e)) => Err(e).context("flock error"),
-        }
+        let repo_dir_fd = repo_dir.as_raw_fd();
+        flock(repo_dir_fd, FlockArg::LockExclusive)?;
+
+        Ok(Self { _file: repo_dir })
     }
 }
 

docker-compose/ext-src/alter_db.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-# We need these settings to get the expected output results.
-# We cannot use the environment variables e.g. PGTZ due to
-# https://github.com/neondatabase/neon/issues/1287
-export DATABASE=${1:-contrib_regression}
-psql -c "ALTER DATABASE ${DATABASE} SET neon.allow_unstable_extensions='on'" \
-     -c "ALTER DATABASE ${DATABASE} SET DateStyle='Postgres,MDY'" \
-     -c "ALTER DATABASE ${DATABASE} SET TimeZone='America/Los_Angeles'" \

docker-compose/ext-src/pg_graphql-src/regular-test.sh

@@ -18,7 +18,6 @@ TESTS=${TESTS/row_level_security/}
 TESTS=${TESTS/sqli_connection/}
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 psql -v ON_ERROR_STOP=1 -f test/fixtures.sql -d contrib_regression
 ${REGRESS} --use-existing --dbname=contrib_regression --inputdir=${TESTDIR} ${TESTS}
 

docker-compose/ext-src/pgrag-src/regular-test.sh

@@ -3,7 +3,6 @@ set -ex
 cd "$(dirname "${0}")"
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 psql -d contrib_regression -c "CREATE EXTENSION vector" -c "CREATE EXTENSION rag"
 PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
 ${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin'    --use-existing --load-extension=vector --load-extension=rag --dbname=contrib_regression basic_functions text_processing api_keys chunking_functions document_processing embedding_api_functions voyageai_functions

docker-compose/ext-src/pgx_ulid-src/Makefile

@@ -20,6 +20,5 @@ installcheck: regression-test
 regression-test:
 	dropdb --if-exists contrib_regression
 	createdb contrib_regression
-	../alter_db.sh
 	psql -d contrib_regression -c "CREATE EXTENSION $(EXTNAME)"
 	$(PG_REGRESS) --inputdir=. --outputdir=. --use-existing --dbname=contrib_regression $(REGRESS)

docker-compose/ext-src/plv8-src/regular-test.sh

@@ -3,7 +3,6 @@ set -ex
 cd "$(dirname ${0})"
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
 REGRESS="$(make -n installcheck | awk '{print substr($0,index($0,"init-extension"));}')"
 REGRESS="${REGRESS/startup_perms/}"

docker-compose/ext-src/rag_bge_small_en_v15-src/Makefile

@@ -11,6 +11,5 @@ PG_REGRESS := $(dir $(PGXS))../../src/test/regress/pg_regress
 installcheck:
 	dropdb --if-exists contrib_regression
 	createdb contrib_regression
-	../alter_db.sh
 	psql -d contrib_regression -c "CREATE EXTENSION vector" -c "CREATE EXTENSION rag_bge_small_en_v15"
 	$(PG_REGRESS) --use-existing --dbname=contrib_regression $(REGRESS)

docker-compose/ext-src/rag_jina_reranker_v1_tiny_en-src/Makefile

@@ -11,6 +11,5 @@ PG_REGRESS := $(dir $(PGXS))../../src/test/regress/pg_regress
 installcheck:
 	dropdb --if-exists contrib_regression
 	createdb contrib_regression
-	../alter_db.sh
 	psql -d contrib_regression -c "CREATE EXTENSION vector" -c "CREATE EXTENSION rag_jina_reranker_v1_tiny_en"
 	$(PG_REGRESS) --use-existing --dbname=contrib_regression $(REGRESS)

docker-compose/ext-src/rum-src/regular-test.sh

@@ -3,6 +3,5 @@ set -ex
 cd "$(dirname ${0})"
 dropdb --if-exist contrib_regression
 createdb contrib_regression
-. ../alter_db.sh
 PG_REGRESS=$(dirname "$(pg_config --pgxs)")/../test/regress/pg_regress
 ${PG_REGRESS} --inputdir=./ --bindir='/usr/local/pgsql/bin' --use-existing --dbname=contrib_regression rum rum_hash ruminv timestamp orderby orderby_hash altorder altorder_hash limits int2 int4 int8 float4 float8 money oid time timetz date interval macaddr inet cidr text varchar char bytea bit varbit numeric rum_weight expr array

docs/rfcs/043-bottom-most-gc-compaction.md

@@ -1,194 +0,0 @@
-# Bottommost Garbage-Collection Compaction
-
-## Summary
-
-The goal of this doc is to propose a way to reliably collect garbages below the GC horizon. This process is called bottom-most garbage-collect-compaction, and is part of the broader legacy-enhanced compaction that we plan to implement in the future.
-
-## Motivation
-
-The current GC algorithm will wait until the covering via image layers before collecting the garbages of a key region. Relying on image layer generation to generate covering images is not reliable. There are prior arts to generate feedbacks from the GC algorithm to the image generation process to accelerate garbage collection, but it slows down the system and creates write amplification.
-
-# Basic Idea
-
-![](images/036-bottom-most-gc-compaction/01-basic-idea.svg)
-
-The idea of bottom-most compaction is simple: we rewrite all layers that are below or intersect with the GC horizon to produce a flat level of image layers at the GC horizon and deltas above the GC horizon. In this process,
-
-- All images and deltas ≤ GC horizon LSN will be dropped. This process collects garbages.
-- We produce images for all keys involved in the compaction process at the GC horizon.
-
-Therefore, it can precisely collect all garbages below the horizon, and reduce the space amplification, i.e., in the staircase pattern (test_gc_feedback).
-
-![The staircase pattern in test_gc_feedback in the original compaction algorithm. The goal is to collect garbage below the red horizontal line.](images/036-bottom-most-gc-compaction/12-staircase-test-gc-feedback.png)
-
-The staircase pattern in test_gc_feedback in the original compaction algorithm. The goal is to collect garbage below the red horizontal line.
-
-# Branches
-
-With branches, the bottom-most compaction should retain a snapshot of the keyspace at the `retain_lsn` so that the child branch can access data at the branch point. This requires some modifications to the basic bottom-most compaction algorithm that we sketched above. 
-
-![](images/036-bottom-most-gc-compaction/03-retain-lsn.svg)
-
-## Single Timeline w/ Snapshots: handle `retain_lsn`
-
-First let’s look into the case where we create branches over the main branch but don’t write any data to them (aka “snapshots”).
-
-The bottom-most compaction algorithm collects all deltas and images of a key and can make decisions on what data to retain. Given that we have a single key’s history as below:
-
-```
-LSN 0x10 -> A
-LSN 0x20 -> append B
-retain_lsn: 0x20
-LSN 0x30 -> append C
-LSN 0x40 -> append D
-retain_lsn: 0x40
-LSN 0x50 -> append E
-GC horizon: 0x50
-LSN 0x60 -> append F
-```
-
-The algorithm will produce:
-
-```
-LSN 0x20 -> AB
-(drop all history below the earliest retain_lsn)
-LSN 0x40 -> ABCD
-(assume the cost of replaying 2 deltas is higher than storing the full image, we generate an image here)
-LSN 0x50 -> append E
-(replay one delta is cheap)
-LSN 0x60 -> append F
-(keep everything as-is above the GC horizon)
-```
-
-![](images/036-bottom-most-gc-compaction/05-btmgc-parent.svg)
-
-What happens is that we balance the space taken by each retain_lsn and the cost of replaying deltas during the bottom-most compaction process. This is controlled by a threshold. If `count(deltas) < $threshold`, the deltas will be retained. Otherwise, an image will be generated and the deltas will be dropped.
-
-In the example above, the `$threshold` is 2.
-
-## Child Branches with data: pull + partial images
-
-In the previous section we have shown how bottom-most compaction respects `retain_lsn` so that all data that was readable at branch creation remains readable. But branches can have data on their own, and that data can fall out of the branch’s PITR window. So, this section explains how we deal with that.
-
-We will run the same bottom-most compaction for these branches, to ensure the space amplification on the child branch is reasonable. 
-
-```
-branch_lsn: 0x20
-LSN 0x30 -> append P
-LSN 0x40 -> append Q
-LSN 0x50 -> append R
-GC horizon: 0x50
-LSN 0x60 -> append S
-```
-
-Note that bottom-most compaction happens on a per-timeline basis. When it processes this key, it only reads the history from LSN 0x30 without a base image. Therefore, on child branches, the bottom-most compaction process will make image creation decisions based on the same `count(deltas) < $threshold` criteria, and if it decides to create an image, the base image will be retrieved from the ancestor branch.
-
-```
-branch_lsn: 0x20
-LSN 0x50 -> ABPQR
-(we pull the image at LSN 0x20 from the ancestor branch to get AB, and then apply append PQ to the page; we replace the record at 0x40 with an image and drop the delta)
-GC horizon: 0x50
-LSN 0x60 -> append S
-```
-
-![](images/036-bottom-most-gc-compaction/06-btmgc-child.svg)
-
-Note that for child branches, we do not create image layers for the images when bottom-most compaction runs. Instead, we drop the 0x30/0x40/0x50 delta records and directly place the image ABPQR@0x50 into the delta layer, which serves as a sparse image layer. For child branches, if we create image layers, we will need to put all keys in the range into the image layer. This causes space bloat and slow compactions. In this proposal, the compaction process will only compact and process keys modified inside the child branch.
-
-# Result
-
-Bottom-most compaction ensures all garbage under the GC horizon gets collected right away (compared with “eventually” in the current algorithm). Meanwhile, it generates images at each of the retain_lsn to ensure branch reads are fast. As we make per-key decisions on whether to generate an image or not, the theoretical lower bound of the storage space we need to retain for a branch is lower than before.
-
-Before: min(sum(logs for each key), sum(image for each key)), for each partition — we always generate image layers on a key range
-
-After: sum(min(logs for each key, image for each key))
-
-# Compaction Trigger
-
-The bottom-most compaction can be automatically triggered. The goal of the trigger is that it should ensure a constant factor for write amplification. Say that the user write 1GB of WAL into the system, we should write 1GB x C data to S3. The legacy compaction algorithm does not have such a constant factor C. The data we write to S3 is quadratic to the logical size of the database (see [A Theoretical View of Neon Storage](https://www.notion.so/A-Theoretical-View-of-Neon-Storage-8d7ad7555b0c41b2a3597fa780911194?pvs=21)).
-
-We propose the following compaction trigger that generates a constant write amplification factor. Write amplification >= total writes to S3 / total user writes. We only analyze the write amplification caused by the bottom-most GC-compaction process, ignoring the legacy create image layers amplification.
-
-Given that we have ***X*** bytes of the delta layers above the GC horizon, ***A*** bytes of the delta layers intersecting with the GC horizon, ***B*** bytes of the delta layers below the GC horizon, and ***C*** bytes of the image layers below the GC horizon.
-
-The legacy GC + compaction loop will always keep ***A*** unchanged, reduce ***B and C*** when there are image layers covering the key range. This yields 0 write amplification (only file deletions) and extra ***B*** bytes of space.
-
-![](images/036-bottom-most-gc-compaction/09-btmgc-analysis-2.svg)
-
-The bottom-most compaction proposed here will split ***A*** into deltas above the GC horizon and below the GC horizon. Everything below the GC horizon will be image layers after the compaction (not considering branches). Therefore, this yields ***A+C*** extra write traffic each iteration, plus 0 extra space.
-
-![](images/036-bottom-most-gc-compaction/07-btmgc-analysis-1.svg)
-
-Also considering read amplification (below the GC horizon). When a read request reaches the GC horizon, the read amplification will be (A+B+C)/C=1+(A+B)/C. Reducing ***A*** and ***B*** can help reduce the read amplification below the GC horizon.
-
-The metrics-based trigger will wait until a point that space amplification is not that large and write amplification is not that large before the compaction gets triggered. The trigger is defined as **(A+B)/C ≥ 1 (or some other ratio)**.
-
-To reason about this trigger, consider the two cases:
-
-**Data Ingestion**
-
-User keeps ingesting data into the database, which indicates that WAL size roughly equals to the database logical size. The compaction gets triggered only when the newly-written WAL roughly equals to the current bottom-most image size (=X). Therefore, it’s triggered when the database size gets doubled. This is a reasonable amount of work. Write amplification is 2X/X=1 for the X amount of data written.
-
-![](images/036-bottom-most-gc-compaction/10-btmgc-analysis-3.svg)
-
-**Updates/Deletion**
-
-In this case, WAL size will be larger than the database logical size ***D***. The compaction gets triggered for every ***D*** bytes of WAL written. Therefore, for every ***D*** bytes of WAL, we rewrite the bottom-most layer, which produces an extra ***D*** bytes of write amplification. This incurs exactly 2x write amplification (by the write of D), 1.5x write amplification (if we count from the start of the process) and no space amplification. 
-
-![](images/036-bottom-most-gc-compaction/11-btmgc-analysis-4.svg)
-
-Note that here I try to reason that write amplification is a constant (i.e., the data we write to S3 is proportional to the data the user write). The main problem with the current legacy compaction algorithm is that write amplification is proportional to the database size.
-
-The next step is to optimize the write amplification above the GC horizon (i.e., change the image creation criteria, top-most compaction, or introduce tiered compaction), to ensure the write amplification of the whole system is a constant factor.
-
-20GB layers → +20GB layers → delete 20GB, need 40GB temporary space
-
-# Sub-Compactions
-
-The gc-compaction algorithm may take a long time and we need to split the job into multiple sub-compaction jobs.
-
-![](images/036-bottom-most-gc-compaction/13-job-split.svg)
-
-As in the figure, the auto-trigger schedules a compaction job covering the full keyspace below a specific LSN. In such case that we cannot finish compacting it in one run in a reasonable amount of time, the algorithm will vertically split it into multiple jobs (in this case, 5).
-
-Each gc-compaction job will create one level of delta layers and one flat level of image layers for each LSN. Those layers will be automatically split based on size, which means that if the sub-compaction job produces 1GB of deltas, it will produce 4 * 256MB delta layers. For those layers that is not fully contained within the sub-compaction job rectangles, it will be rewritten to only contain the keys outside of the key range.
-
-# Implementation
-
-The main implementation of gc-compaction is in `compaction.rs`.
-
-* `compact_with_gc`: The main loop of gc-compaction. It takes a rectangle range of the layer map and compact that specific range. It selects layers intersecting with the rectangle, downloads the layers, creates the k-merge iterator to read those layers in the key-lsn order, and decide which keys to keep or insert a reconstructed page. The process is the basic unit of a gc-compaction and is not interruptable. If the process gets preempted by L0 compaction, it has to be restarted from scratch. For layers overlaps with the rectangle but not fully inside, the main loop will also rewrite them so that the new layer (or two layers if both left and right ends are outside of the rectangle) has the same LSN range as the original one but only contain the keys outside of the compaction range.
-* `gc_compaction_split_jobs`: Splits a big gc-compaction job into sub-compactions based on heuristics in the layer map. The function looks at the layer map and splits the compaction job based on the size of the layers so that each compaction job only pulls ~4GB of layer files.
-* `generate_key_retention` and `KeyHistoryRetention`: Implements the algorithm described in the "basic idea" and "branch" chapter of this RFC. It takes a vector of history of a key (key-lsn-value) and decides which LSNs of the key to retain. If there are too many deltas between two retain_lsns, it will reconstruct the page and insert an image into the compaction result. Also, we implement `KeyHistoryRetention::verify` to ensure the generated result is not corrupted -- all retain_lsns and all LSNs above the gc-horizon should be accessible.
-* `GcCompactionQueue`: the automatic trigger implementation for gc-compaction. `GcCompactionQueue::iteration` is called at the end of the tenant compaction loop. It will then call `trigger_auto_compaction` to decide whether to trigger a gc-compaction job for this tenant. If yes, the compaction-job will be added to the compaction queue, and the queue will be slowly drained once there are no other compaction jobs running. gc-compaction has the lowest priority. If a sub-compaction job is not successful or gets preempted by L0 compaction (see limitations for reasons why a compaction job would fail), it will _not_ be retried.
-* Changes to `index_part.json`: we added a `last_completed_lsn` field to the index part for the auto-trigger to decide when to trigger a compaction.
-* Changes to the read path: when gc-compaction updates the layer map, all reads need to wait. See `gc_compaction_layer_update_lock` and comments in the code path for more information.
-
-Gc-compaction can also be scheduled over the HTTP API. Example:
-
-```
-curl 'localhost:9898/v1/tenant/:tenant_id/timeline/:timeline_id/compact?enhanced_gc_bottom_most_compaction=true&dry_run=true' -X PUT -H "Content-Type: application/json" -d '{"scheduled": true, "compact_key_range": { "start": "000000067F0000A0000002A1CF0100000000", "end": "000000067F0000A0000002A1D70100000000" } }'
-```
-
-The `dry_run` mode can be specified in the query string so that the compaction will go through all layers to estimate how much space can be saved without writing the compaction result into the layer map.
-
-The auto-trigger is controlled by tenant-level flag `gc_compaction_enabled`. If this is set to false, no gc-compaction will be automatically scheduled on this tenant (but manual trigger still works).
-
-# Next Steps
-
-There are still some limitations of gc-compaction itself that needs to be resolved and tested,
-
-- gc-compaction is currently only automatically triggered on root branches. We have not tested gc-compaction on child branches in staging.
-- gc-compaction will skip aux key regions because of the possible conflict with the assumption of aux file tombstones.
-- gc-compaction does not consider keyspaces at retain_lsns and only look at keys in the layers. This also causes us giving up some sub-compaction jobs because a key might have part of its history available due to traditional GC removing part of the history.
-- We limit gc-compaction to run over shards <= 150GB to avoid gc-compaction taking too much time blocking other compaction jobs. The sub-compaction split algorithm needs to be improved to be able to split vertically and horizontally. Also, we need to move the download layer process out of the compaction loop so that we don't block other compaction jobs for too long.
-- The compaction trigger always schedules gc-compaction from the lowest LSN to the gc-horizon. Currently we do not schedule compaction jobs that only selects layers in the middle. Allowing this could potentially reduce the number of layers read/write throughout the process.
-- gc-compaction will give up if there are too many layers to rewrite or if there are not enough disk space for the compaction.
-- gc-compaction sometimes fails with "no key produced during compaction", which means that all existing keys within the compaction range can be collected; but we don't have a way to write this information back to the layer map -- we cannot generate an empty image layer.
-- We limit the maximum size of deltas for a single key to 512MB. If above this size, gc-compaction will give up. This can be resolved by changing `generate_key_retention` to be a stream instead of requiring to collect all the key history.
-
-In the future,
-
-- Top-most compaction: ensure we always have an image coverage for the latest data (or near the latest data), so that reads will be fast at the latest LSN.
-- Tiered compaction on deltas: ensure read from any LSN is fast.
-- Per-timeline compaction → tenant-wide compaction?

docs/rfcs/images/036-bottom-most-gc-compaction/01-basic-idea.svg

@@ -1,135 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="82 284 863 375" width="863" height="375">
-  <defs/>
-  <g id="01-basic-idea" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>01-basic-idea</title>
-    <rect fill="white" x="82" y="284" width="863" height="375"/>
-    <g id="01-basic-idea_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_2">
-        <rect x="234" y="379.5" width="203.5" height="17.5" fill="white"/>
-        <rect x="234" y="379.5" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_3">
-        <rect x="453.5" y="379.5" width="203.5" height="17.5" fill="white"/>
-        <rect x="453.5" y="379.5" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_4">
-        <rect x="672.5" y="379.5" width="203.5" height="17.5" fill="white"/>
-        <rect x="672.5" y="379.5" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_5">
-        <rect x="234" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="234" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_6">
-        <rect x="375" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="375" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <rect x="516" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="516" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_8">
-        <rect x="657" y="288.5" width="127" height="77.5" fill="white"/>
-        <rect x="657" y="288.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_9">
-        <rect x="798" y="288.5" width="78" height="77.5" fill="white"/>
-        <rect x="798" y="288.5" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_11">
-        <line x1="185.5" y1="326.75" x2="943.7734" y2="326.75" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_12">
-        <text transform="translate(87 318.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_13">
-        <text transform="translate(106.41 372.886)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.39" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="29132252e-19" y="28.447998" xml:space="preserve">at earlier LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_14">
-        <text transform="translate(121.92 289.578)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_15">
-        <path d="M 517.125 423.5 L 553.375 423.5 L 553.375 482 L 571.5 482 L 535.25 512 L 499 482 L 517.125 482 Z" fill="white"/>
-        <path d="M 517.125 423.5 L 553.375 423.5 L 553.375 482 L 571.5 482 L 535.25 512 L 499 482 L 517.125 482 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <rect x="234" y="599.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="234" y="599.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <rect x="453.5" y="599.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="453.5" y="599.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_24">
-        <rect x="672.5" y="599.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="672.5" y="599.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_23">
-        <rect x="234" y="533" width="127" height="52.974" fill="white"/>
-        <rect x="234" y="533" width="127" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_22">
-        <rect x="375" y="533" width="310.5" height="52.974" fill="white"/>
-        <rect x="375" y="533" width="310.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_21">
-        <rect x="702.5" y="533" width="173.5" height="52.974" fill="white"/>
-        <rect x="702.5" y="533" width="173.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_18">
-        <line x1="185.5" y1="607.724" x2="943.7734" y2="607.724" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_16">
-        <text transform="translate(121.92 538)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_27">
-        <text transform="translate(114.8 592.86)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="3488765e-18" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="4.01" y="28.447998" xml:space="preserve">at GC LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <rect x="243.06836" y="300" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(248.06836 301.068)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_30">
-        <rect x="243.06836" y="335.5" width="624.3633" height="17.5" fill="#c0ffff"/>
-        <text transform="translate(248.06836 336.568)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.89414" y="12" xml:space="preserve">Deltas below GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_32">
-        <rect x="243.06836" y="550.737" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(248.06836 551.805)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_33">
-        <rect x="304" y="630.474" width="485.5" height="28.447998" fill="#c0ffff"/>
-        <text transform="translate(309 637.016)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="63.095" y="12" xml:space="preserve">Deltas and image below GC Horizon gets garbage-collected</tspan>
-        </text>
-      </g>
-      <g id="Graphic_34">
-        <text transform="translate(576.5 444.0325)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="0" y="11" xml:space="preserve">WAL replay of deltas+image below GC Horizon</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="0" y="25.336" xml:space="preserve">Reshuffle deltas</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/03-retain-lsn.svg

@@ -1,141 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-104 215 863 335" width="863" height="335">
-  <defs>
-    <marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -4 10 8" markerWidth="10" markerHeight="8" color="#7f8080">
-      <g>
-        <path d="M 8 0 L 0 -3 L 0 3 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
-      </g>
-    </marker>
-  </defs>
-  <g id="03-retain-lsn" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>03-retain-lsn</title>
-    <rect fill="white" x="-104" y="215" width="863" height="335"/>
-    <g id="03-retain-lsn_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_28">
-        <rect x="48" y="477" width="203.5" height="9.990005" fill="white"/>
-        <rect x="48" y="477" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_27">
-        <rect x="267.5" y="477" width="203.5" height="9.990005" fill="white"/>
-        <rect x="267.5" y="477" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <rect x="486.5" y="477" width="203.5" height="9.990005" fill="white"/>
-        <rect x="486.5" y="477" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-.5" y1="387.172" x2="757.7734" y2="387.172" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-99 378.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_31">
-        <rect x="48.25" y="410" width="203.5" height="9.990005" fill="white"/>
-        <rect x="48.25" y="410" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_30">
-        <rect x="267.75" y="410" width="203.5" height="9.990005" fill="white"/>
-        <rect x="267.75" y="410" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <rect x="486.75" y="410" width="203.5" height="9.990005" fill="white"/>
-        <rect x="486.75" y="410" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_34">
-        <rect x="48.25" y="431.495" width="113.75" height="34" fill="white"/>
-        <rect x="48.25" y="431.495" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_33">
-        <rect x="172.5" y="431.495" width="203.5" height="34" fill="white"/>
-        <rect x="172.5" y="431.495" width="203.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_32">
-        <rect x="386.5" y="431.495" width="303.5" height="34" fill="white"/>
-        <rect x="386.5" y="431.495" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_37">
-        <rect x="48" y="498.495" width="203.5" height="9.990005" fill="white"/>
-        <rect x="48" y="498.495" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_36">
-        <rect x="267.5" y="498.495" width="203.5" height="9.990005" fill="white"/>
-        <rect x="267.5" y="498.495" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_35">
-        <rect x="486.5" y="498.495" width="203.5" height="9.990005" fill="white"/>
-        <rect x="486.5" y="498.495" width="203.5" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_38">
-        <line x1="-10.48" y1="535.5395" x2="39.318294" y2="508.24794" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_39">
-        <text transform="translate(-96.984 526.3155)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 1</tspan>
-        </text>
-      </g>
-      <g id="Line_41">
-        <line x1="-10.48" y1="507.0915" x2="38.90236" y2="485.8992" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_40">
-        <text transform="translate(-96.984 497.8675)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 2</tspan>
-        </text>
-      </g>
-      <g id="Line_43">
-        <line x1="-10.48" y1="478.6435" x2="39.44267" y2="453.01616" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_42">
-        <text transform="translate(-96.984 469.4195)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 3</tspan>
-        </text>
-      </g>
-      <g id="Line_45">
-        <line x1="-10.48" y1="448.495" x2="39.65061" y2="419.90015" marker-end="url(#FilledArrow_Marker)" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_44">
-        <text transform="translate(-96.984 439.271)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 4</tspan>
-        </text>
-      </g>
-      <g id="Graphic_46">
-        <rect x="335.46477" y="215.5" width="353.4299" height="125.495" fill="white"/>
-        <rect x="335.46477" y="215.5" width="353.4299" height="125.495" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_48">
-        <text transform="translate(549.3766 317.547)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="6536993e-19" y="15" xml:space="preserve">Dependent Branch</tspan>
-        </text>
-      </g>
-      <g id="Graphic_50">
-        <text transform="translate(340.43824 317.547)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="40500936e-20" y="15" xml:space="preserve">retain_lsn 3</tspan>
-        </text>
-      </g>
-      <g id="Line_57">
-        <line x1="323.90685" y1="248.8045" x2="714.9232" y2="248.8045" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_56">
-        <text transform="translate(165.91346 240.0805)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="35811354e-19" y="15" xml:space="preserve">Branch GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_58">
-        <rect x="493.9232" y="301.6405" width="107.45294" height="9.990005" fill="white"/>
-        <rect x="493.9232" y="301.6405" width="107.45294" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_59">
-        <text transform="translate(358.9232 277.276)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">Partial Image Coverage</tspan>
-        </text>
-      </g>
-      <g id="Graphic_60">
-        <rect x="354.1732" y="301.6405" width="107.45294" height="9.990005" fill="white"/>
-        <rect x="354.1732" y="301.6405" width="107.45294" height="9.990005" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/05-btmgc-parent.svg

@@ -1,187 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-235 426 864 366" width="864" height="366">
-  <defs/>
-  <g id="05-btmgc-parent" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>05-btmgc-parent</title>
-    <rect fill="white" x="-235" y="426" width="864" height="366"/>
-    <g id="05-btmgc-parent_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_23">
-        <rect x="-83" y="510.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-83" y="510.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-78 516.178)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="51.714" y="11" xml:space="preserve">Append C@0x30</tspan>
-        </text>
-      </g>
-      <g id="Graphic_22">
-        <rect x="136.5" y="510.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="136.5" y="510.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_21">
-        <rect x="355.5" y="510.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="355.5" y="510.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-100.448" y1="459.224" x2="626.77344" y2="459.224" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-230 450.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_18">
-        <rect x="-82.75" y="426.748" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-82.75" y="426.748" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-77.75 432.776)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.602" y="11" xml:space="preserve">Append F@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <rect x="136.75" y="426.748" width="203.5" height="26.391998" fill="white"/>
-        <rect x="136.75" y="426.748" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_16">
-        <rect x="355.75" y="426.748" width="203.5" height="26.391998" fill="white"/>
-        <rect x="355.75" y="426.748" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_15">
-        <rect x="-82.75" y="464.645" width="113.75" height="34" fill="white"/>
-        <rect x="-82.75" y="464.645" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-77.75 467.309)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.505" y="11" xml:space="preserve">Append E@0x50</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="6.947" y="25.336" xml:space="preserve">Append D@0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_14">
-        <rect x="41.5" y="464.645" width="203.5" height="34" fill="white"/>
-        <rect x="41.5" y="464.645" width="203.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_13">
-        <rect x="255.5" y="464.645" width="303.5" height="34" fill="white"/>
-        <rect x="255.5" y="464.645" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_12">
-        <rect x="-83" y="548.047" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-83" y="548.047" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-78 554.075)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="26.796" y="11" xml:space="preserve">A@0x10, Append B@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_11">
-        <rect x="136.5" y="548.047" width="203.5" height="26.391998" fill="white"/>
-        <rect x="136.5" y="548.047" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_10">
-        <rect x="355.5" y="548.047" width="203.5" height="26.391998" fill="white"/>
-        <rect x="355.5" y="548.047" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_24">
-        <line x1="-104" y1="542" x2="610.5" y2="542" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <text transform="translate(-139.604 534.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <text transform="translate(-139.604 452.556)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_30">
-        <line x1="-100.448" y1="481.145" x2="614.052" y2="481.145" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <text transform="translate(-139.604 473.449)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Line_48">
-        <line x1="-99.448" y1="701.513" x2="627.77344" y2="701.513" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_47">
-        <text transform="translate(-229 692.789)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_46">
-        <rect x="-81.75" y="670.496" width="113.75" height="26.391998" fill="white"/>
-        <rect x="-81.75" y="670.496" width="113.75" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-76.75 676.524)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.727" y="11" xml:space="preserve">Append F@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_43">
-        <rect x="-81.75" y="708.393" width="113.75" height="34" fill="white"/>
-        <rect x="-81.75" y="708.393" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-76.75 718.225)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.505" y="11" xml:space="preserve">Append E@0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_37">
-        <line x1="-101" y1="777.2665" x2="613.5" y2="777.2665" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_36">
-        <text transform="translate(-138.604 769.7665)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_33">
-        <text transform="translate(-138.604 694.845)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_32">
-        <line x1="-99.448" y1="755.089" x2="615.052" y2="755.089" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_31">
-        <text transform="translate(-138.604 747.393)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_40">
-        <rect x="-82" y="770.909" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-82" y="770.909" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-77 770.7945)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="70.836" y="11" xml:space="preserve">AB@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_39">
-        <rect x="137.5" y="770.909" width="203.5" height="14.107002" fill="white"/>
-        <rect x="137.5" y="770.909" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_38">
-        <rect x="356.5" y="770.909" width="203.5" height="14.107002" fill="white"/>
-        <rect x="356.5" y="770.909" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_54">
-        <rect x="-81.75" y="748.5355" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-81.75" y="748.5355" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-76.75 748.421)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="62.28" y="11" xml:space="preserve">ABCD@0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_53">
-        <rect x="137.75" y="748.5355" width="203.5" height="14.107002" fill="white"/>
-        <rect x="137.75" y="748.5355" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_52">
-        <rect x="356.75" y="748.5355" width="203.5" height="14.107002" fill="white"/>
-        <rect x="356.75" y="748.5355" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_57">
-        <path d="M 211.32422 585 L 265.17578 585 L 265.17578 611.332 L 287.84375 611.332 L 238.25 633.117 L 188.65625 611.332 L 211.32422 611.332 Z" fill="white"/>
-        <path d="M 211.32422 585 L 265.17578 585 L 265.17578 611.332 L 287.84375 611.332 L 238.25 633.117 L 188.65625 611.332 L 211.32422 611.332 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_60">
-        <rect x="359" y="692.858" width="203.5" height="14.107002" fill="white"/>
-        <rect x="359" y="692.858" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_59">
-        <rect x="41.5" y="693.858" width="303" height="14.107002" fill="white"/>
-        <rect x="41.5" y="693.858" width="303" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/06-btmgc-child.svg

@@ -1,184 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-413 471 931 354" width="931" height="354">
-  <defs/>
-  <g id="06-btmgc-child" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>06-btmgc-child</title>
-    <rect fill="white" x="-413" y="471" width="931" height="354"/>
-    <g id="06-btmgc-child_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_47">
-        <rect x="-412" y="594.402" width="928" height="28.447998" fill="white"/>
-        <rect x="-412" y="594.402" width="928" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_46">
-        <rect x="-205" y="555.552" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-205" y="555.552" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-200 561.58)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.158" y="11" xml:space="preserve">Append P@0x30</tspan>
-        </text>
-      </g>
-      <g id="Graphic_45">
-        <rect x="14.5" y="555.552" width="203.5" height="26.391998" fill="white"/>
-        <rect x="14.5" y="555.552" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_44">
-        <rect x="233.5" y="555.552" width="203.5" height="26.391998" fill="white"/>
-        <rect x="233.5" y="555.552" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_43">
-        <line x1="-222.448" y1="504.724" x2="504.77344" y2="504.724" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_42">
-        <text transform="translate(-352 496)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_41">
-        <rect x="-204.75" y="472.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-204.75" y="472.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199.75 478.178)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.158" y="11" xml:space="preserve">Append S@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_40">
-        <rect x="14.75" y="472.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="14.75" y="472.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_39">
-        <rect x="233.75" y="472.15" width="203.5" height="26.391998" fill="white"/>
-        <rect x="233.75" y="472.15" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_38">
-        <rect x="-204.75" y="510.047" width="113.75" height="34" fill="white"/>
-        <rect x="-204.75" y="510.047" width="113.75" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199.75 512.711)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="7.061" y="11" xml:space="preserve">Append R@0x50</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="6.611" y="25.336" xml:space="preserve">Append Q@0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_37">
-        <rect x="-80.5" y="510.047" width="203.5" height="34" fill="white"/>
-        <rect x="-80.5" y="510.047" width="203.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_36">
-        <rect x="133.5" y="510.047" width="303.5" height="34" fill="white"/>
-        <rect x="133.5" y="510.047" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_33">
-        <text transform="translate(-261.604 498.056)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_30">
-        <line x1="-224" y1="607.9115" x2="490.5" y2="607.9115" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <text transform="translate(-261.604 600.4115)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <rect x="-205" y="601.554" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-205" y="601.554" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-200 601.4395)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="70.836" y="11" xml:space="preserve">AB@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_27">
-        <rect x="14.5" y="601.554" width="203.5" height="14.107002" fill="white"/>
-        <rect x="14.5" y="601.554" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <rect x="233.5" y="601.554" width="203.5" height="14.107002" fill="white"/>
-        <rect x="233.5" y="601.554" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <text transform="translate(-407 599.1875)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">Ancestor Branch</tspan>
-        </text>
-      </g>
-      <g id="Graphic_24">
-        <rect x="-411" y="795.46" width="928" height="28.447998" fill="white"/>
-        <rect x="-411" y="795.46" width="928" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-221.448" y1="755.528" x2="505.77344" y2="755.528" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-351 746.804)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_18">
-        <rect x="-203.75" y="723.579" width="203.25" height="26.391998" fill="white"/>
-        <rect x="-203.75" y="723.579" width="203.25" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-198.75 729.607)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" x="52.033" y="11" xml:space="preserve">Append S@0x60</tspan>
-        </text>
-      </g>
-      <g id="Graphic_10">
-        <text transform="translate(-260.604 748.86)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Line_7">
-        <line x1="-223" y1="808.9695" x2="491.5" y2="808.9695" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_6">
-        <text transform="translate(-260.604 801.4695)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_5">
-        <rect x="-204" y="802.612" width="203.5" height="14.107002" fill="white"/>
-        <rect x="-204" y="802.612" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199 802.4975)" fill="#b1001c">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="#b1001c" x="70.836" y="11" xml:space="preserve">AB</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" y="11" xml:space="preserve">@0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_4">
-        <rect x="15.5" y="802.612" width="203.5" height="14.107002" fill="white"/>
-        <rect x="15.5" y="802.612" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_3">
-        <rect x="234.5" y="802.612" width="203.5" height="14.107002" fill="white"/>
-        <rect x="234.5" y="802.612" width="203.5" height="14.107002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_2">
-        <text transform="translate(-406 800.2455)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">Ancestor Branch</tspan>
-        </text>
-      </g>
-      <g id="Graphic_48">
-        <path d="M 89.32422 639.081 L 143.17578 639.081 L 143.17578 665.413 L 165.84375 665.413 L 116.25 687.198 L 66.65625 665.413 L 89.32422 665.413 Z" fill="white"/>
-        <path d="M 89.32422 639.081 L 143.17578 639.081 L 143.17578 665.413 L 165.84375 665.413 L 116.25 687.198 L 66.65625 665.413 L 89.32422 665.413 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_49">
-        <rect x="-204" y="762.428" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-204" y="762.428" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-199 768.456)" fill="#b1001c">
-          <tspan font-family="Helvetica Neue" font-size="12" fill="#b1001c" x="58.278" y="11" xml:space="preserve">AB</tspan>
-          <tspan font-family="Helvetica Neue" font-size="12" fill="black" y="11" xml:space="preserve">PQR@0x50</tspan>
-        </text>
-      </g>
-      <g id="Graphic_59">
-        <rect x="14.5" y="723.579" width="203.5" height="26.391998" fill="white"/>
-        <rect x="14.5" y="723.579" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_58">
-        <rect x="233.5" y="723.579" width="203.5" height="26.391998" fill="white"/>
-        <rect x="233.5" y="723.579" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_63">
-        <rect x="9" y="762.085" width="203.5" height="26.391998" fill="white"/>
-        <rect x="9" y="762.085" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_62">
-        <rect x="225" y="762.085" width="213" height="26.391998" fill="white"/>
-        <rect x="225" y="762.085" width="213" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/07-btmgc-analysis-1.svg

@@ -1,180 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-556 476 923 411" width="923" height="411">
-  <defs/>
-  <g id="07-btmgc-analysis-1" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>07-btmgc-analysis-1</title>
-    <rect fill="white" x="-556" y="476" width="923" height="411"/>
-    <g id="07-btmgc-analysis-1_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_85">
-        <rect x="-404" y="609.062" width="203.5" height="17.5" fill="white"/>
-        <rect x="-404" y="609.062" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_84">
-        <rect x="-184.5" y="609.062" width="203.5" height="17.5" fill="white"/>
-        <rect x="-184.5" y="609.062" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_83">
-        <rect x="34.5" y="609.062" width="203.5" height="17.5" fill="white"/>
-        <rect x="34.5" y="609.062" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_82">
-        <rect x="-404" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-404" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_81">
-        <rect x="-263" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-263" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_80">
-        <rect x="-122" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-122" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_79">
-        <rect x="19" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="19" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_78">
-        <rect x="160" y="479.922" width="78" height="77.5" fill="white"/>
-        <rect x="160" y="479.922" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_77">
-        <line x1="-452.5" y1="518.172" x2="251" y2="518.172" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_76">
-        <text transform="translate(-551 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_75">
-        <text transform="translate(-531.59 602.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.39" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="29132252e-19" y="28.447998" xml:space="preserve">at earlier LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_74">
-        <text transform="translate(-516.08 481)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_73">
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" fill="white"/>
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_72">
-        <rect x="-403.8" y="827.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="-403.8" y="827.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_71">
-        <rect x="-184.3" y="827.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="-184.3" y="827.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_70">
-        <rect x="34.7" y="827.474" width="203.5" height="17.5" fill="white"/>
-        <rect x="34.7" y="827.474" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_69">
-        <rect x="-403.8" y="761" width="127" height="52.974" fill="white"/>
-        <rect x="-403.8" y="761" width="127" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_68">
-        <rect x="-262.8" y="761" width="310.5" height="52.974" fill="white"/>
-        <rect x="-262.8" y="761" width="310.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_67">
-        <rect x="64.7" y="761" width="173.5" height="52.974" fill="white"/>
-        <rect x="64.7" y="761" width="173.5" height="52.974" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_66">
-        <line x1="-452.3" y1="835.724" x2="251.2" y2="835.724" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_65">
-        <text transform="translate(-515.88 766)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8739676e-19" y="15" xml:space="preserve">Deltas</tspan>
-        </text>
-      </g>
-      <g id="Graphic_64">
-        <text transform="translate(-523 820.86)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="3488765e-18" y="15" xml:space="preserve">Images </tspan>
-          <tspan font-family="Helvetica Neue" font-size="10" fill="black" x="4.01" y="28.447998" xml:space="preserve">at GC LSN</tspan>
-        </text>
-      </g>
-      <g id="Graphic_63">
-        <rect x="-394.93164" y="491.422" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(-389.93164 492.49)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_62">
-        <rect x="-394.93164" y="526.922" width="624.3633" height="17.5" fill="#c0ffff"/>
-        <text transform="translate(-389.93164 527.99)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.89414" y="12" xml:space="preserve">Deltas below GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_61">
-        <rect x="-394.73164" y="778.737" width="624.3633" height="17.5" fill="#c0ffc0"/>
-        <text transform="translate(-389.73164 779.805)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="233.52364" y="12" xml:space="preserve">Deltas above GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_60">
-        <rect x="-333.8" y="858.474" width="485.5" height="28.447998" fill="#c0ffff"/>
-        <text transform="translate(-328.8 865.016)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="13" fill="black" x="63.095" y="12" xml:space="preserve">Deltas and image below GC Horizon gets garbage-collected</tspan>
-        </text>
-      </g>
-      <g id="Graphic_86">
-        <text transform="translate(263 499.724)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="32" fill="black" x="0" y="30" xml:space="preserve">size=A</tspan>
-        </text>
-      </g>
-      <g id="Line_87">
-        <line x1="260.87012" y1="479.068" x2="360.71387" y2="479.068" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_88">
-        <line x1="260.87012" y1="561" x2="360.71387" y2="561" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_89">
-        <rect x="-403.8" y="569" width="161.8" height="28.447998" fill="white"/>
-        <rect x="-403.8" y="569" width="161.8" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_90">
-        <rect x="-229.5" y="569.018" width="277.2" height="28.447998" fill="white"/>
-        <rect x="-229.5" y="569.018" width="277.2" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_91">
-        <rect x="64.7" y="569.018" width="173.5" height="28.447998" fill="white"/>
-        <rect x="64.7" y="569.018" width="173.5" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_92">
-        <line x1="262" y1="602" x2="361.84375" y2="602" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_93">
-        <line x1="263" y1="625.562" x2="362.84375" y2="625.562" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_94">
-        <text transform="translate(264.53787 562.276)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="32" fill="black" x="14210855e-21" y="30" xml:space="preserve">size=B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_95">
-        <text transform="translate(285.12 599.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="24" fill="black" x="0" y="23" xml:space="preserve">size=C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_98">
-        <text transform="translate(264.53787 773.772)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="8881784e-19" y="25" xml:space="preserve">A</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-      <g id="Graphic_97">
-        <text transform="translate(265.87013 815.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="6536993e-19" y="25" xml:space="preserve">B</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/08-optimization.svg

@@ -1,158 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-235 406 586 424" width="586" height="424">
-  <defs/>
-  <g id="08-optimization" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>08-optimization</title>
-    <rect fill="white" x="-235" y="406" width="586" height="424"/>
-    <g id="08-optimization_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_22">
-        <rect x="-100.448" y="509.902" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="509.902" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_21">
-        <rect x="118.552" y="509.902" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="509.902" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_20">
-        <line x1="-101.79572" y1="420.322" x2="349.5" y2="420.322" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_19">
-        <text transform="translate(-230 411.598)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <rect x="-100.198" y="426.5" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.198" y="426.5" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_16">
-        <rect x="118.802" y="426.5" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.802" y="426.5" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_14">
-        <rect x="-100.198" y="464.397" width="108.25" height="34" fill="white"/>
-        <rect x="-100.198" y="464.397" width="108.25" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_13">
-        <rect x="18.552" y="464.397" width="303.5" height="34" fill="white"/>
-        <rect x="18.552" y="464.397" width="303.5" height="34" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_11">
-        <rect x="-100.448" y="547.799" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="547.799" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_10">
-        <rect x="118.552" y="547.799" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="547.799" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_24">
-        <line x1="-104" y1="542" x2="339.4011" y2="542" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_25">
-        <text transform="translate(-139.604 534.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Line_27">
-        <line x1="-101.79572" y1="459.098" x2="341.6054" y2="459.098" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <text transform="translate(-139.604 451.402)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-      <g id="Graphic_28">
-        <text transform="translate(-139.604 413.654)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x60</tspan>
-        </text>
-      </g>
-      <g id="Line_30">
-        <line x1="-101.79572" y1="481.145" x2="341.6054" y2="481.145" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_29">
-        <text transform="translate(-139.604 473.449)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_77">
-        <rect x="-100.448" y="765.19595" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="765.19595" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_76">
-        <rect x="118.552" y="765.19595" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="765.19595" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_75">
-        <line x1="-101.79572" y1="637.317" x2="349.5" y2="637.317" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_74">
-        <text transform="translate(-230 628.593)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_73">
-        <rect x="-100.198" y="681.794" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.198" y="681.794" width="203.5" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_72">
-        <rect x="118.802" y="681.794" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.802" y="681.794" width="203.5" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_71">
-        <rect x="-100.198" y="719.69096" width="108.25" height="34" fill="white"/>
-        <rect x="-100.198" y="719.69096" width="108.25" height="34" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_70">
-        <rect x="18.552" y="719.69096" width="303.5" height="34" fill="white"/>
-        <rect x="18.552" y="719.69096" width="303.5" height="34" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_69">
-        <rect x="-100.448" y="803.09295" width="203.5" height="26.391998" fill="white"/>
-        <rect x="-100.448" y="803.09295" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_68">
-        <rect x="118.552" y="803.09295" width="203.5" height="26.391998" fill="white"/>
-        <rect x="118.552" y="803.09295" width="203.5" height="26.391998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_67">
-        <line x1="-104" y1="797.294" x2="339.4011" y2="797.294" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_66">
-        <text transform="translate(-139.604 789.794)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x20</tspan>
-        </text>
-      </g>
-      <g id="Graphic_63">
-        <text transform="translate(-139.604 630.649)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x70</tspan>
-        </text>
-      </g>
-      <g id="Line_62">
-        <line x1="-101.79572" y1="736.439" x2="341.6054" y2="736.439" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="4.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_61">
-        <text transform="translate(-139.604 728.743)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="black" x="0" y="13" xml:space="preserve">0x40</tspan>
-        </text>
-      </g>
-      <g id="Graphic_79">
-        <rect x="-100.198" y="644.393" width="168.198" height="26.391998" fill="white"/>
-        <rect x="-100.198" y="644.393" width="168.198" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_78">
-        <rect x="80" y="644.393" width="242.302" height="26.391998" fill="white"/>
-        <rect x="80" y="644.393" width="242.302" height="26.391998" stroke="#b1001c" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_81">
-        <line x1="-101.79572" y1="714.139" x2="341.6054" y2="714.139" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="1.0,4.0" stroke-width="1"/>
-      </g>
-      <g id="Graphic_80">
-        <text transform="translate(-139.604 706.443)" fill="#a5a5a5">
-          <tspan font-family="Helvetica Neue" font-size="14" fill="#a5a5a5" x="0" y="13" xml:space="preserve">0x50</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/09-btmgc-analysis-2.svg

@@ -1,184 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-562 479 876 429" width="876" height="429">
-  <defs/>
-  <g id="09-btmgc-analysis-2" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>09-btmgc-analysis-2</title>
-    <rect fill="white" x="-562" y="479" width="876" height="429"/>
-    <g id="09-btmgc-analysis-2_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_85">
-        <rect x="-404" y="622.386" width="203.5" height="17.5" fill="white"/>
-        <rect x="-404" y="622.386" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-399 621.912)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_84">
-        <rect x="-184.5" y="622.386" width="203.5" height="17.5" fill="white"/>
-        <rect x="-184.5" y="622.386" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-179.5 621.912)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_83">
-        <rect x="34.5" y="622.386" width="203.5" height="17.5" fill="white"/>
-        <rect x="34.5" y="622.386" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(39.5 621.912)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_82">
-        <rect x="-404" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-404" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-399 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_81">
-        <rect x="-263" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-263" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-258 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_80">
-        <rect x="-122" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="-122" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-117 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_79">
-        <rect x="19" y="479.922" width="127" height="77.5" fill="white"/>
-        <rect x="19" y="479.922" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(24 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_78">
-        <rect x="160" y="479.922" width="78" height="77.5" fill="white"/>
-        <rect x="160" y="479.922" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(165 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="28.816" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Line_77">
-        <line x1="-452.5" y1="518.172" x2="251" y2="518.172" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_76">
-        <text transform="translate(-551 509.448)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_73">
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" fill="white"/>
-        <path d="M -120.675 651.5 L -84.425 651.5 L -84.425 710 L -66.3 710 L -102.55 740 L -138.8 710 L -120.675 710 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_89">
-        <rect x="-403.8" y="582.324" width="161.8" height="28.447998" fill="white"/>
-        <rect x="-403.8" y="582.324" width="161.8" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-398.8 587.324)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="70.42" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_90">
-        <rect x="-229.5" y="582.342" width="277.2" height="28.447998" fill="white"/>
-        <rect x="-229.5" y="582.342" width="277.2" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-224.5 587.342)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="128.12" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_91">
-        <rect x="64.7" y="582.342" width="173.5" height="28.447998" fill="white"/>
-        <rect x="64.7" y="582.342" width="173.5" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(69.7 587.342)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="76.27" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_97">
-        <rect x="-403.8" y="564.842" width="490.8" height="12.157997" fill="white"/>
-        <rect x="-403.8" y="564.842" width="490.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-398.8 561.697)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="234.624" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_109">
-        <rect x="28.6" y="889.964" width="203.5" height="17.5" fill="white"/>
-        <rect x="28.6" y="889.964" width="203.5" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(33.6 889.49)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="90.974" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_108">
-        <rect x="-409.9" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="-409.9" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-404.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_107">
-        <rect x="-268.9" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="-268.9" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-263.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_106">
-        <rect x="-127.9" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="-127.9" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-122.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_105">
-        <rect x="13.1" y="747.5" width="127" height="77.5" fill="white"/>
-        <rect x="13.1" y="747.5" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(18.1 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="53.316" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Graphic_104">
-        <rect x="154.1" y="747.5" width="78" height="77.5" fill="white"/>
-        <rect x="154.1" y="747.5" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(159.1 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="28.816" y="15" xml:space="preserve">A</tspan>
-        </text>
-      </g>
-      <g id="Line_103">
-        <line x1="-458.4" y1="785.75" x2="245.1" y2="785.75" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_102">
-        <text transform="translate(-556.9 777.026)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_99">
-        <rect x="58.8" y="849.92" width="173.5" height="28.447998" fill="white"/>
-        <rect x="58.8" y="849.92" width="173.5" height="28.447998" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(63.8 854.92)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="76.27" y="15" xml:space="preserve">B</tspan>
-        </text>
-      </g>
-      <g id="Graphic_98">
-        <rect x="-409.7" y="832.42" width="490.8" height="12.157997" fill="white"/>
-        <rect x="-409.7" y="832.42" width="490.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(-404.7 829.275)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="234.624" y="15" xml:space="preserve">C</tspan>
-        </text>
-      </g>
-      <g id="Graphic_112">
-        <text transform="translate(273 797.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="6536993e-19" y="25" xml:space="preserve">B</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-      <g id="Graphic_113">
-        <text transform="translate(273 833.974)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="26" fill="black" x="42277293e-20" y="25" xml:space="preserve">C</tspan>
-          <tspan font-family="Lucida Grande" font-size="26" fill="black" y="25" xml:space="preserve">↓</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/10-btmgc-analysis-3.svg

@@ -1,81 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-12 920 809 269" width="809" height="269">
-  <defs/>
-  <g id="10-btmgc-analysis-3" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>10-btmgc-analysis-3</title>
-    <rect fill="white" x="-12" y="920" width="809" height="269"/>
-    <g id="10-btmgc-analysis-3_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_13">
-        <rect x="433.7" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="433.7" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(438.7 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_12">
-        <rect x="503.7654" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="503.7654" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(508.7654 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_11">
-        <rect x="574.8318" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="574.8318" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(579.8318 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_10">
-        <rect x="645.3977" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="645.3977" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(650.3977 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-      <g id="Line_8">
-        <line x1="92" y1="934.276" x2="795.5" y2="934.276" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <text transform="translate(-6.500003 925.552)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_2">
-        <rect x="113.2" y="1033.92" width="321.3" height="12.157997" fill="white"/>
-        <rect x="113.2" y="1033.92" width="321.3" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(118.2 1030.775)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="150.762" y="15" xml:space="preserve">X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" fill="white"/>
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_27">
-        <line x1="93" y1="1164.224" x2="796.5" y2="1164.224" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <text transform="translate(-5.5000034 1155.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_25">
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" fill="white"/>
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(119 1170.355)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="306.564" y="15" xml:space="preserve">2X</tspan>
-        </text>
-      </g>
-      <g id="Graphic_33">
-        <rect x="715.96355" y="949" width="63.559346" height="77.5" fill="white"/>
-        <rect x="715.96355" y="949" width="63.559346" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(720.96355 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="8.107673" y="15" xml:space="preserve">1/5 X</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/11-btmgc-analysis-4.svg

@@ -1,81 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" xmlns:dc="http://purl.org/dc/elements/1.1/" version="1.1" viewBox="-12 920 809 269" width="809" height="269">
-  <defs/>
-  <g id="11-btmgc-analysis-4" stroke-opacity="1" stroke-dasharray="none" stroke="none" fill="none" fill-opacity="1">
-    <title>11-btmgc-analysis-4</title>
-    <rect fill="white" x="-12" y="920" width="809" height="269"/>
-    <g id="11-btmgc-analysis-4_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_13">
-        <rect x="113" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="113" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(118 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_12">
-        <rect x="253" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="253" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(258 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_11">
-        <rect x="395" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="395" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(400 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_10">
-        <rect x="536" y="949" width="127" height="77.5" fill="white"/>
-        <rect x="536" y="949" width="127" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(541 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="39.084" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_9">
-        <rect x="677" y="949" width="78" height="77.5" fill="white"/>
-        <rect x="677" y="949" width="78" height="77.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(682 978.526)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="14.584" y="15" xml:space="preserve">1/5 D</tspan>
-        </text>
-      </g>
-      <g id="Line_8">
-        <line x1="92" y1="934.276" x2="795.5" y2="934.276" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <text transform="translate(-6.500003 925.552)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_2">
-        <rect x="113.2" y="1033.92" width="641.8" height="12.157997" fill="white"/>
-        <rect x="113.2" y="1033.92" width="641.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(118.2 1030.775)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="310.268" y="15" xml:space="preserve">D</tspan>
-        </text>
-      </g>
-      <g id="Graphic_17">
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" fill="white"/>
-        <path d="M 420.125 1062 L 456.375 1062 L 456.375 1120.5 L 474.5 1120.5 L 438.25 1150.5 L 402 1120.5 L 420.125 1120.5 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_27">
-        <line x1="93" y1="1164.224" x2="796.5" y2="1164.224" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_26">
-        <text transform="translate(-5.5000034 1155.5)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">GC Horizon</tspan>
-        </text>
-      </g>
-      <g id="Graphic_25">
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" fill="white"/>
-        <rect x="114" y="1173.5" width="641.8" height="12.157997" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(119 1170.355)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="310.268" y="15" xml:space="preserve">D</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

docs/rfcs/images/036-bottom-most-gc-compaction/13-job-split.svg

@@ -1,176 +0,0 @@
-<?xml version="1.0" encoding="UTF-8" standalone="no"?>
-<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg version="1.1" xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" viewBox="210 271 870 514" width="870" height="514">
-  <defs/>
-  <g id="gc-compaction-split" stroke-dasharray="none" fill-opacity="1" stroke="none" fill="none" stroke-opacity="1">
-    <title>gc-compaction-split</title>
-    <rect fill="white" x="210" y="271" width="870" height="514"/>
-    <g id="gc-compaction-split_Layer_1">
-      <title>Layer 1</title>
-      <g id="Graphic_12">
-        <rect x="241" y="272" width="213" height="50.5" fill="white"/>
-        <rect x="241" y="272" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_11">
-        <rect x="468.72266" y="272" width="213" height="50.5" fill="white"/>
-        <rect x="468.72266" y="272" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_10">
-        <rect x="695.72266" y="272" width="213" height="50.5" fill="white"/>
-        <rect x="695.72266" y="272" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_9">
-        <rect x="241" y="337.3711" width="303.5" height="50.5" fill="white"/>
-        <rect x="241" y="337.3711" width="303.5" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_8">
-        <rect x="556.2617" y="337.3711" width="352.46094" height="50.5" fill="white"/>
-        <rect x="556.2617" y="337.3711" width="352.46094" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_7">
-        <rect x="241" y="402.7422" width="667.72266" height="50.5" fill="white"/>
-        <rect x="241" y="402.7422" width="667.72266" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_6">
-        <line x1="211" y1="355.5" x2="947.4961" y2="355.5" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_5">
-        <text transform="translate(952.4961 346.776)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">branch point</tspan>
-        </text>
-      </g>
-      <g id="Line_4">
-        <line x1="212" y1="438.5182" x2="948.4961" y2="438.5182" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_3">
-        <text transform="translate(953.4961 429.7942)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">last branch point</tspan>
-        </text>
-      </g>
-      <g id="Graphic_13">
-        <rect x="241" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(246 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 1</tspan>
-        </text>
-      </g>
-      <g id="Graphic_57">
-        <rect x="359" y="647.96484" width="551.72266" height="50.5" fill="white"/>
-        <rect x="359" y="647.96484" width="551.72266" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_54">
-        <rect x="359" y="517.22266" width="96" height="50.5" fill="white"/>
-        <rect x="359" y="517.22266" width="96" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_53">
-        <rect x="469.72266" y="517.22266" width="213" height="50.5" fill="white"/>
-        <rect x="469.72266" y="517.22266" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_52">
-        <rect x="696.72266" y="517.22266" width="213" height="50.5" fill="white"/>
-        <rect x="696.72266" y="517.22266" width="213" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_51">
-        <rect x="359" y="582.59375" width="186.5" height="50.5" fill="white"/>
-        <rect x="359" y="582.59375" width="186.5" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_50">
-        <rect x="557.2617" y="582.59375" width="352.46094" height="50.5" fill="white"/>
-        <rect x="557.2617" y="582.59375" width="352.46094" height="50.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Line_49">
-        <line x1="212" y1="600.72266" x2="948.4961" y2="600.72266" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_48">
-        <text transform="translate(953.4961 591.99866)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">branch point</tspan>
-        </text>
-      </g>
-      <g id="Line_47">
-        <line x1="213" y1="683.74084" x2="949.4961" y2="683.74084" stroke="#7f8080" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_46">
-        <text transform="translate(954.4961 675.01685)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="0" y="15" xml:space="preserve">last branch point</tspan>
-        </text>
-      </g>
-      <g id="Graphic_63">
-        <rect x="376.72525" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(381.72525 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 2</tspan>
-        </text>
-      </g>
-      <g id="Graphic_64">
-        <rect x="511.39405" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(516.39405 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 3</tspan>
-        </text>
-      </g>
-      <g id="Graphic_65">
-        <rect x="646.06285" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(651.06285 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 4</tspan>
-        </text>
-      </g>
-      <g id="Graphic_66">
-        <rect x="780.73165" y="272" width="127.99101" height="181.24219" fill="#3a8eed" fill-opacity=".5"/>
-        <text transform="translate(785.73165 353.3971)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="38.835502" y="15" xml:space="preserve">Job 5</tspan>
-        </text>
-      </g>
-      <g id="Graphic_56">
-        <rect x="243.5" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="243.5" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_55">
-        <rect x="243.5" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="243.5" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_68">
-        <rect x="379.22525" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="379.22525" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_67">
-        <rect x="379.22525" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="379.22525" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_70">
-        <rect x="514.22525" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="514.22525" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_69">
-        <rect x="514.22525" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="514.22525" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_72">
-        <rect x="649.22525" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="649.22525" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_71">
-        <rect x="649.22525" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="649.22525" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_74">
-        <rect x="785.23165" y="517.22266" width="125.49101" height="181.24219" fill="#ccc"/>
-        <rect x="785.23165" y="517.22266" width="125.49101" height="181.24219" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_73">
-        <rect x="785.23165" y="673.46484" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="785.23165" y="673.46484" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-      </g>
-      <g id="Graphic_78">
-        <rect x="241" y="731.3359" width="125.49101" height="27.26953" fill="#ccc"/>
-        <rect x="241" y="731.3359" width="125.49101" height="27.26953" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(246 735.7467)" fill="black">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="black" x="17.297502" y="15" xml:space="preserve">Delta Layer</tspan>
-        </text>
-      </g>
-      <g id="Graphic_79">
-        <rect x="241" y="766.759" width="125.49101" height="17.5" fill="#6b7ca5"/>
-        <rect x="241" y="766.759" width="125.49101" height="17.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="1"/>
-        <text transform="translate(246 766.285)" fill="white">
-          <tspan font-family="Helvetica Neue" font-size="16" fill="white" x="13.737502" y="15" xml:space="preserve">Image Layer</tspan>
-        </text>
-      </g>
-    </g>
-  </g>
-</svg>

libs/pageserver_api/src/config.rs

@@ -305,7 +305,6 @@ impl From<OtelExporterProtocol> for tracing_utils::Protocol {
 pub struct TimelineImportConfig {
     pub import_job_concurrency: NonZeroUsize,
     pub import_job_soft_size_limit: NonZeroUsize,
-    pub import_job_checkpoint_threshold: NonZeroUsize,
 }
 
 pub mod statvfs {
@@ -640,15 +639,23 @@ impl Default for ConfigToml {
             tenant_config: TenantConfigToml::default(),
             no_sync: None,
             wal_receiver_protocol: DEFAULT_WAL_RECEIVER_PROTOCOL,
-            page_service_pipelining: PageServicePipeliningConfig::Pipelined(
-                PageServicePipeliningConfigPipelined {
+            page_service_pipelining: if !cfg!(test) {
+                PageServicePipeliningConfig::Serial
+            } else {
+                // Do not turn this into the default until scattered reads have been
+                // validated and rolled-out fully.
+                PageServicePipeliningConfig::Pipelined(PageServicePipeliningConfigPipelined {
                     max_batch_size: NonZeroUsize::new(32).unwrap(),
                     execution: PageServiceProtocolPipelinedExecutionStrategy::ConcurrentFutures,
                     batching: PageServiceProtocolPipelinedBatchingStrategy::ScatteredLsn,
-                },
-            ),
-            get_vectored_concurrent_io: GetVectoredConcurrentIo::SidecarTask,
-            enable_read_path_debugging: if cfg!(feature = "testing") {
+                })
+            },
+            get_vectored_concurrent_io: if !cfg!(test) {
+                GetVectoredConcurrentIo::Sequential
+            } else {
+                GetVectoredConcurrentIo::SidecarTask
+            },
+            enable_read_path_debugging: if cfg!(test) || cfg!(feature = "testing") {
                 Some(true)
             } else {
                 None
@@ -662,7 +669,6 @@ impl Default for ConfigToml {
             timeline_import_config: TimelineImportConfig {
                 import_job_concurrency: NonZeroUsize::new(128).unwrap(),
                 import_job_soft_size_limit: NonZeroUsize::new(1024 * 1024 * 1024).unwrap(),
-                import_job_checkpoint_threshold: NonZeroUsize::new(128).unwrap(),
             },
         }
     }

libs/pageserver_api/src/key.rs

@@ -910,11 +910,6 @@ impl Key {
         self.field1 == 0x00 && self.field4 != 0 && self.field6 != 0xffffffff
     }
 
-    #[inline(always)]
-    pub fn is_rel_block_of_rel(&self, rel: Oid) -> bool {
-        self.is_rel_block_key() && self.field4 == rel
-    }
-
     #[inline(always)]
     pub fn is_rel_dir_key(&self) -> bool {
         self.field1 == 0x00

libs/pageserver_api/src/models.rs

@@ -336,30 +336,14 @@ impl TimelineCreateRequest {
 
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
 pub enum ShardImportStatus {
-    InProgress(Option<ShardImportProgress>),
+    InProgress,
     Done,
     Error(String),
 }
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub enum ShardImportProgress {
-    V1(ShardImportProgressV1),
-}
-
-#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
-pub struct ShardImportProgressV1 {
-    /// Total number of jobs in the import plan
-    pub jobs: usize,
-    /// Number of jobs completed
-    pub completed: usize,
-    /// Hash of the plan
-    pub import_plan_hash: u64,
-}
-
 impl ShardImportStatus {
     pub fn is_terminal(&self) -> bool {
         match self {
-            ShardImportStatus::InProgress(_) => false,
+            ShardImportStatus::InProgress => false,
             ShardImportStatus::Done | ShardImportStatus::Error(_) => true,
         }
     }
@@ -1819,6 +1803,7 @@ pub struct TopTenantShardsResponse {
 }
 
 pub mod virtual_file {
+    use std::sync::LazyLock;
 
     #[derive(
         Copy,
@@ -1866,7 +1851,15 @@ pub mod virtual_file {
 
     impl IoMode {
         pub fn preferred() -> Self {
-            IoMode::DirectRw
+            // The default behavior when running Rust unit tests without any further
+            // flags is to use the newest behavior (DirectRw).
+            // The CI uses the environment variable to unit tests for all different modes.
+            // NB: the Python regression & perf tests have their own defaults management
+            // that writes pageserver.toml; they do not use this variable.
+            static ENV_OVERRIDE: LazyLock<Option<IoMode>> = LazyLock::new(|| {
+                utils::env::var_serde_json_string("NEON_PAGESERVER_UNIT_TEST_VIRTUAL_FILE_IO_MODE")
+            });
+            ENV_OVERRIDE.unwrap_or(IoMode::DirectRw)
         }
     }
 

libs/pageserver_api/src/upcall_api.rs

@@ -4,7 +4,6 @@
 //! See docs/rfcs/025-generation-numbers.md
 
 use serde::{Deserialize, Serialize};
-use utils::generation::Generation;
 use utils::id::{NodeId, TimelineId};
 
 use crate::controller_api::NodeRegisterRequest;
@@ -64,17 +63,9 @@ pub struct ValidateResponseTenant {
     pub valid: bool,
 }
 
-#[derive(Serialize, Deserialize)]
-pub struct TimelineImportStatusRequest {
-    pub tenant_shard_id: TenantShardId,
-    pub timeline_id: TimelineId,
-    pub generation: Generation,
-}
-
 #[derive(Serialize, Deserialize)]
 pub struct PutTimelineImportStatusRequest {
     pub tenant_shard_id: TenantShardId,
     pub timeline_id: TimelineId,
     pub status: ShardImportStatus,
-    pub generation: Generation,
 }

libs/pageserver_api/src/value.rs

@@ -36,24 +36,6 @@ impl Value {
             Value::WalRecord(rec) => rec.will_init(),
         }
     }
-
-    #[inline(always)]
-    pub fn estimated_size(&self) -> usize {
-        match self {
-            Value::Image(image) => image.len(),
-            Value::WalRecord(NeonWalRecord::AuxFile {
-                content: Some(content),
-                ..
-            }) => content.len(),
-            Value::WalRecord(NeonWalRecord::Postgres { rec, .. }) => rec.len(),
-            Value::WalRecord(NeonWalRecord::ClogSetAborted { xids }) => xids.len() * 4,
-            Value::WalRecord(NeonWalRecord::ClogSetCommitted { xids, .. }) => xids.len() * 4,
-            Value::WalRecord(NeonWalRecord::MultixactMembersCreate { members, .. }) => {
-                members.len() * 8
-            }
-            _ => 8192, /* use image size as the estimation */
-        }
-    }
 }
 
 #[derive(Debug, PartialEq)]

libs/posthog_client_lite/Cargo.toml

@@ -1,14 +0,0 @@
-[package]
-name = "posthog_client_lite"
-version = "0.1.0"
-edition = "2024"
-license.workspace = true
-
-[dependencies]
-anyhow.workspace = true
-reqwest.workspace = true
-serde.workspace = true
-serde_json.workspace = true
-sha2.workspace = true
-workspace_hack.workspace = true
-thiserror.workspace = true

libs/posthog_client_lite/src/lib.rs

@@ -1,634 +0,0 @@
-//! A lite version of the PostHog client that only supports local evaluation of feature flags.
-
-use std::collections::HashMap;
-
-use serde::{Deserialize, Serialize};
-use serde_json::json;
-use sha2::Digest;
-
-#[derive(Debug, thiserror::Error)]
-pub enum PostHogEvaluationError {
-    /// The feature flag is not available, for example, because the local evaluation data is not populated yet.
-    #[error("Feature flag not available: {0}")]
-    NotAvailable(String),
-    #[error("No condition group is matched")]
-    NoConditionGroupMatched,
-    /// Real errors, e.g., the rollout percentage does not add up to 100.
-    #[error("Failed to evaluate feature flag: {0}")]
-    Internal(String),
-}
-
-#[derive(Deserialize)]
-pub struct LocalEvaluationResponse {
-    #[allow(dead_code)]
-    flags: Vec<LocalEvaluationFlag>,
-}
-
-#[derive(Deserialize)]
-pub struct LocalEvaluationFlag {
-    key: String,
-    filters: LocalEvaluationFlagFilters,
-    active: bool,
-}
-
-#[derive(Deserialize)]
-pub struct LocalEvaluationFlagFilters {
-    groups: Vec<LocalEvaluationFlagFilterGroup>,
-    multivariate: LocalEvaluationFlagMultivariate,
-}
-
-#[derive(Deserialize)]
-pub struct LocalEvaluationFlagFilterGroup {
-    variant: Option<String>,
-    properties: Option<Vec<LocalEvaluationFlagFilterProperty>>,
-    rollout_percentage: i64,
-}
-
-#[derive(Deserialize)]
-pub struct LocalEvaluationFlagFilterProperty {
-    key: String,
-    value: PostHogFlagFilterPropertyValue,
-    operator: String,
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(untagged)]
-pub enum PostHogFlagFilterPropertyValue {
-    String(String),
-    Number(f64),
-    Boolean(bool),
-    List(Vec<String>),
-}
-
-#[derive(Deserialize)]
-pub struct LocalEvaluationFlagMultivariate {
-    variants: Vec<LocalEvaluationFlagMultivariateVariant>,
-}
-
-#[derive(Deserialize)]
-pub struct LocalEvaluationFlagMultivariateVariant {
-    key: String,
-    rollout_percentage: i64,
-}
-
-pub struct FeatureStore {
-    flags: HashMap<String, LocalEvaluationFlag>,
-}
-
-impl Default for FeatureStore {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-enum GroupEvaluationResult {
-    MatchedAndOverride(String),
-    MatchedAndEvaluate,
-    Unmatched,
-}
-
-impl FeatureStore {
-    pub fn new() -> Self {
-        Self {
-            flags: HashMap::new(),
-        }
-    }
-
-    pub fn set_flags(&mut self, flags: Vec<LocalEvaluationFlag>) {
-        self.flags.clear();
-        for flag in flags {
-            self.flags.insert(flag.key.clone(), flag);
-        }
-    }
-
-    /// Generate a consistent hash for a user ID (e.g., tenant ID).
-    ///
-    /// The implementation is different from PostHog SDK. In PostHog SDK, it is sha1 of `user_id.distinct_id.salt`.
-    /// However, as we do not upload all of our tenant IDs to PostHog, we do not have the PostHog distinct_id for a
-    /// tenant. Therefore, the way we compute it is sha256 of `user_id.feature_id.salt`.
-    fn consistent_hash(user_id: &str, flag_key: &str, salt: &str) -> f64 {
-        let mut hasher = sha2::Sha256::new();
-        hasher.update(user_id);
-        hasher.update(".");
-        hasher.update(flag_key);
-        hasher.update(".");
-        hasher.update(salt);
-        let hash = hasher.finalize();
-        let hash_int = u64::from_le_bytes(hash[..8].try_into().unwrap());
-        hash_int as f64 / u64::MAX as f64
-    }
-
-    /// Evaluate a condition. Returns an error if the condition cannot be evaluated due to parsing error or missing
-    /// property.
-    fn evaluate_condition(
-        &self,
-        operator: &str,
-        provided: &PostHogFlagFilterPropertyValue,
-        requested: &PostHogFlagFilterPropertyValue,
-    ) -> Result<bool, PostHogEvaluationError> {
-        match operator {
-            "exact" => {
-                let PostHogFlagFilterPropertyValue::String(provided) = provided else {
-                    // Left should be a string
-                    return Err(PostHogEvaluationError::Internal(format!(
-                        "The left side of the condition is not a string: {:?}",
-                        provided
-                    )));
-                };
-                let PostHogFlagFilterPropertyValue::List(requested) = requested else {
-                    // Right should be a list of string
-                    return Err(PostHogEvaluationError::Internal(format!(
-                        "The right side of the condition is not a list: {:?}",
-                        requested
-                    )));
-                };
-                Ok(requested.contains(provided))
-            }
-            "lt" | "gt" => {
-                let PostHogFlagFilterPropertyValue::String(requested) = requested else {
-                    // Right should be a string
-                    return Err(PostHogEvaluationError::Internal(format!(
-                        "The right side of the condition is not a string: {:?}",
-                        requested
-                    )));
-                };
-                let Ok(requested) = requested.parse::<f64>() else {
-                    return Err(PostHogEvaluationError::Internal(format!(
-                        "Can not parse the right side of the condition as a number: {:?}",
-                        requested
-                    )));
-                };
-                // Left can either be a number or a string
-                let provided = match provided {
-                    PostHogFlagFilterPropertyValue::Number(provided) => *provided,
-                    PostHogFlagFilterPropertyValue::String(provided) => {
-                        let Ok(provided) = provided.parse::<f64>() else {
-                            return Err(PostHogEvaluationError::Internal(format!(
-                                "Can not parse the left side of the condition as a number: {:?}",
-                                provided
-                            )));
-                        };
-                        provided
-                    }
-                    _ => {
-                        return Err(PostHogEvaluationError::Internal(format!(
-                            "The left side of the condition is not a number or a string: {:?}",
-                            provided
-                        )));
-                    }
-                };
-                match operator {
-                    "lt" => Ok(provided < requested),
-                    "gt" => Ok(provided > requested),
-                    op => Err(PostHogEvaluationError::Internal(format!(
-                        "Unsupported operator: {}",
-                        op
-                    ))),
-                }
-            }
-            _ => Err(PostHogEvaluationError::Internal(format!(
-                "Unsupported operator: {}",
-                operator
-            ))),
-        }
-    }
-
-    /// Evaluate a percentage.
-    fn evaluate_percentage(&self, mapped_user_id: f64, percentage: i64) -> bool {
-        mapped_user_id <= percentage as f64 / 100.0
-    }
-
-    /// Evaluate a filter group for a feature flag. Returns an error if there are errors during the evaluation.
-    ///
-    /// Return values:
-    /// Ok(GroupEvaluationResult::MatchedAndOverride(variant)): matched and evaluated to this value
-    /// Ok(GroupEvaluationResult::MatchedAndEvaluate): condition matched but no variant override, use the global rollout percentage
-    /// Ok(GroupEvaluationResult::Unmatched): condition unmatched
-    fn evaluate_group(
-        &self,
-        group: &LocalEvaluationFlagFilterGroup,
-        hash_on_group_rollout_percentage: f64,
-        provided_properties: &HashMap<String, PostHogFlagFilterPropertyValue>,
-    ) -> Result<GroupEvaluationResult, PostHogEvaluationError> {
-        if let Some(ref properties) = group.properties {
-            for property in properties {
-                if let Some(value) = provided_properties.get(&property.key) {
-                    // The user provided the property value
-                    if !self.evaluate_condition(
-                        property.operator.as_ref(),
-                        value,
-                        &property.value,
-                    )? {
-                        return Ok(GroupEvaluationResult::Unmatched);
-                    }
-                } else {
-                    // We cannot evaluate, the property is not available
-                    return Err(PostHogEvaluationError::NotAvailable(format!(
-                        "The required property in the condition is not available: {}",
-                        property.key
-                    )));
-                }
-            }
-        }
-
-        // The group has no condition matchers or we matched the properties
-        if self.evaluate_percentage(hash_on_group_rollout_percentage, group.rollout_percentage) {
-            if let Some(ref variant_override) = group.variant {
-                Ok(GroupEvaluationResult::MatchedAndOverride(
-                    variant_override.clone(),
-                ))
-            } else {
-                Ok(GroupEvaluationResult::MatchedAndEvaluate)
-            }
-        } else {
-            Ok(GroupEvaluationResult::Unmatched)
-        }
-    }
-
-    /// Evaluate a multivariate feature flag. Returns `None` if the flag is not available or if there are errors
-    /// during the evaluation.
-    ///
-    /// The parsing logic is as follows:
-    ///
-    /// * Match each filter group.
-    ///   - If a group is matched, it will first determine whether the user is in the range of the group's rollout
-    ///     percentage. We will generate a consistent hash for the user ID on the group rollout percentage. This hash
-    ///     is shared across all groups.
-    ///   - If the hash falls within the group's rollout percentage, return the variant if it's overridden, or
-    ///   - Evaluate the variant using the global config and the global rollout percentage.
-    /// * Otherwise, continue with the next group until all groups are evaluated and no group is within the
-    ///   rollout percentage.
-    /// * If there are no matching groups, return an error.
-    ///
-    /// Example: we have a multivariate flag with 3 groups of the configured global rollout percentage: A (10%), B (20%), C (70%).
-    /// There is a single group with a condition that has a rollout percentage of 10% and it does not have a variant override.
-    /// Then, we will have 1% of the users evaluated to A, 2% to B, and 7% to C.
-    pub fn evaluate_multivariate(
-        &self,
-        flag_key: &str,
-        user_id: &str,
-    ) -> Result<String, PostHogEvaluationError> {
-        let hash_on_global_rollout_percentage =
-            Self::consistent_hash(user_id, flag_key, "multivariate");
-        let hash_on_group_rollout_percentage =
-            Self::consistent_hash(user_id, flag_key, "within_group");
-        self.evaluate_multivariate_inner(
-            flag_key,
-            hash_on_global_rollout_percentage,
-            hash_on_group_rollout_percentage,
-            &HashMap::new(),
-        )
-    }
-
-    /// Evaluate a multivariate feature flag. Note that we directly take the mapped user ID
-    /// (a consistent hash ranging from 0 to 1) so that it is easier to use it in the tests
-    /// and avoid duplicate computations.
-    ///
-    /// Use a different consistent hash for evaluating the group rollout percentage.
-    /// The behavior: if the condition is set to rolling out to 10% of the users, and
-    /// we set the variant A to 20% in the global config, then 2% of the total users will
-    /// be evaluated to variant A.
-    ///
-    /// Note that the hash to determine group rollout percentage is shared across all groups. So if we have two
-    /// exactly-the-same conditions with 10% and 20% rollout percentage respectively, a total of 20% of the users
-    /// will be evaluated (versus 30% if group evaluation is done independently).
-    pub(crate) fn evaluate_multivariate_inner(
-        &self,
-        flag_key: &str,
-        hash_on_global_rollout_percentage: f64,
-        hash_on_group_rollout_percentage: f64,
-        properties: &HashMap<String, PostHogFlagFilterPropertyValue>,
-    ) -> Result<String, PostHogEvaluationError> {
-        if let Some(flag_config) = self.flags.get(flag_key) {
-            if !flag_config.active {
-                return Err(PostHogEvaluationError::NotAvailable(format!(
-                    "The feature flag is not active: {}",
-                    flag_key
-                )));
-            }
-            // TODO: sort the groups so that variant overrides always get evaluated first and it follows the PostHog
-            // Python SDK behavior; for now we do not configure conditions without variant overrides in Neon so it
-            // does not matter.
-            for group in &flag_config.filters.groups {
-                match self.evaluate_group(group, hash_on_group_rollout_percentage, properties)? {
-                    GroupEvaluationResult::MatchedAndOverride(variant) => return Ok(variant),
-                    GroupEvaluationResult::MatchedAndEvaluate => {
-                        let mut percentage = 0;
-                        for variant in &flag_config.filters.multivariate.variants {
-                            percentage += variant.rollout_percentage;
-                            if self
-                                .evaluate_percentage(hash_on_global_rollout_percentage, percentage)
-                            {
-                                return Ok(variant.key.clone());
-                            }
-                        }
-                        // This should not happen because the rollout percentage always adds up to 100, but just in case that PostHog
-                        // returned invalid spec, we return an error.
-                        return Err(PostHogEvaluationError::Internal(format!(
-                            "Rollout percentage does not add up to 100: {}",
-                            flag_key
-                        )));
-                    }
-                    GroupEvaluationResult::Unmatched => continue,
-                }
-            }
-            // If no group is matched, the feature is not available, and up to the caller to decide what to do.
-            Err(PostHogEvaluationError::NoConditionGroupMatched)
-        } else {
-            // The feature flag is not available yet
-            Err(PostHogEvaluationError::NotAvailable(format!(
-                "Not found in the local evaluation spec: {}",
-                flag_key
-            )))
-        }
-    }
-}
-
-/// A lite PostHog client.
-///
-/// At the point of writing this code, PostHog does not have a functional Rust client with feature flag support.
-/// This is a lite version that only supports local evaluation of feature flags and only supports those JSON specs
-/// that will be used within Neon.
-///
-/// PostHog is designed as a browser-server system: the browser (client) side uses the client key and is exposed
-/// to the end users; the server side uses a server key and is not exposed to the end users. The client and the
-/// server has different API keys and provide a different set of APIs. In Neon, we only have the server (that is
-/// pageserver), and it will use both the client API and the server API. So we need to store two API keys within
-/// our PostHog client.
-///
-/// The server API is used to fetch the feature flag specs. The client API is used to capture events in case we
-/// want to report the feature flag usage back to PostHog. The current plan is to use PostHog only as an UI to
-/// configure feature flags so it is very likely that the client API will not be used.
-pub struct PostHogClient {
-    /// The server API key.
-    server_api_key: String,
-    /// The client API key.
-    client_api_key: String,
-    /// The project ID.
-    project_id: String,
-    /// The private API URL.
-    private_api_url: String,
-    /// The public API URL.
-    public_api_url: String,
-    /// The HTTP client.
-    client: reqwest::Client,
-}
-
-impl PostHogClient {
-    pub fn new(
-        server_api_key: String,
-        client_api_key: String,
-        project_id: String,
-        private_api_url: String,
-        public_api_url: String,
-    ) -> Self {
-        let client = reqwest::Client::new();
-        Self {
-            server_api_key,
-            client_api_key,
-            project_id,
-            private_api_url,
-            public_api_url,
-            client,
-        }
-    }
-
-    pub fn new_with_us_region(
-        server_api_key: String,
-        client_api_key: String,
-        project_id: String,
-    ) -> Self {
-        Self::new(
-            server_api_key,
-            client_api_key,
-            project_id,
-            "https://us.posthog.com".to_string(),
-            "https://us.i.posthog.com".to_string(),
-        )
-    }
-
-    /// Fetch the feature flag specs from the server.
-    ///
-    /// This is unfortunately an undocumented API at:
-    /// - <https://posthog.com/docs/api/feature-flags#get-api-projects-project_id-feature_flags-local_evaluation>
-    /// - <https://posthog.com/docs/feature-flags/local-evaluation>
-    ///
-    /// The handling logic in [`FeatureStore`] mostly follows the Python API implementation.
-    /// See `_compute_flag_locally` in <https://github.com/PostHog/posthog-python/blob/master/posthog/client.py>
-    pub async fn get_feature_flags_local_evaluation(
-        &self,
-    ) -> anyhow::Result<LocalEvaluationResponse> {
-        // BASE_URL/api/projects/:project_id/feature_flags/local_evaluation
-        // with bearer token of self.server_api_key
-        let url = format!(
-            "{}/api/projects/{}/feature_flags/local_evaluation",
-            self.private_api_url, self.project_id
-        );
-        let response = self
-            .client
-            .get(url)
-            .bearer_auth(&self.server_api_key)
-            .send()
-            .await?;
-        let body = response.text().await?;
-        Ok(serde_json::from_str(&body)?)
-    }
-
-    /// Capture an event. This will only be used to report the feature flag usage back to PostHog, though
-    /// it also support a lot of other functionalities.
-    ///
-    /// <https://posthog.com/docs/api/capture>
-    pub async fn capture_event(
-        &self,
-        event: &str,
-        distinct_id: &str,
-        properties: &HashMap<String, PostHogFlagFilterPropertyValue>,
-    ) -> anyhow::Result<()> {
-        // PUBLIC_URL/capture/
-        // with bearer token of self.client_api_key
-        let url = format!("{}/capture/", self.public_api_url);
-        self.client
-            .post(url)
-            .body(serde_json::to_string(&json!({
-                "api_key": self.client_api_key,
-                "distinct_id": distinct_id,
-                "event": event,
-                "properties": properties,
-            }))?)
-            .send()
-            .await?;
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    fn data() -> &'static str {
-        r#"{
-            "flags": [
-                {
-                    "id": 132794,
-                    "team_id": 152860,
-                    "name": "",
-                    "key": "gc-compaction",
-                    "filters": {
-                        "groups": [
-                            {
-                                "variant": "enabled-stage-2",
-                                "properties": [
-                                    {
-                                        "key": "plan_type",
-                                        "type": "person",
-                                        "value": [
-                                            "free"
-                                        ],
-                                        "operator": "exact"
-                                    },
-                                    {
-                                        "key": "pageserver_remote_size",
-                                        "type": "person",
-                                        "value": "10000000",
-                                        "operator": "lt"
-                                    }
-                                ],
-                                "rollout_percentage": 50
-                            },
-                            {
-                                "properties": [
-                                    {
-                                        "key": "plan_type",
-                                        "type": "person",
-                                        "value": [
-                                            "free"
-                                        ],
-                                        "operator": "exact"
-                                    },
-                                    {
-                                        "key": "pageserver_remote_size",
-                                        "type": "person",
-                                        "value": "10000000",
-                                        "operator": "lt"
-                                    }
-                                ],
-                                "rollout_percentage": 80
-                            }
-                        ],
-                        "payloads": {},
-                        "multivariate": {
-                            "variants": [
-                                {
-                                    "key": "disabled",
-                                    "name": "",
-                                    "rollout_percentage": 90
-                                },
-                                {
-                                    "key": "enabled-stage-1",
-                                    "name": "",
-                                    "rollout_percentage": 10
-                                },
-                                {
-                                    "key": "enabled-stage-2",
-                                    "name": "",
-                                    "rollout_percentage": 0
-                                },
-                                {
-                                    "key": "enabled-stage-3",
-                                    "name": "",
-                                    "rollout_percentage": 0
-                                },
-                                {
-                                    "key": "enabled",
-                                    "name": "",
-                                    "rollout_percentage": 0
-                                }
-                            ]
-                        }
-                    },
-                    "deleted": false,
-                    "active": true,
-                    "ensure_experience_continuity": false,
-                    "has_encrypted_payloads": false,
-                    "version": 6
-                }
-            ],
-            "group_type_mapping": {},
-            "cohorts": {}
-        }"#
-    }
-
-    #[test]
-    fn parse_local_evaluation() {
-        let data = data();
-        let _: LocalEvaluationResponse = serde_json::from_str(data).unwrap();
-    }
-
-    #[test]
-    fn evaluate_multivariate() {
-        let mut store = FeatureStore::new();
-        let response: LocalEvaluationResponse = serde_json::from_str(data()).unwrap();
-        store.set_flags(response.flags);
-
-        // This lacks the required properties and cannot be evaluated.
-        let variant =
-            store.evaluate_multivariate_inner("gc-compaction", 1.00, 0.40, &HashMap::new());
-        assert!(matches!(
-            variant,
-            Err(PostHogEvaluationError::NotAvailable(_))
-        ),);
-
-        let properties_unmatched = HashMap::from([
-            (
-                "plan_type".to_string(),
-                PostHogFlagFilterPropertyValue::String("paid".to_string()),
-            ),
-            (
-                "pageserver_remote_size".to_string(),
-                PostHogFlagFilterPropertyValue::Number(1000.0),
-            ),
-        ]);
-
-        // This does not match any group so there will be an error.
-        let variant =
-            store.evaluate_multivariate_inner("gc-compaction", 1.00, 0.40, &properties_unmatched);
-        assert!(matches!(
-            variant,
-            Err(PostHogEvaluationError::NoConditionGroupMatched)
-        ),);
-        let variant =
-            store.evaluate_multivariate_inner("gc-compaction", 0.80, 0.80, &properties_unmatched);
-        assert!(matches!(
-            variant,
-            Err(PostHogEvaluationError::NoConditionGroupMatched)
-        ),);
-
-        let properties = HashMap::from([
-            (
-                "plan_type".to_string(),
-                PostHogFlagFilterPropertyValue::String("free".to_string()),
-            ),
-            (
-                "pageserver_remote_size".to_string(),
-                PostHogFlagFilterPropertyValue::Number(1000.0),
-            ),
-        ]);
-
-        // It matches the first group as 0.10 <= 0.50 and the properties are matched. Then it gets evaluated to the variant override.
-        let variant = store.evaluate_multivariate_inner("gc-compaction", 0.10, 0.10, &properties);
-        assert_eq!(variant.unwrap(), "enabled-stage-2".to_string());
-
-        // It matches the second group as 0.50 <= 0.60 <= 0.80 and the properties are matched. Then it gets evaluated using the global percentage.
-        let variant = store.evaluate_multivariate_inner("gc-compaction", 0.99, 0.60, &properties);
-        assert_eq!(variant.unwrap(), "enabled-stage-1".to_string());
-        let variant = store.evaluate_multivariate_inner("gc-compaction", 0.80, 0.60, &properties);
-        assert_eq!(variant.unwrap(), "disabled".to_string());
-
-        // It matches the group conditions but not the group rollout percentage.
-        let variant = store.evaluate_multivariate_inner("gc-compaction", 1.00, 0.90, &properties);
-        assert!(matches!(
-            variant,
-            Err(PostHogEvaluationError::NoConditionGroupMatched)
-        ),);
-    }
-}

libs/remote_storage/src/azure_blob.rs

@@ -330,18 +330,11 @@ impl AzureBlobStorage {
                 if let Err(DownloadError::Timeout) = &next_item {
                     timeout_try_cnt += 1;
                     if timeout_try_cnt <= 5 {
-                        continue 'outer;
+                        continue;
                     }
                 }
 
-                let next_item = match next_item {
-                    Ok(next_item) => next_item,
-                    Err(e) => {
-                        // The error is potentially retryable, so we must rewind the loop after yielding.
-                        yield Err(e);
-                        continue 'outer;
-                    },
-                };
+                let next_item = next_item?;
 
                 // Log a warning if we saw two timeouts in a row before a successful request
                 if timeout_try_cnt > 2 {

libs/remote_storage/src/s3_bucket.rs

@@ -657,14 +657,7 @@ impl RemoteStorage for S3Bucket {
                     res = request => Ok(res),
                     _ = tokio::time::sleep(self.timeout) => Err(DownloadError::Timeout),
                     _ = cancel.cancelled() => Err(DownloadError::Cancelled),
-                };
-
-                if let Err(DownloadError::Timeout) = &response {
-                    yield Err(DownloadError::Timeout);
-                    continue 'outer;
-                }
-
-                let response = response?; // always yield cancellation errors and stop the stream
+                }?;
 
                 let response = response
                     .context("Failed to list S3 prefixes")

libs/utils/src/crashsafe.rs

@@ -1,7 +1,7 @@
 use std::borrow::Cow;
 use std::fs::{self, File};
 use std::io::{self, Write};
-use std::os::fd::AsFd;
+use std::os::fd::AsRawFd;
 
 use camino::{Utf8Path, Utf8PathBuf};
 
@@ -210,13 +210,13 @@ pub fn overwrite(
 
 /// Syncs the filesystem for the given file descriptor.
 #[cfg_attr(target_os = "macos", allow(unused_variables))]
-pub fn syncfs(fd: impl AsFd) -> anyhow::Result<()> {
+pub fn syncfs(fd: impl AsRawFd) -> anyhow::Result<()> {
     // Linux guarantees durability for syncfs.
     // POSIX doesn't have syncfs, and further does not actually guarantee durability of sync().
     #[cfg(target_os = "linux")]
     {
         use anyhow::Context;
-        nix::unistd::syncfs(fd).context("syncfs")?;
+        nix::unistd::syncfs(fd.as_raw_fd()).context("syncfs")?;
     }
     #[cfg(target_os = "macos")]
     {

libs/utils/src/fs_ext/rename_noreplace.rs

@@ -11,9 +11,9 @@ pub fn rename_noreplace<P1: ?Sized + NixPath, P2: ?Sized + NixPath>(
         #[cfg(all(target_os = "linux", target_env = "gnu"))]
         {
             nix::fcntl::renameat2(
-                nix::fcntl::AT_FDCWD,
+                None,
                 src,
-                nix::fcntl::AT_FDCWD,
+                None,
                 dst,
                 nix::fcntl::RenameFlags::RENAME_NOREPLACE,
             )

libs/utils/src/lock_file.rs

@@ -1,6 +1,6 @@
 //! A module to create and read lock files.
 //!
-//! File locking is done using [`nix::fcntl::Flock`] exclusive locks.
+//! File locking is done using [`fcntl::flock`] exclusive locks.
 //! The only consumer of this module is currently
 //! [`pid_file`](crate::pid_file). See the module-level comment
 //! there for potential pitfalls with lock files that are used
@@ -9,25 +9,26 @@
 use std::fs;
 use std::io::{Read, Write};
 use std::ops::Deref;
+use std::os::unix::prelude::AsRawFd;
 
 use anyhow::Context;
 use camino::{Utf8Path, Utf8PathBuf};
 use nix::errno::Errno::EAGAIN;
-use nix::fcntl::{Flock, FlockArg};
+use nix::fcntl;
 
 use crate::crashsafe;
 
-/// A handle to an open and flocked, but not-yet-written lock file.
+/// A handle to an open and unlocked, but not-yet-written lock file.
 /// Returned by [`create_exclusive`].
 #[must_use]
 pub struct UnwrittenLockFile {
     path: Utf8PathBuf,
-    file: Flock<fs::File>,
+    file: fs::File,
 }
 
 /// Returned by [`UnwrittenLockFile::write_content`].
 #[must_use]
-pub struct LockFileGuard(Flock<fs::File>);
+pub struct LockFileGuard(fs::File);
 
 impl Deref for LockFileGuard {
     type Target = fs::File;
@@ -66,14 +67,17 @@ pub fn create_exclusive(lock_file_path: &Utf8Path) -> anyhow::Result<UnwrittenLo
         .open(lock_file_path)
         .context("open lock file")?;
 
-    let res = Flock::lock(lock_file, FlockArg::LockExclusiveNonblock);
+    let res = fcntl::flock(
+        lock_file.as_raw_fd(),
+        fcntl::FlockArg::LockExclusiveNonblock,
+    );
     match res {
-        Ok(lock_file) => Ok(UnwrittenLockFile {
+        Ok(()) => Ok(UnwrittenLockFile {
             path: lock_file_path.to_owned(),
             file: lock_file,
         }),
-        Err((_, EAGAIN)) => anyhow::bail!("file is already locked"),
-        Err((_, e)) => Err(e).context("flock error"),
+        Err(EAGAIN) => anyhow::bail!("file is already locked"),
+        Err(e) => Err(e).context("flock error"),
     }
 }
 
@@ -101,37 +105,32 @@ pub enum LockFileRead {
 /// Check the [`LockFileRead`] variants for details.
 pub fn read_and_hold_lock_file(path: &Utf8Path) -> anyhow::Result<LockFileRead> {
     let res = fs::OpenOptions::new().read(true).open(path);
-    let lock_file = match res {
+    let mut lock_file = match res {
         Ok(f) => f,
         Err(e) => match e.kind() {
             std::io::ErrorKind::NotFound => return Ok(LockFileRead::NotExist),
             _ => return Err(e).context("open lock file"),
         },
     };
-    let res = Flock::lock(lock_file, FlockArg::LockExclusiveNonblock);
+    let res = fcntl::flock(
+        lock_file.as_raw_fd(),
+        fcntl::FlockArg::LockExclusiveNonblock,
+    );
     // We need the content regardless of lock success / failure.
     // But, read it after flock so that, if it succeeded, the content is consistent.
+    let mut content = String::new();
+    lock_file
+        .read_to_string(&mut content)
+        .context("read lock file")?;
     match res {
-        Ok(mut locked_file) => {
-            let mut content = String::new();
-            locked_file
-                .read_to_string(&mut content)
-                .context("read lock file")?;
-            Ok(LockFileRead::NotHeldByAnyProcess(
-                LockFileGuard(locked_file),
-                content,
-            ))
-        }
-        Err((mut not_locked_file, EAGAIN)) => {
-            let mut content = String::new();
-            not_locked_file
-                .read_to_string(&mut content)
-                .context("read lock file")?;
-            Ok(LockFileRead::LockedByOtherProcess {
-                not_locked_file,
-                content,
-            })
-        }
-        Err((_, e)) => Err(e).context("flock error"),
+        Ok(()) => Ok(LockFileRead::NotHeldByAnyProcess(
+            LockFileGuard(lock_file),
+            content,
+        )),
+        Err(EAGAIN) => Ok(LockFileRead::LockedByOtherProcess {
+            not_locked_file: lock_file,
+            content,
+        }),
+        Err(e) => Err(e).context("flock error"),
     }
 }

libs/utils/src/tracing_span_assert.rs

@@ -127,12 +127,12 @@ macro_rules! __check_fields_present {
 
             match check_fields_present0($extractors) {
                 Ok(FoundEverything) => Ok(()),
-                Ok(Unconfigured) if cfg!(feature = "testing") => {
+                Ok(Unconfigured) if cfg!(test) => {
                     // allow unconfigured in tests
                     Ok(())
                 },
                 Ok(Unconfigured) => {
-                    panic!(r#"utils::tracing_span_assert: outside of #[cfg(feature = "testing")] expected tracing to be configured with tracing_error::ErrorLayer"#)
+                    panic!("utils::tracing_span_assert: outside of #[cfg(test)] expected tracing to be configured with tracing_error::ErrorLayer")
                 },
                 Err(missing) => Err(missing)
             }

pageserver/Cargo.toml

@@ -96,7 +96,6 @@ strum.workspace = true
 strum_macros.workspace = true
 wal_decoder.workspace = true
 smallvec.workspace = true
-twox-hash.workspace = true
 
 [target.'cfg(target_os = "linux")'.dependencies]
 procfs.workspace = true

pageserver/client/src/mgmt_api.rs

@@ -1,6 +1,5 @@
 use std::collections::HashMap;
 use std::error::Error as _;
-use std::time::Duration;
 
 use bytes::Bytes;
 use detach_ancestor::AncestorDetached;
@@ -820,25 +819,4 @@ impl Client {
             .await
             .map(|resp| resp.status())
     }
-
-    pub async fn activate_post_import(
-        &self,
-        tenant_shard_id: TenantShardId,
-        timeline_id: TimelineId,
-        activate_timeline_timeout: Duration,
-    ) -> Result<TimelineInfo> {
-        let uri = format!(
-            "{}/v1/tenant/{}/timeline/{}/activate_post_import?timeline_activate_timeout_ms={}",
-            self.mgmt_api_endpoint,
-            tenant_shard_id,
-            timeline_id,
-            activate_timeline_timeout.as_millis()
-        );
-
-        self.request(Method::PUT, uri, ())
-            .await?
-            .json()
-            .await
-            .map_err(Error::ReceiveBody)
-    }
 }

pageserver/pagebench/src/cmd/getpage_latest_lsn.rs

@@ -65,9 +65,6 @@ pub(crate) struct Args {
     #[clap(long, default_value = "1")]
     queue_depth: NonZeroUsize,
 
-    #[clap(long)]
-    only_relnode: Option<u32>,
-
     targets: Option<Vec<TenantTimelineId>>,
 }
 
@@ -209,12 +206,7 @@ async fn main_impl(
                     for r in partitioning.keys.ranges.iter() {
                         let mut i = r.start;
                         while i != r.end {
-                            let mut include = true;
-                            include &= i.is_rel_block_key();
-                            if let Some(only_relnode) = args.only_relnode {
-                                include &= i.is_rel_block_of_rel(only_relnode);
-                            }
-                            if include {
+                            if i.is_rel_block_key() {
                                 filtered.add_key(i);
                             }
                             i = i.next();

pageserver/src/controller_upcall_client.rs

@@ -7,7 +7,7 @@ use pageserver_api::models::ShardImportStatus;
 use pageserver_api::shard::TenantShardId;
 use pageserver_api::upcall_api::{
     PutTimelineImportStatusRequest, ReAttachRequest, ReAttachResponse, ReAttachResponseTenant,
-    TimelineImportStatusRequest, ValidateRequest, ValidateRequestTenant, ValidateResponse,
+    ValidateRequest, ValidateRequestTenant, ValidateResponse,
 };
 use reqwest::Certificate;
 use serde::Serialize;
@@ -51,15 +51,8 @@ pub trait StorageControllerUpcallApi {
         &self,
         tenant_shard_id: TenantShardId,
         timeline_id: TimelineId,
-        generation: Generation,
         status: ShardImportStatus,
     ) -> impl Future<Output = Result<(), RetryForeverError>> + Send;
-    fn get_timeline_import_status(
-        &self,
-        tenant_shard_id: TenantShardId,
-        timeline_id: TimelineId,
-        generation: Generation,
-    ) -> impl Future<Output = Result<ShardImportStatus, RetryForeverError>> + Send;
 }
 
 impl StorageControllerUpcallClient {
@@ -104,7 +97,6 @@ impl StorageControllerUpcallClient {
         &self,
         url: &url::Url,
         request: R,
-        method: reqwest::Method,
     ) -> Result<T, RetryForeverError>
     where
         R: Serialize,
@@ -114,7 +106,7 @@ impl StorageControllerUpcallClient {
             || async {
                 let response = self
                     .http_client
-                    .request(method.clone(), url.clone())
+                    .post(url.clone())
                     .json(&request)
                     .send()
                     .await?;
@@ -223,9 +215,7 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
             register: register.clone(),
         };
 
-        let response: ReAttachResponse = self
-            .retry_http_forever(&url, request, reqwest::Method::POST)
-            .await?;
+        let response: ReAttachResponse = self.retry_http_forever(&url, request).await?;
         tracing::info!(
             "Received re-attach response with {} tenants (node {}, register: {:?})",
             response.tenants.len(),
@@ -278,9 +268,7 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
                 return Err(RetryForeverError::ShuttingDown);
             }
 
-            let response: ValidateResponse = self
-                .retry_http_forever(&url, request, reqwest::Method::POST)
-                .await?;
+            let response: ValidateResponse = self.retry_http_forever(&url, request).await?;
             for rt in response.tenants {
                 result.insert(rt.id, rt.valid);
             }
@@ -299,7 +287,6 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
         &self,
         tenant_shard_id: TenantShardId,
         timeline_id: TimelineId,
-        generation: Generation,
         status: ShardImportStatus,
     ) -> Result<(), RetryForeverError> {
         let url = self
@@ -310,35 +297,9 @@ impl StorageControllerUpcallApi for StorageControllerUpcallClient {
         let request = PutTimelineImportStatusRequest {
             tenant_shard_id,
             timeline_id,
-            generation,
             status,
         };
 
-        self.retry_http_forever(&url, request, reqwest::Method::POST)
-            .await
-    }
-
-    #[tracing::instrument(skip_all)] // so that warning logs from retry_http_forever have context
-    async fn get_timeline_import_status(
-        &self,
-        tenant_shard_id: TenantShardId,
-        timeline_id: TimelineId,
-        generation: Generation,
-    ) -> Result<ShardImportStatus, RetryForeverError> {
-        let url = self
-            .base_url
-            .join("timeline_import_status")
-            .expect("Failed to build path");
-
-        let request = TimelineImportStatusRequest {
-            tenant_shard_id,
-            timeline_id,
-            generation,
-        };
-
-        let response: ShardImportStatus = self
-            .retry_http_forever(&url, request, reqwest::Method::GET)
-            .await?;
-        Ok(response)
+        self.retry_http_forever(&url, request).await
     }
 }

pageserver/src/deletion_queue.rs

@@ -663,7 +663,6 @@ mod test {
     use camino::Utf8Path;
     use hex_literal::hex;
     use pageserver_api::key::Key;
-    use pageserver_api::models::ShardImportStatus;
     use pageserver_api::shard::ShardIndex;
     use pageserver_api::upcall_api::ReAttachResponseTenant;
     use remote_storage::{RemoteStorageConfig, RemoteStorageKind};
@@ -793,20 +792,10 @@ mod test {
             &self,
             _tenant_shard_id: TenantShardId,
             _timeline_id: TimelineId,
-            _generation: Generation,
             _status: pageserver_api::models::ShardImportStatus,
         ) -> Result<(), RetryForeverError> {
             unimplemented!()
         }
-
-        async fn get_timeline_import_status(
-            &self,
-            _tenant_shard_id: TenantShardId,
-            _timeline_id: TimelineId,
-            _generation: Generation,
-        ) -> Result<ShardImportStatus, RetryForeverError> {
-            unimplemented!()
-        }
     }
 
     async fn setup(test_name: &str) -> anyhow::Result<TestSetup> {

pageserver/src/http/routes.rs

@@ -3500,107 +3500,6 @@ async fn put_tenant_timeline_import_wal(
     }.instrument(span).await
 }
 
-/// Activate a timeline after its import has completed
-///
-/// The endpoint is idempotent and callers are expected to retry all
-/// errors until a successful response.
-async fn activate_post_import_handler(
-    request: Request<Body>,
-    _cancel: CancellationToken,
-) -> Result<Response<Body>, ApiError> {
-    let tenant_shard_id: TenantShardId = parse_request_param(&request, "tenant_shard_id")?;
-    check_permission(&request, Some(tenant_shard_id.tenant_id))?;
-
-    let timeline_id: TimelineId = parse_request_param(&request, "timeline_id")?;
-    const DEFAULT_ACTIVATE_TIMEOUT: Duration = Duration::from_secs(1);
-    let activate_timeout = parse_query_param(&request, "timeline_activate_timeout_ms")?
-        .map(Duration::from_millis)
-        .unwrap_or(DEFAULT_ACTIVATE_TIMEOUT);
-
-    let span = info_span!(
-        "activate_post_import_handler",
-        tenant_id=%tenant_shard_id.tenant_id,
-        timeline_id=%timeline_id,
-        shard_id=%tenant_shard_id.shard_slug()
-    );
-
-    async move {
-        let state = get_state(&request);
-        let tenant = state
-            .tenant_manager
-            .get_attached_tenant_shard(tenant_shard_id)?;
-
-        tenant.wait_to_become_active(ACTIVE_TENANT_TIMEOUT).await?;
-
-        tenant
-            .finalize_importing_timeline(timeline_id)
-            .await
-            .map_err(ApiError::InternalServerError)?;
-
-        match tenant.get_timeline(timeline_id, false) {
-            Ok(_timeline) => {
-                // Timeline is already visible. Reset not required: fall through.
-            }
-            Err(GetTimelineError::NotFound { .. }) => {
-                // This is crude: we reset the whole tenant such that the new timeline is detected
-                // and activated. We can come up with something more granular in the future.
-                //
-                // Note that we only reset the tenant if required: when the timeline is
-                // not present in [`Tenant::timelines`].
-                let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn);
-                state
-                    .tenant_manager
-                    .reset_tenant(tenant_shard_id, false, &ctx)
-                    .await
-                    .map_err(ApiError::InternalServerError)?;
-            }
-            Err(GetTimelineError::ShuttingDown) => {
-                return Err(ApiError::ShuttingDown);
-            }
-            Err(GetTimelineError::NotActive { .. }) => {
-                unreachable!("Called get_timeline with active_only=false");
-            }
-        }
-
-        let timeline = tenant.get_timeline(timeline_id, false)?;
-
-        let ctx = RequestContext::new(TaskKind::MgmtRequest, DownloadBehavior::Warn)
-            .with_scope_timeline(&timeline);
-
-        let result =
-            tokio::time::timeout(activate_timeout, timeline.wait_to_become_active(&ctx)).await;
-        match result {
-            Ok(Ok(())) => {
-                // fallthrough
-            }
-            // Timeline reached some other state that's not active
-            // TODO(vlad): if the tenant is broken, return a permananet error
-            Ok(Err(_timeline_state)) => {
-                return Err(ApiError::InternalServerError(anyhow::anyhow!(
-                    "Timeline activation failed"
-                )));
-            }
-            // Activation timed out
-            Err(_) => {
-                return Err(ApiError::Timeout("Timeline activation timed out".into()));
-            }
-        }
-
-        let timeline_info = build_timeline_info(
-            &timeline, false, // include_non_incremental_logical_size,
-            false, // force_await_initial_logical_size
-            &ctx,
-        )
-        .await
-        .context("get local timeline info")
-        .map_err(ApiError::InternalServerError)?;
-
-        json_response(StatusCode::OK, timeline_info)
-    }
-    .instrument(span)
-    .await
-}
-
 /// Read the end of a tar archive.
 ///
 /// A tar archive normally ends with two consecutive blocks of zeros, 512 bytes each.
@@ -4025,9 +3924,5 @@ pub fn make_router(
             "/v1/tenant/:tenant_id/timeline/:timeline_id/import_wal",
             |r| api_handler(r, put_tenant_timeline_import_wal),
         )
-        .put(
-            "/v1/tenant/:tenant_shard_id/timeline/:timeline_id/activate_post_import",
-            |r| api_handler(r, activate_post_import_handler),
-        )
         .any(handler_404))
 }

pageserver/src/tenant.rs

@@ -50,7 +50,6 @@ use remote_timeline_client::{
 use secondary::heatmap::{HeatMapTenant, HeatMapTimeline};
 use storage_broker::BrokerClientChannel;
 use timeline::compaction::{CompactionOutcome, GcCompactionQueue};
-use timeline::import_pgdata::ImportingTimeline;
 use timeline::offload::{OffloadError, offload_timeline};
 use timeline::{
     CompactFlags, CompactOptions, CompactionError, PreviousHeatmap, ShutdownMode, import_pgdata,
@@ -285,19 +284,6 @@ pub struct TenantShard {
     /// **Lock order**: if acquiring all (or a subset), acquire them in order `timelines`, `timelines_offloaded`, `timelines_creating`
     timelines_offloaded: Mutex<HashMap<TimelineId, Arc<OffloadedTimeline>>>,
 
-    /// Tracks the timelines that are currently importing into this tenant shard.
-    ///
-    /// Note that importing timelines are also present in [`Self::timelines_creating`].
-    /// Keep this in mind when ordering lock acquisition.
-    ///
-    /// Lifetime:
-    /// * An imported timeline is created while scanning the bucket on tenant attach
-    ///   if the index part contains an `import_pgdata` entry and said field marks the import
-    ///   as in progress.
-    /// * Imported timelines are removed when the storage controller calls the post timeline
-    ///   import activation endpoint.
-    timelines_importing: std::sync::Mutex<HashMap<TimelineId, ImportingTimeline>>,
-
     /// The last tenant manifest known to be in remote storage. None if the manifest has not yet
     /// been either downloaded or uploaded. Always Some after tenant attach.
     ///
@@ -937,10 +923,19 @@ enum StartCreatingTimelineResult {
 
 #[allow(clippy::large_enum_variant, reason = "TODO")]
 enum TimelineInitAndSyncResult {
-    ReadyToActivate,
+    ReadyToActivate(Arc<Timeline>),
     NeedsSpawnImportPgdata(TimelineInitAndSyncNeedsSpawnImportPgdata),
 }
 
+impl TimelineInitAndSyncResult {
+    fn ready_to_activate(self) -> Option<Arc<Timeline>> {
+        match self {
+            Self::ReadyToActivate(timeline) => Some(timeline),
+            _ => None,
+        }
+    }
+}
+
 #[must_use]
 struct TimelineInitAndSyncNeedsSpawnImportPgdata {
     timeline: Arc<Timeline>,
@@ -1017,6 +1012,10 @@ enum CreateTimelineCause {
 enum LoadTimelineCause {
     Attach,
     Unoffload,
+    ImportPgdata {
+        create_guard: TimelineCreateGuard,
+        activate: ActivateTimelineArgs,
+    },
 }
 
 #[derive(thiserror::Error, Debug)]
@@ -1098,7 +1097,7 @@ impl TenantShard {
         self: &Arc<Self>,
         timeline_id: TimelineId,
         resources: TimelineResources,
-        index_part: IndexPart,
+        mut index_part: IndexPart,
         metadata: TimelineMetadata,
         previous_heatmap: Option<PreviousHeatmap>,
         ancestor: Option<Arc<Timeline>>,
@@ -1107,7 +1106,7 @@ impl TenantShard {
     ) -> anyhow::Result<TimelineInitAndSyncResult> {
         let tenant_id = self.tenant_shard_id;
 
-        let import_pgdata = index_part.import_pgdata.clone();
+        let import_pgdata = index_part.import_pgdata.take();
         let idempotency = match &import_pgdata {
             Some(import_pgdata) => {
                 CreateTimelineIdempotency::ImportPgdata(CreatingTimelineIdempotencyImportPgdata {
@@ -1128,7 +1127,7 @@ impl TenantShard {
             }
         };
 
-        let (timeline, _timeline_ctx) = self.create_timeline_struct(
+        let (timeline, timeline_ctx) = self.create_timeline_struct(
             timeline_id,
             &metadata,
             previous_heatmap,
@@ -1198,6 +1197,14 @@ impl TenantShard {
 
         match import_pgdata {
             Some(import_pgdata) if !import_pgdata.is_done() => {
+                match cause {
+                    LoadTimelineCause::Attach | LoadTimelineCause::Unoffload => (),
+                    LoadTimelineCause::ImportPgdata { .. } => {
+                        unreachable!(
+                            "ImportPgdata should not be reloading timeline import is done and persisted as such in s3"
+                        )
+                    }
+                }
                 let mut guard = self.timelines_creating.lock().unwrap();
                 if !guard.insert(timeline_id) {
                     // We should never try and load the same timeline twice during startup
@@ -1253,7 +1260,26 @@ impl TenantShard {
                     "Timeline has no ancestor and no layer files"
                 );
 
-                Ok(TimelineInitAndSyncResult::ReadyToActivate)
+                match cause {
+                    LoadTimelineCause::Attach | LoadTimelineCause::Unoffload => (),
+                    LoadTimelineCause::ImportPgdata {
+                        create_guard,
+                        activate,
+                    } => {
+                        // TODO: see the comment in the task code above how I'm not so certain
+                        // it is safe to activate here because of concurrent shutdowns.
+                        match activate {
+                            ActivateTimelineArgs::Yes { broker_client } => {
+                                info!("activating timeline after reload from pgdata import task");
+                                timeline.activate(self.clone(), broker_client, None, &timeline_ctx);
+                            }
+                            ActivateTimelineArgs::No => (),
+                        }
+                        drop(create_guard);
+                    }
+                }
+
+                Ok(TimelineInitAndSyncResult::ReadyToActivate(timeline))
             }
         }
     }
@@ -1742,7 +1768,7 @@ impl TenantShard {
                 })?;
 
             match effect {
-                TimelineInitAndSyncResult::ReadyToActivate => {
+                TimelineInitAndSyncResult::ReadyToActivate(_) => {
                     // activation happens later, on Tenant::activate
                 }
                 TimelineInitAndSyncResult::NeedsSpawnImportPgdata(
@@ -1752,24 +1778,13 @@ impl TenantShard {
                         guard,
                     },
                 ) => {
-                    let timeline_id = timeline.timeline_id;
-                    let import_task_handle =
-                        tokio::task::spawn(self.clone().create_timeline_import_pgdata_task(
-                            timeline.clone(),
-                            import_pgdata,
-                            guard,
-                            ctx.detached_child(TaskKind::ImportPgdata, DownloadBehavior::Warn),
-                        ));
-
-                    let prev = self.timelines_importing.lock().unwrap().insert(
-                        timeline_id,
-                        ImportingTimeline {
-                            timeline: timeline.clone(),
-                            import_task_handle,
-                        },
-                    );
-
-                    assert!(prev.is_none());
+                    tokio::task::spawn(self.clone().create_timeline_import_pgdata_task(
+                        timeline,
+                        import_pgdata,
+                        ActivateTimelineArgs::No,
+                        guard,
+                        ctx.detached_child(TaskKind::ImportPgdata, DownloadBehavior::Warn),
+                    ));
                 }
             }
         }
@@ -2663,7 +2678,14 @@ impl TenantShard {
                     .await?
             }
             CreateTimelineParams::ImportPgdata(params) => {
-                self.create_timeline_import_pgdata(params, ctx).await?
+                self.create_timeline_import_pgdata(
+                    params,
+                    ActivateTimelineArgs::Yes {
+                        broker_client: broker_client.clone(),
+                    },
+                    ctx,
+                )
+                .await?
             }
         };
 
@@ -2737,6 +2759,7 @@ impl TenantShard {
     async fn create_timeline_import_pgdata(
         self: &Arc<Self>,
         params: CreateTimelineParamsImportPgdata,
+        activate: ActivateTimelineArgs,
         ctx: &RequestContext,
     ) -> Result<CreateTimelineResult, CreateTimelineError> {
         let CreateTimelineParamsImportPgdata {
@@ -2817,71 +2840,24 @@ impl TenantShard {
 
         let (timeline, timeline_create_guard) = uninit_timeline.finish_creation_myself();
 
-        let import_task_handle = tokio::spawn(self.clone().create_timeline_import_pgdata_task(
+        tokio::spawn(self.clone().create_timeline_import_pgdata_task(
             timeline.clone(),
             index_part,
+            activate,
             timeline_create_guard,
             timeline_ctx.detached_child(TaskKind::ImportPgdata, DownloadBehavior::Warn),
         ));
 
-        let prev = self.timelines_importing.lock().unwrap().insert(
-            timeline.timeline_id,
-            ImportingTimeline {
-                timeline: timeline.clone(),
-                import_task_handle,
-            },
-        );
-
-        // Idempotency is enforced higher up the stack
-        assert!(prev.is_none());
-
         // NB: the timeline doesn't exist in self.timelines at this point
         Ok(CreateTimelineResult::ImportSpawned(timeline))
     }
 
-    /// Finalize the import of a timeline on this shard by marking it complete in
-    /// the index part. If the import task hasn't finished yet, returns an error.
-    ///
-    /// This method is idempotent. If the import was finalized once, the next call
-    /// will be a no-op.
-    pub(crate) async fn finalize_importing_timeline(
-        &self,
-        timeline_id: TimelineId,
-    ) -> anyhow::Result<()> {
-        let timeline = {
-            let locked = self.timelines_importing.lock().unwrap();
-            match locked.get(&timeline_id) {
-                Some(importing_timeline) => {
-                    if !importing_timeline.import_task_handle.is_finished() {
-                        return Err(anyhow::anyhow!("Import task not done yet"));
-                    }
-
-                    importing_timeline.timeline.clone()
-                }
-                None => {
-                    return Ok(());
-                }
-            }
-        };
-
-        timeline
-            .remote_client
-            .schedule_index_upload_for_import_pgdata_finalize()?;
-        timeline.remote_client.wait_completion().await?;
-
-        self.timelines_importing
-            .lock()
-            .unwrap()
-            .remove(&timeline_id);
-
-        Ok(())
-    }
-
     #[instrument(skip_all, fields(tenant_id=%self.tenant_shard_id.tenant_id, shard_id=%self.tenant_shard_id.shard_slug(), timeline_id=%timeline.timeline_id))]
     async fn create_timeline_import_pgdata_task(
         self: Arc<TenantShard>,
         timeline: Arc<Timeline>,
         index_part: import_pgdata::index_part_format::Root,
+        activate: ActivateTimelineArgs,
         timeline_create_guard: TimelineCreateGuard,
         ctx: RequestContext,
     ) {
@@ -2893,6 +2869,7 @@ impl TenantShard {
             .create_timeline_import_pgdata_task_impl(
                 timeline,
                 index_part,
+                activate,
                 timeline_create_guard,
                 ctx,
             )
@@ -2908,15 +2885,60 @@ impl TenantShard {
         self: Arc<TenantShard>,
         timeline: Arc<Timeline>,
         index_part: import_pgdata::index_part_format::Root,
-        _timeline_create_guard: TimelineCreateGuard,
+        activate: ActivateTimelineArgs,
+        timeline_create_guard: TimelineCreateGuard,
         ctx: RequestContext,
     ) -> Result<(), anyhow::Error> {
         info!("importing pgdata");
-        let ctx = ctx.with_scope_timeline(&timeline);
         import_pgdata::doit(&timeline, index_part, &ctx, self.cancel.clone())
             .await
             .context("import")?;
-        info!("import done - waiting for activation");
+        info!("import done");
+
+        //
+        // Reload timeline from remote.
+        // This proves that the remote state is attachable, and it reuses the code.
+        //
+        // TODO: think about whether this is safe to do with concurrent TenantShard::shutdown.
+        // timeline_create_guard hols the tenant gate open, so, shutdown cannot _complete_ until we exit.
+        // But our activate() call might launch new background tasks after TenantShard::shutdown
+        // already went past shutting down the TenantShard::timelines, which this timeline here is no part of.
+        // I think the same problem exists with the bootstrap & branch mgmt API tasks (tenant shutting
+        // down while bootstrapping/branching + activating), but, the race condition is much more likely
+        // to manifest because of the long runtime of this import task.
+
+        //        in theory this shouldn't even .await anything except for coop yield
+        info!("shutting down timeline");
+        timeline.shutdown(ShutdownMode::Hard).await;
+        info!("timeline shut down, reloading from remote");
+        // TODO: we can't do the following check because create_timeline_import_pgdata must return an Arc<Timeline>
+        // let Some(timeline) = Arc::into_inner(timeline) else {
+        //     anyhow::bail!("implementation error: timeline that we shut down was still referenced from somewhere");
+        // };
+        let timeline_id = timeline.timeline_id;
+
+        // load from object storage like TenantShard::attach does
+        let resources = self.build_timeline_resources(timeline_id);
+        let index_part = resources
+            .remote_client
+            .download_index_file(&self.cancel)
+            .await?;
+        let index_part = match index_part {
+            MaybeDeletedIndexPart::Deleted(_) => {
+                // likely concurrent delete call, cplane should prevent this
+                anyhow::bail!(
+                    "index part says deleted but we are not done creating yet, this should not happen but"
+                )
+            }
+            MaybeDeletedIndexPart::IndexPart(p) => p,
+        };
+        let metadata = index_part.metadata.clone();
+        self
+            .load_remote_timeline(timeline_id, index_part, metadata, None, resources, LoadTimelineCause::ImportPgdata{
+                create_guard: timeline_create_guard, activate, }, &ctx)
+            .await?
+            .ready_to_activate()
+            .context("implementation error: reloaded timeline still needs import after import reported success")?;
 
         anyhow::Ok(())
     }
@@ -3453,14 +3475,6 @@ impl TenantShard {
                 timeline.defuse_for_tenant_drop();
             });
         }
-        {
-            let mut timelines_importing = self.timelines_importing.lock().unwrap();
-            timelines_importing
-                .drain()
-                .for_each(|(_timeline_id, importing_timeline)| {
-                    importing_timeline.shutdown();
-                });
-        }
         // test_long_timeline_create_then_tenant_delete is leaning on this message
         tracing::info!("Waiting for timelines...");
         while let Some(res) = js.join_next().await {
@@ -3935,6 +3949,13 @@ where
     Ok(result)
 }
 
+enum ActivateTimelineArgs {
+    Yes {
+        broker_client: storage_broker::BrokerClientChannel,
+    },
+    No,
+}
+
 impl TenantShard {
     pub fn tenant_specific_overrides(&self) -> pageserver_api::models::TenantConfig {
         self.tenant_conf.load().tenant_conf.clone()
@@ -4301,7 +4322,6 @@ impl TenantShard {
             timelines: Mutex::new(HashMap::new()),
             timelines_creating: Mutex::new(HashSet::new()),
             timelines_offloaded: Mutex::new(HashMap::new()),
-            timelines_importing: Mutex::new(HashMap::new()),
             remote_tenant_manifest: Default::default(),
             gc_cs: tokio::sync::Mutex::new(()),
             walredo_mgr,

pageserver/src/tenant/remote_timeline_client.rs

@@ -949,35 +949,6 @@ impl RemoteTimelineClient {
         Ok(())
     }
 
-    /// If the `import_pgdata` field marks the timeline as having an import in progress,
-    /// launch an index-file upload operation that transitions it to done in the background
-    pub(crate) fn schedule_index_upload_for_import_pgdata_finalize(
-        self: &Arc<Self>,
-    ) -> anyhow::Result<()> {
-        use import_pgdata::index_part_format;
-
-        let mut guard = self.upload_queue.lock().unwrap();
-        let upload_queue = guard.initialized_mut()?;
-        let to_update = match &upload_queue.dirty.import_pgdata {
-            Some(import) if !import.is_done() => Some(import),
-            Some(_) | None => None,
-        };
-
-        if let Some(old) = to_update {
-            let new =
-                index_part_format::Root::V1(index_part_format::V1::Done(index_part_format::Done {
-                    idempotency_key: old.idempotency_key().clone(),
-                    started_at: *old.started_at(),
-                    finished_at: chrono::Utc::now().naive_utc(),
-                }));
-
-            upload_queue.dirty.import_pgdata = Some(new);
-            self.schedule_index_upload(upload_queue);
-        }
-
-        Ok(())
-    }
-
     /// Launch an index-file upload operation in the background, setting `gc_compaction_state` field.
     pub(crate) fn schedule_index_upload_for_gc_compaction_state_update(
         self: &Arc<Self>,

pageserver/src/tenant/secondary/downloader.rs

@@ -668,9 +668,7 @@ impl From<DownloadError> for UpdateError {
 
 impl From<std::io::Error> for UpdateError {
     fn from(value: std::io::Error) -> Self {
-        if let Some(nix::errno::Errno::ENOSPC) =
-            value.raw_os_error().map(nix::errno::Errno::from_raw)
-        {
+        if let Some(nix::errno::Errno::ENOSPC) = value.raw_os_error().map(nix::errno::from_i32) {
             UpdateError::NoSpace
         } else if value
             .get_ref()

pageserver/src/tenant/timeline/compaction.rs

@@ -3435,7 +3435,6 @@ impl Timeline {
 
         // Step 2: Produce images+deltas.
         let mut accumulated_values = Vec::new();
-        let mut accumulated_values_estimated_size = 0;
         let mut last_key: Option<Key> = None;
 
         // Only create image layers when there is no ancestor branches. TODO: create covering image layer
@@ -3612,16 +3611,12 @@ impl Timeline {
                 if last_key.is_none() {
                     last_key = Some(key);
                 }
-                accumulated_values_estimated_size += val.estimated_size();
                 accumulated_values.push((key, lsn, val));
 
-                // Accumulated values should never exceed 512MB.
-                if accumulated_values_estimated_size >= 1024 * 1024 * 512 {
+                if accumulated_values.len() >= 65536 {
+                    // Assume all of them are images, that would be 512MB of data in memory for a single key.
                     return Err(CompactionError::Other(anyhow!(
-                        "too many values for a single key: {} for key {}, {} items",
-                        accumulated_values_estimated_size,
-                        key,
-                        accumulated_values.len()
+                        "too many values for a single key, giving up gc-compaction"
                     )));
                 }
             } else {
@@ -3656,7 +3651,6 @@ impl Timeline {
                     .map_err(CompactionError::Other)?;
                 accumulated_values.clear();
                 *last_key = key;
-                accumulated_values_estimated_size = val.estimated_size();
                 accumulated_values.push((key, lsn, val));
             }
         }

pageserver/src/tenant/timeline/import_pgdata.rs

@@ -1,10 +1,8 @@
 use std::sync::Arc;
 
 use anyhow::{Context, bail};
-use importbucket_client::{ControlFile, RemoteStorageWrapper};
 use pageserver_api::models::ShardImportStatus;
 use remote_storage::RemotePath;
-use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 use tracing::info;
 use utils::lsn::Lsn;
@@ -19,17 +17,6 @@ mod importbucket_client;
 mod importbucket_format;
 pub(crate) mod index_part_format;
 
-pub(crate) struct ImportingTimeline {
-    pub import_task_handle: JoinHandle<()>,
-    pub timeline: Arc<Timeline>,
-}
-
-impl ImportingTimeline {
-    pub(crate) fn shutdown(self) {
-        self.import_task_handle.abort();
-    }
-}
-
 pub async fn doit(
     timeline: &Arc<Timeline>,
     index_part: index_part_format::Root,
@@ -39,225 +26,173 @@ pub async fn doit(
     let index_part_format::Root::V1(v1) = index_part;
     let index_part_format::InProgress {
         location,
-        idempotency_key: _,
-        started_at: _,
+        idempotency_key,
+        started_at,
     } = match v1 {
         index_part_format::V1::Done(_) => return Ok(()),
         index_part_format::V1::InProgress(in_progress) => in_progress,
     };
 
-    let storcon_client = StorageControllerUpcallClient::new(timeline.conf, &cancel);
+    let storage = importbucket_client::new(timeline.conf, &location, cancel.clone()).await?;
 
-    let shard_status = storcon_client
-        .get_timeline_import_status(
-            timeline.tenant_shard_id,
-            timeline.timeline_id,
-            timeline.generation,
-        )
-        .await
-        .map_err(|_err| anyhow::anyhow!("Shut down while getting timeline import status"))?;
-
-    info!(?shard_status, "peeking shard status");
-    match shard_status {
-        ShardImportStatus::InProgress(maybe_progress) => {
-            let storage =
-                importbucket_client::new(timeline.conf, &location, cancel.clone()).await?;
-
-            let control_file_res = if maybe_progress.is_none() {
-                // Only prepare the import once when there's no progress.
-                prepare_import(timeline, storage.clone(), &cancel).await
-            } else {
-                storage.get_control_file().await
-            };
-
-            let control_file = match control_file_res {
-                Ok(cf) => cf,
-                Err(err) => {
-                    return Err(
-                        terminate_flow_with_error(timeline, err, &storcon_client, &cancel).await,
-                    );
-                }
-            };
-
-            let res = flow::run(
-                timeline.clone(),
-                control_file,
-                storage.clone(),
-                maybe_progress,
-                ctx,
-            )
-            .await;
-            if let Err(err) = res {
-                return Err(
-                    terminate_flow_with_error(timeline, err, &storcon_client, &cancel).await,
-                );
-            }
-
-            // Communicate that shard is done.
-            // Ensure at-least-once delivery of the upcall to storage controller
-            // before we mark the task as done and never come here again.
-            //
-            // Note that we do not mark the import complete in the index part now.
-            // This happens in [`Tenant::finalize_importing_timeline`] in response
-            // to the storage controller calling
-            // `/v1/tenant/:tenant_id/timeline/:timeline_id/activate_post_import`.
-            storcon_client
-                .put_timeline_import_status(
-                    timeline.tenant_shard_id,
-                    timeline.timeline_id,
-                    timeline.generation,
-                    ShardImportStatus::Done,
-                )
-                .await
-                .map_err(|_err| {
-                    anyhow::anyhow!("Shut down while putting timeline import status")
-                })?;
-        }
-        ShardImportStatus::Error(err) => {
-            info!(
-                "shard status indicates that the shard is done (error), skipping import {}",
-                err
-            );
-        }
-        ShardImportStatus::Done => {
-            info!("shard status indicates that the shard is done (success), skipping import");
-        }
-    }
-
-    Ok(())
-}
-
-async fn prepare_import(
-    timeline: &Arc<Timeline>,
-    storage: RemoteStorageWrapper,
-    cancel: &CancellationToken,
-) -> anyhow::Result<ControlFile> {
-    // Wipe the slate clean before starting the import as a precaution.
-    // This method is only called when there's no recorded checkpoint for the import
-    // in the storage controller.
-    //
-    // Note that this is split-brain safe (two imports for same timeline shards running in
-    // different generations) because we go through the usual deletion path, including deletion queue.
-    info!("wipe the slate clean");
-    {
-        // TODO: do we need to hold GC lock for this?
-        let mut guard = timeline.layers.write().await;
-        assert!(
-            guard.layer_map()?.open_layer.is_none(),
-            "while importing, there should be no in-memory layer" // this just seems like a good place to assert it
-        );
-        let all_layers_keys = guard.all_persistent_layers();
-        let all_layers: Vec<_> = all_layers_keys
-            .iter()
-            .map(|key| guard.get_from_key(key))
-            .collect();
-        let open = guard.open_mut().context("open_mut")?;
-
-        timeline.remote_client.schedule_gc_update(&all_layers)?;
-        open.finish_gc_timeline(&all_layers);
-    }
-
-    //
-    // Wait for pgdata to finish uploading
-    //
-    info!("wait for pgdata to reach status 'done'");
     let status_prefix = RemotePath::from_string("status").unwrap();
-    let pgdata_status_key = status_prefix.join("pgdata");
-    loop {
-        let res = async {
-            let pgdata_status: Option<importbucket_format::PgdataStatus> = storage
-                .get_json(&pgdata_status_key)
-                .await
-                .context("get pgdata status")?;
-            info!(?pgdata_status, "peeking pgdata status");
-            if pgdata_status.map(|st| st.done).unwrap_or(false) {
-                Ok(())
-            } else {
-                Err(anyhow::anyhow!("pgdata not done yet"))
-            }
+
+    //
+    // See if shard is done.
+    // TODO: incorporate generations into status key for split brain safety. Figure out together with checkpointing.
+    //
+    let shard_status_key =
+        status_prefix.join(format!("shard-{}", timeline.tenant_shard_id.shard_slug()));
+    let shard_status: Option<importbucket_format::ShardStatus> =
+        storage.get_json(&shard_status_key).await?;
+    info!(?shard_status, "peeking shard status");
+    if shard_status.map(|st| st.done).unwrap_or(false) {
+        info!("shard status indicates that the shard is done, skipping import");
+    } else {
+        // TODO: checkpoint the progress into the IndexPart instead of restarting
+        // from the beginning.
+
+        //
+        // Wipe the slate clean - the flow does not allow resuming.
+        // We can implement resuming in the future by checkpointing the progress into the IndexPart.
+        //
+        info!("wipe the slate clean");
+        {
+            // TODO: do we need to hold GC lock for this?
+            let mut guard = timeline.layers.write().await;
+            assert!(
+                guard.layer_map()?.open_layer.is_none(),
+                "while importing, there should be no in-memory layer" // this just seems like a good place to assert it
+            );
+            let all_layers_keys = guard.all_persistent_layers();
+            let all_layers: Vec<_> = all_layers_keys
+                .iter()
+                .map(|key| guard.get_from_key(key))
+                .collect();
+            let open = guard.open_mut().context("open_mut")?;
+
+            timeline.remote_client.schedule_gc_update(&all_layers)?;
+            open.finish_gc_timeline(&all_layers);
         }
-        .await;
-        match res {
-            Ok(_) => break,
-            Err(err) => {
-                info!(?err, "indefinitely waiting for pgdata to finish");
-                if tokio::time::timeout(std::time::Duration::from_secs(10), cancel.cancelled())
+
+        //
+        // Wait for pgdata to finish uploading
+        //
+        info!("wait for pgdata to reach status 'done'");
+        let pgdata_status_key = status_prefix.join("pgdata");
+        loop {
+            let res = async {
+                let pgdata_status: Option<importbucket_format::PgdataStatus> = storage
+                    .get_json(&pgdata_status_key)
                     .await
-                    .is_ok()
-                {
-                    bail!("cancelled while waiting for pgdata");
+                    .context("get pgdata status")?;
+                info!(?pgdata_status, "peeking pgdata status");
+                if pgdata_status.map(|st| st.done).unwrap_or(false) {
+                    Ok(())
+                } else {
+                    Err(anyhow::anyhow!("pgdata not done yet"))
+                }
+            }
+            .await;
+            match res {
+                Ok(_) => break,
+                Err(err) => {
+                    info!(?err, "indefinitely waiting for pgdata to finish");
+                    if tokio::time::timeout(std::time::Duration::from_secs(10), cancel.cancelled())
+                        .await
+                        .is_ok()
+                    {
+                        bail!("cancelled while waiting for pgdata");
+                    }
                 }
             }
         }
-    }
 
-    let control_file = storage.get_control_file().await?;
-    let base_lsn = control_file.base_lsn();
+        //
+        // Do the import
+        //
+        info!("do the import");
+        let control_file = storage.get_control_file().await?;
+        let base_lsn = control_file.base_lsn();
 
-    info!("update TimelineMetadata based on LSNs from control file");
-    {
-        let pg_version = control_file.pg_version();
-        async move {
-            // FIXME: The 'disk_consistent_lsn' should be the LSN at the *end* of the
-            // checkpoint record, and prev_record_lsn should point to its beginning.
-            // We should read the real end of the record from the WAL, but here we
-            // just fake it.
-            let disk_consistent_lsn = Lsn(base_lsn.0 + 8);
-            let prev_record_lsn = base_lsn;
-            let metadata = TimelineMetadata::new(
-                disk_consistent_lsn,
-                Some(prev_record_lsn),
-                None,     // no ancestor
-                Lsn(0),   // no ancestor lsn
-                base_lsn, // latest_gc_cutoff_lsn
-                base_lsn, // initdb_lsn
-                pg_version,
-            );
+        info!("update TimelineMetadata based on LSNs from control file");
+        {
+            let pg_version = control_file.pg_version();
+            let _ctx: &RequestContext = ctx;
+            async move {
+                // FIXME: The 'disk_consistent_lsn' should be the LSN at the *end* of the
+                // checkpoint record, and prev_record_lsn should point to its beginning.
+                // We should read the real end of the record from the WAL, but here we
+                // just fake it.
+                let disk_consistent_lsn = Lsn(base_lsn.0 + 8);
+                let prev_record_lsn = base_lsn;
+                let metadata = TimelineMetadata::new(
+                    disk_consistent_lsn,
+                    Some(prev_record_lsn),
+                    None,     // no ancestor
+                    Lsn(0),   // no ancestor lsn
+                    base_lsn, // latest_gc_cutoff_lsn
+                    base_lsn, // initdb_lsn
+                    pg_version,
+                );
 
-            let _start_lsn = disk_consistent_lsn + 1;
+                let _start_lsn = disk_consistent_lsn + 1;
 
-            timeline
-                .remote_client
-                .schedule_index_upload_for_full_metadata_update(&metadata)?;
+                timeline
+                    .remote_client
+                    .schedule_index_upload_for_full_metadata_update(&metadata)?;
 
-            timeline.remote_client.wait_completion().await?;
+                timeline.remote_client.wait_completion().await?;
 
-            anyhow::Ok(())
+                anyhow::Ok(())
+            }
         }
-    }
-    .await?;
+        .await?;
 
-    Ok(control_file)
-}
+        flow::run(timeline.clone(), control_file, storage.clone(), ctx).await?;
 
-async fn terminate_flow_with_error(
-    timeline: &Arc<Timeline>,
-    error: anyhow::Error,
-    storcon_client: &StorageControllerUpcallClient,
-    cancel: &CancellationToken,
-) -> anyhow::Error {
-    // The import task is a aborted on tenant shutdown, so in principle, it should
-    // never be cancelled. To be on the safe side, check the cancellation tokens
-    // before marking the import as failed.
-    if !(cancel.is_cancelled() || timeline.cancel.is_cancelled()) {
-        let notify_res = storcon_client
+        //
+        // Communicate that shard is done.
+        // Ensure at-least-once delivery of the upcall to storage controller
+        // before we mark the task as done and never come here again.
+        //
+        let storcon_client = StorageControllerUpcallClient::new(timeline.conf, &cancel);
+        storcon_client
             .put_timeline_import_status(
                 timeline.tenant_shard_id,
                 timeline.timeline_id,
-                timeline.generation,
-                ShardImportStatus::Error(format!("{error:#}")),
+                // TODO(vlad): What about import errors?
+                ShardImportStatus::Done,
             )
-            .await;
+            .await
+            .map_err(|_err| anyhow::anyhow!("Shut down while putting timeline import status"))?;
 
-        if let Err(_notify_error) = notify_res {
-            // The [`StorageControllerUpcallClient::put_timeline_import_status`] retries
-            // forever internally, so errors returned by it can only be due to cancellation.
-            info!("failed to notify storcon about permanent import error");
-        }
-
-        // Will be logged by [`Tenant::create_timeline_import_pgdata_task`]
-        error
-    } else {
-        anyhow::anyhow!("Import task cancelled")
+        storage
+            .put_json(
+                &shard_status_key,
+                &importbucket_format::ShardStatus { done: true },
+            )
+            .await
+            .context("put shard status")?;
     }
+
+    //
+    // Mark as done in index_part.
+    // This makes subsequent timeline loads enter the normal load code path
+    // instead of spawning the import task and calling this here function.
+    //
+    info!("mark import as complete in index part");
+    timeline
+        .remote_client
+        .schedule_index_upload_for_import_pgdata_state_update(Some(index_part_format::Root::V1(
+            index_part_format::V1::Done(index_part_format::Done {
+                idempotency_key,
+                started_at,
+                finished_at: chrono::Utc::now().naive_utc(),
+            }),
+        )))?;
+
+    timeline.remote_client.wait_completion().await?;
+
+    Ok(())
 }

pageserver/src/tenant/timeline/import_pgdata/flow.rs

@@ -29,11 +29,10 @@
 //! - version-specific CheckPointData (=> pgv abstraction, already exists for regular walingest)
 
 use std::collections::HashSet;
-use std::hash::{Hash, Hasher};
 use std::ops::Range;
 use std::sync::Arc;
 
-use anyhow::ensure;
+use anyhow::{bail, ensure};
 use bytes::Bytes;
 use futures::stream::FuturesOrdered;
 use itertools::Itertools;
@@ -44,7 +43,6 @@ use pageserver_api::key::{
     slru_segment_size_to_key,
 };
 use pageserver_api::keyspace::{contiguous_range_len, is_contiguous_range, singleton_range};
-use pageserver_api::models::{ShardImportProgress, ShardImportProgressV1, ShardImportStatus};
 use pageserver_api::reltag::{RelTag, SlruKind};
 use pageserver_api::shard::ShardIdentity;
 use postgres_ffi::relfile_utils::parse_relfilename;
@@ -55,42 +53,21 @@ use tokio_stream::StreamExt;
 use tracing::{debug, instrument};
 use utils::bin_ser::BeSer;
 use utils::lsn::Lsn;
-use utils::pausable_failpoint;
 
 use super::Timeline;
 use super::importbucket_client::{ControlFile, RemoteStorageWrapper};
 use crate::assert_u64_eq_usize::UsizeIsU64;
 use crate::context::{DownloadBehavior, RequestContext};
-use crate::controller_upcall_client::{StorageControllerUpcallApi, StorageControllerUpcallClient};
 use crate::pgdatadir_mapping::{
     DbDirectory, RelDirectory, SlruSegmentDirectory, TwoPhaseDirectory,
 };
 use crate::task_mgr::TaskKind;
-use crate::tenant::storage_layer::{AsLayerDesc, ImageLayerWriter, Layer};
+use crate::tenant::storage_layer::{ImageLayerWriter, Layer};
 
 pub async fn run(
     timeline: Arc<Timeline>,
     control_file: ControlFile,
     storage: RemoteStorageWrapper,
-    import_progress: Option<ShardImportProgress>,
-    ctx: &RequestContext,
-) -> anyhow::Result<()> {
-    // Match how we run the import based on the progress version.
-    // If there's no import progress, it means that this is a new import
-    // and we can use whichever version we want.
-    match import_progress {
-        Some(ShardImportProgress::V1(progress)) => {
-            run_v1(timeline, control_file, storage, Some(progress), ctx).await
-        }
-        None => run_v1(timeline, control_file, storage, None, ctx).await,
-    }
-}
-
-async fn run_v1(
-    timeline: Arc<Timeline>,
-    control_file: ControlFile,
-    storage: RemoteStorageWrapper,
-    import_progress: Option<ShardImportProgressV1>,
     ctx: &RequestContext,
 ) -> anyhow::Result<()> {
     let planner = Planner {
@@ -102,32 +79,7 @@ async fn run_v1(
 
     let import_config = &timeline.conf.timeline_import_config;
     let plan = planner.plan(import_config).await?;
-
-    // Hash the plan and compare with the hash of the plan we got back from the storage controller.
-    // If the two match, it means that the planning stage had the same output.
-    //
-    // This is not intended to be a cryptographically secure hash.
-    const SEED: u64 = 42;
-    let mut hasher = twox_hash::XxHash64::with_seed(SEED);
-    plan.hash(&mut hasher);
-    let plan_hash = hasher.finish();
-
-    if let Some(progress) = &import_progress {
-        if plan_hash != progress.import_plan_hash {
-            anyhow::bail!("Import plan does not match storcon metadata");
-        }
-
-        // Handle collisions on jobs of unequal length
-        if progress.jobs != plan.jobs.len() {
-            anyhow::bail!("Import plan job length does not match storcon metadata")
-        }
-    }
-
-    pausable_failpoint!("import-timeline-pre-execute-pausable");
-
-    let start_from_job_idx = import_progress.map(|progress| progress.completed);
-    plan.execute(timeline, start_from_job_idx, plan_hash, import_config, ctx)
-        .await
+    plan.execute(timeline, import_config, ctx).await
 }
 
 struct Planner {
@@ -137,11 +89,8 @@ struct Planner {
     tasks: Vec<AnyImportTask>,
 }
 
-#[derive(Hash)]
 struct Plan {
     jobs: Vec<ChunkProcessingJob>,
-    // Included here such that it ends up in the hash for the plan
-    shard: ShardIdentity,
 }
 
 impl Planner {
@@ -245,10 +194,7 @@ impl Planner {
             pgdata_lsn,
         ));
 
-        Ok(Plan {
-            jobs,
-            shard: self.shard,
-        })
+        Ok(Plan { jobs })
     }
 
     #[instrument(level = tracing::Level::DEBUG, skip_all, fields(dboid=%db.dboid, tablespace=%db.spcnode, path=%db.path))]
@@ -377,45 +323,25 @@ impl Plan {
     async fn execute(
         self,
         timeline: Arc<Timeline>,
-        start_after_job_idx: Option<usize>,
-        import_plan_hash: u64,
         import_config: &TimelineImportConfig,
         ctx: &RequestContext,
     ) -> anyhow::Result<()> {
-        let storcon_client = StorageControllerUpcallClient::new(timeline.conf, &timeline.cancel);
-
         let mut work = FuturesOrdered::new();
         let semaphore = Arc::new(Semaphore::new(import_config.import_job_concurrency.into()));
 
         let jobs_in_plan = self.jobs.len();
 
-        let mut jobs = self
-            .jobs
-            .into_iter()
-            .enumerate()
-            .map(|(idx, job)| (idx + 1, job))
-            .filter(|(idx, _job)| {
-                // Filter out any jobs that have been done already
-                if let Some(start_after) = start_after_job_idx {
-                    *idx > start_after
-                } else {
-                    true
-                }
-            })
-            .peekable();
-
-        let mut last_completed_job_idx = start_after_job_idx.unwrap_or(0);
-        let checkpoint_every: usize = import_config.import_job_checkpoint_threshold.into();
+        let mut jobs = self.jobs.into_iter().enumerate().peekable();
+        let mut results = Vec::new();
 
         // Run import jobs concurrently up to the limit specified by the pageserver configuration.
         // Note that we process completed futures in the oreder of insertion. This will be the
         // building block for resuming imports across pageserver restarts or tenant migrations.
-        while last_completed_job_idx < jobs_in_plan {
+        while results.len() < jobs_in_plan {
             tokio::select! {
                 permit = semaphore.clone().acquire_owned(), if jobs.peek().is_some() => {
                     let permit = permit.expect("never closed");
                     let (job_idx, job) = jobs.next().expect("we peeked");
-
                     let job_timeline = timeline.clone();
                     let ctx = ctx.detached_child(TaskKind::ImportPgdata, DownloadBehavior::Error);
 
@@ -427,35 +353,13 @@ impl Plan {
                 },
                 maybe_complete_job_idx = work.next() => {
                     match maybe_complete_job_idx {
-                        Some(Ok((job_idx, res))) => {
-                            assert!(last_completed_job_idx.checked_add(1).unwrap() == job_idx);
-
-                            res?;
-                            last_completed_job_idx = job_idx;
-
-                            if last_completed_job_idx % checkpoint_every == 0 {
-                                let progress = ShardImportProgressV1 {
-                                    jobs: jobs_in_plan,
-                                    completed: last_completed_job_idx,
-                                    import_plan_hash,
-                                };
-
-                                storcon_client.put_timeline_import_status(
-                                    timeline.tenant_shard_id,
-                                    timeline.timeline_id,
-                                    timeline.generation,
-                                    ShardImportStatus::InProgress(Some(ShardImportProgress::V1(progress)))
-                                )
-                                .await
-                                .map_err(|_err| {
-                                    anyhow::anyhow!("Shut down while putting timeline import status")
-                                })?;
-                            }
+                        Some(Ok((_job_idx, res))) => {
+                            results.push(res);
                         },
                         Some(Err(_)) => {
-                            anyhow::bail!(
-                                "import job panicked or cancelled"
-                            );
+                            results.push(Err(anyhow::anyhow!(
+                                "parallel job panicked or cancelled, check pageserver logs"
+                            )));
                         }
                         None => {}
                     }
@@ -463,7 +367,17 @@ impl Plan {
             }
         }
 
-        Ok(())
+        if results.iter().all(|r| r.is_ok()) {
+            Ok(())
+        } else {
+            let mut msg = String::new();
+            for result in results {
+                if let Err(err) = result {
+                    msg.push_str(&format!("{err:?}\n\n"));
+                }
+            }
+            bail!("Some parallel jobs failed:\n\n{msg}");
+        }
     }
 }
 
@@ -635,15 +549,6 @@ struct ImportSingleKeyTask {
     buf: Bytes,
 }
 
-impl Hash for ImportSingleKeyTask {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let ImportSingleKeyTask { key, buf } = self;
-
-        key.hash(state);
-        buf.hash(state);
-    }
-}
-
 impl ImportSingleKeyTask {
     fn new(key: Key, buf: Bytes) -> Self {
         ImportSingleKeyTask { key, buf }
@@ -672,20 +577,6 @@ struct ImportRelBlocksTask {
     storage: RemoteStorageWrapper,
 }
 
-impl Hash for ImportRelBlocksTask {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let ImportRelBlocksTask {
-            shard_identity: _,
-            key_range,
-            path,
-            storage: _,
-        } = self;
-
-        key_range.hash(state);
-        path.hash(state);
-    }
-}
-
 impl ImportRelBlocksTask {
     fn new(
         shard_identity: ShardIdentity,
@@ -770,19 +661,6 @@ struct ImportSlruBlocksTask {
     storage: RemoteStorageWrapper,
 }
 
-impl Hash for ImportSlruBlocksTask {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        let ImportSlruBlocksTask {
-            key_range,
-            path,
-            storage: _,
-        } = self;
-
-        key_range.hash(state);
-        path.hash(state);
-    }
-}
-
 impl ImportSlruBlocksTask {
     fn new(key_range: Range<Key>, path: &RemotePath, storage: RemoteStorageWrapper) -> Self {
         ImportSlruBlocksTask {
@@ -825,7 +703,6 @@ impl ImportTask for ImportSlruBlocksTask {
     }
 }
 
-#[derive(Hash)]
 enum AnyImportTask {
     SingleKey(ImportSingleKeyTask),
     RelBlocks(ImportRelBlocksTask),
@@ -872,7 +749,6 @@ impl From<ImportSlruBlocksTask> for AnyImportTask {
     }
 }
 
-#[derive(Hash)]
 struct ChunkProcessingJob {
     range: Range<Key>,
     tasks: Vec<AnyImportTask>,
@@ -910,51 +786,17 @@ impl ChunkProcessingJob {
 
         let resident_layer = if nimages > 0 {
             let (desc, path) = writer.finish(ctx).await?;
-
-            {
-                let guard = timeline.layers.read().await;
-                let existing_layer = guard.try_get_from_key(&desc.key());
-                if let Some(layer) = existing_layer {
-                    if layer.metadata().generation != timeline.generation {
-                        return Err(anyhow::anyhow!(
-                            "Import attempted to rewrite layer file in the same generation: {}",
-                            layer.local_path()
-                        ));
-                    }
-                }
-            }
-
             Layer::finish_creating(timeline.conf, &timeline, desc, &path)?
         } else {
             // dropping the writer cleans up
             return Ok(());
         };
 
-        // The same import job might run multiple times since not each job is checkpointed.
-        // Hence, we must support the cases where the layer already exists. We cannot be
-        // certain that the existing layer is identical to the new one, so in that case
-        // we replace the old layer with the one we just generated.
-
+        // this is sharing the same code as create_image_layers
         let mut guard = timeline.layers.write().await;
-
-        let existing_layer = guard
-            .try_get_from_key(&resident_layer.layer_desc().key())
-            .cloned();
-        match existing_layer {
-            Some(existing) => {
-                guard.open_mut()?.rewrite_layers(
-                    &[(existing.clone(), resident_layer.clone())],
-                    &[],
-                    &timeline.metrics,
-                );
-            }
-            None => {
-                guard
-                    .open_mut()?
-                    .track_new_image_layers(&[resident_layer.clone()], &timeline.metrics);
-            }
-        }
-
+        guard
+            .open_mut()?
+            .track_new_image_layers(&[resident_layer.clone()], &timeline.metrics);
         crate::tenant::timeline::drop_wlock(guard);
 
         timeline

pageserver/src/tenant/timeline/import_pgdata/importbucket_client.rs

@@ -190,6 +190,31 @@ impl RemoteStorageWrapper {
         Ok(Some(res))
     }
 
+    #[instrument(level = tracing::Level::DEBUG, skip_all, fields(%path))]
+    pub async fn put_json<T>(&self, path: &RemotePath, value: &T) -> anyhow::Result<()>
+    where
+        T: serde::Serialize,
+    {
+        let buf = serde_json::to_vec(value)?;
+        let bytes = Bytes::from(buf);
+        utils::backoff::retry(
+            || async {
+                let size = bytes.len();
+                let bytes = futures::stream::once(futures::future::ready(Ok(bytes.clone())));
+                self.storage
+                    .upload_storage_object(bytes, size, path, &self.cancel)
+                    .await
+            },
+            remote_storage::TimeoutOrCancel::caused_by_cancel,
+            1,
+            u32::MAX,
+            &format!("put json {path}"),
+            &self.cancel,
+        )
+        .await
+        .expect("practically infinite retries")
+    }
+
     #[instrument(level = tracing::Level::DEBUG, skip_all, fields(%path))]
     pub async fn get_range(
         &self,

pageserver/src/tenant/timeline/import_pgdata/importbucket_format.rs

@@ -5,3 +5,9 @@ pub struct PgdataStatus {
     pub done: bool,
     // TODO: remaining fields
 }
+
+#[derive(Deserialize, Serialize, Debug, Clone, PartialEq, Eq)]
+pub struct ShardStatus {
+    pub done: bool,
+    // TODO: remaining fields
+}

pageserver/src/tenant/timeline/import_pgdata/index_part_format.rs

@@ -64,12 +64,4 @@ impl Root {
             },
         }
     }
-    pub fn started_at(&self) -> &chrono::NaiveDateTime {
-        match self {
-            Root::V1(v1) => match v1 {
-                V1::InProgress(in_progress) => &in_progress.started_at,
-                V1::Done(done) => &done.started_at,
-            },
-        }
-    }
 }

pageserver/src/virtual_file.rs

@@ -408,7 +408,7 @@ impl OpenFiles {
 /// error types may be elegible for retry.
 pub(crate) fn is_fatal_io_error(e: &std::io::Error) -> bool {
     use nix::errno::Errno::*;
-    match e.raw_os_error().map(nix::errno::Errno::from_raw) {
+    match e.raw_os_error().map(nix::errno::from_i32) {
         Some(EIO) => {
             // Terminate on EIO because we no longer trust the device to store
             // data safely, or to uphold persistence guarantees on fsync.

pageserver/src/virtual_file/io_engine.rs

@@ -124,7 +124,9 @@ pub(super) fn epoll_uring_error_to_std(
 ) -> std::io::Error {
     match e {
         tokio_epoll_uring::Error::Op(e) => e,
-        tokio_epoll_uring::Error::System(system) => std::io::Error::other(system),
+        tokio_epoll_uring::Error::System(system) => {
+            std::io::Error::new(std::io::ErrorKind::Other, system)
+        }
     }
 }
 

pgxn/neon/pagestore_smgr.c

@@ -1281,24 +1281,75 @@ neon_read_at_lsn(NRelFileInfo rinfo, ForkNumber forkNum, BlockNumber blkno,
 	communicator_read_at_lsnv(rinfo, forkNum, blkno, &request_lsns, &buffer, 1, NULL);
 }
 
-#ifdef DEBUG_COMPARE_LOCAL
+#if PG_MAJORVERSION_NUM < 17
+/*
+ *	neon_read() -- Read the specified block from a relation.
+ */
+#if PG_MAJORVERSION_NUM < 16
 static void
-compare_with_local(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void* buffer, XLogRecPtr request_lsn)
+neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, char *buffer)
+#else
+static void
+neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void *buffer)
+#endif
 {
+	neon_request_lsns request_lsns;
+	bits8		present;
+	void	   *bufferp;
+
+	switch (reln->smgr_relpersistence)
+	{
+		case 0:
+			neon_log(ERROR, "cannot call smgrread() on rel with unknown persistence");
+
+		case RELPERSISTENCE_PERMANENT:
+			break;
+
+		case RELPERSISTENCE_TEMP:
+		case RELPERSISTENCE_UNLOGGED:
+			mdread(reln, forkNum, blkno, buffer);
+			return;
+
+		default:
+			neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence);
+	}
+
+	/* Try to read PS results if they are available */
+	communicator_prefetch_pump_state();
+
+	neon_get_request_lsns(InfoFromSMgrRel(reln), forkNum, blkno, &request_lsns, 1);
+
+	present = 0;
+	bufferp = buffer;
+	if (communicator_prefetch_lookupv(InfoFromSMgrRel(reln), forkNum, blkno, &request_lsns, 1, &bufferp, &present))
+	{
+		/* Prefetch hit */
+		return;
+	}
+
+	/* Try to read from local file cache */
+	if (lfc_read(InfoFromSMgrRel(reln), forkNum, blkno, buffer))
+	{
+		MyNeonCounters->file_cache_hits_total++;
+		return;
+	}
+
+	neon_read_at_lsn(InfoFromSMgrRel(reln), forkNum, blkno, request_lsns, buffer);
+
+	/*
+	 * Try to receive prefetch results once again just to make sure we don't leave the smgr code while the OS might still have buffered bytes.
+	 */
+	communicator_prefetch_pump_state();
+
+#ifdef DEBUG_COMPARE_LOCAL
 	if (forkNum == MAIN_FORKNUM && IS_LOCAL_REL(reln))
 	{
 		char		pageserver_masked[BLCKSZ];
 		PGIOAlignedBlock mdbuf;
 		PGIOAlignedBlock mdbuf_masked;
+		XLogRecPtr  request_lsn = request_lsns.request_lsn;
 
-#if PG_MAJORVERSION_NUM >= 17
-		{
-			void* mdbuffers[1] = { mdbuf.data };
-			mdreadv(reln, forkNum, blkno, mdbuffers, 1);
-		}
-#else
 		mdread(reln, forkNum, blkno, mdbuf.data);
-#endif
 
 		memcpy(pageserver_masked, buffer, BLCKSZ);
 		memcpy(mdbuf_masked.data, mdbuf.data, BLCKSZ);
@@ -1362,105 +1413,11 @@ compare_with_local(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, voi
 			}
 		}
 	}
-}
-#endif
-
-
-#if PG_MAJORVERSION_NUM < 17
-
-/*
- *	neon_read() -- Read the specified block from a relation.
- */
-#if PG_MAJORVERSION_NUM < 16
-static void
-neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, char *buffer)
-#else
-static void
-neon_read(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void *buffer)
-#endif
-{
-	neon_request_lsns request_lsns;
-	bits8		present;
-	void	   *bufferp;
-
-	switch (reln->smgr_relpersistence)
-	{
-		case 0:
-			neon_log(ERROR, "cannot call smgrread() on rel with unknown persistence");
-
-		case RELPERSISTENCE_PERMANENT:
-			break;
-
-		case RELPERSISTENCE_TEMP:
-		case RELPERSISTENCE_UNLOGGED:
-			mdread(reln, forkNum, blkno, buffer);
-			return;
-
-		default:
-			neon_log(ERROR, "unknown relpersistence '%c'", reln->smgr_relpersistence);
-	}
-
-	/* Try to read PS results if they are available */
-	communicator_prefetch_pump_state();
-
-	neon_get_request_lsns(InfoFromSMgrRel(reln), forkNum, blkno, &request_lsns, 1);
-
-	present = 0;
-	bufferp = buffer;
-	if (communicator_prefetch_lookupv(InfoFromSMgrRel(reln), forkNum, blkno, &request_lsns, 1, &bufferp, &present))
-	{
-		/* Prefetch hit */
-#ifdef DEBUG_COMPARE_LOCAL
-		compare_with_local(reln, forkNum, blkno, buffer, request_lsns.request_lsn);
-#else
-		return;
-#endif
-	}
-
-	/* Try to read from local file cache */
-	if (lfc_read(InfoFromSMgrRel(reln), forkNum, blkno, buffer))
-	{
-		MyNeonCounters->file_cache_hits_total++;
-#ifdef DEBUG_COMPARE_LOCAL
-		compare_with_local(reln, forkNum, blkno, buffer, request_lsns.request_lsn);
-#else
-		return;
-#endif
-	}
-
-	neon_read_at_lsn(InfoFromSMgrRel(reln), forkNum, blkno, request_lsns, buffer);
-
-	/*
-	 * Try to receive prefetch results once again just to make sure we don't leave the smgr code while the OS might still have buffered bytes.
-	 */
-	communicator_prefetch_pump_state();
-
-#ifdef DEBUG_COMPARE_LOCAL
-	compare_with_local(reln, forkNum, blkno, buffer, request_lsns.request_lsn);
 #endif
 }
 #endif /* PG_MAJORVERSION_NUM <= 16 */
 
 #if PG_MAJORVERSION_NUM >= 17
-
-#ifdef DEBUG_COMPARE_LOCAL
-static void
-compare_with_localv(SMgrRelation reln, ForkNumber forkNum, BlockNumber blkno, void** buffers, BlockNumber nblocks, neon_request_lsns* request_lsns, bits8* read_pages)
-{
-	if (forkNum == MAIN_FORKNUM && IS_LOCAL_REL(reln))
-	{
-		for (BlockNumber i = 0; i < nblocks; i++)
-		{
-			if (BITMAP_ISSET(read_pages, i))
-			{
-				compare_with_local(reln, forkNum, blkno + i, buffers[i], request_lsns[i].request_lsn);
-			}
-		}
-	}
-}
-#endif
-
-
 static void
 neon_readv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
 		   void **buffers, BlockNumber nblocks)
@@ -1503,13 +1460,8 @@ neon_readv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
 													blocknum, request_lsns, nblocks,
 													buffers, read_pages);
 
-#ifdef DEBUG_COMPARE_LOCAL
-	compare_with_localv(reln, forknum, blocknum, buffers, nblocks, request_lsns, read_pages);
-	memset(read_pages, 0, sizeof(read_pages));
-#else
 	if (prefetch_result == nblocks)
 		return;
-#endif
 
 	/* Try to read from local file cache */
 	lfc_result = lfc_readv_select(InfoFromSMgrRel(reln), forknum, blocknum, buffers,
@@ -1518,14 +1470,9 @@ neon_readv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
 	if (lfc_result > 0)
 		MyNeonCounters->file_cache_hits_total += lfc_result;
 
-#ifdef DEBUG_COMPARE_LOCAL
-	compare_with_localv(reln, forknum, blocknum, buffers, nblocks, request_lsns, read_pages);
-	memset(read_pages, 0, sizeof(read_pages));
-#else
 	/* Read all blocks from LFC, so we're done */
 	if (prefetch_result + lfc_result == nblocks)
 		return;
-#endif
 
 	communicator_read_at_lsnv(InfoFromSMgrRel(reln), forknum, blocknum, request_lsns,
 							  buffers, nblocks, read_pages);
@@ -1536,8 +1483,91 @@ neon_readv(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum,
 	communicator_prefetch_pump_state();
 
 #ifdef DEBUG_COMPARE_LOCAL
-	memset(read_pages, 0xFF, sizeof(read_pages));
-	compare_with_localv(reln, forknum, blocknum, buffers, nblocks, request_lsns, read_pages);
+	if (forknum == MAIN_FORKNUM && IS_LOCAL_REL(reln))
+	{
+		char		pageserver_masked[BLCKSZ];
+		PGIOAlignedBlock mdbuf;
+		PGIOAlignedBlock mdbuf_masked;
+		XLogRecPtr  request_lsn = request_lsns->request_lsn;
+
+		for (int i = 0; i < nblocks; i++)
+		{
+			BlockNumber blkno = blocknum + i;
+			if (!BITMAP_ISSET(read_pages, i))
+				continue;
+
+#if PG_MAJORVERSION_NUM >= 17
+			{
+				void* mdbuffers[1] = { mdbuf.data };
+				mdreadv(reln, forknum, blkno, mdbuffers, 1);
+			}
+#else
+			mdread(reln, forknum, blkno, mdbuf.data);
+#endif
+
+			memcpy(pageserver_masked, buffers[i], BLCKSZ);
+			memcpy(mdbuf_masked.data, mdbuf.data, BLCKSZ);
+
+			if (PageIsNew((Page) mdbuf.data))
+			{
+				if (!PageIsNew((Page) pageserver_masked))
+				{
+					neon_log(PANIC, "page is new in MD but not in Page Server at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n",
+						 blkno,
+						 RelFileInfoFmt(InfoFromSMgrRel(reln)),
+						 forknum,
+						 (uint32) (request_lsn >> 32), (uint32) request_lsn,
+						 hexdump_page(buffers[i]));
+				}
+			}
+			else if (PageIsNew((Page) buffers[i]))
+			{
+				neon_log(PANIC, "page is new in Page Server but not in MD at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n%s\n",
+					 blkno,
+					 RelFileInfoFmt(InfoFromSMgrRel(reln)),
+					 forknum,
+					 (uint32) (request_lsn >> 32), (uint32) request_lsn,
+					 hexdump_page(mdbuf.data));
+			}
+			else if (PageGetSpecialSize(mdbuf.data) == 0)
+			{
+				/* assume heap */
+				RmgrTable[RM_HEAP_ID].rm_mask(mdbuf_masked.data, blkno);
+				RmgrTable[RM_HEAP_ID].rm_mask(pageserver_masked, blkno);
+
+				if (memcmp(mdbuf_masked.data, pageserver_masked, BLCKSZ) != 0)
+				{
+					neon_log(PANIC, "heap buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n",
+						 blkno,
+						 RelFileInfoFmt(InfoFromSMgrRel(reln)),
+						 forknum,
+						 (uint32) (request_lsn >> 32), (uint32) request_lsn,
+						 hexdump_page(mdbuf_masked.data),
+						 hexdump_page(pageserver_masked));
+				}
+			}
+			else if (PageGetSpecialSize(mdbuf.data) == MAXALIGN(sizeof(BTPageOpaqueData)))
+			{
+				if (((BTPageOpaqueData *) PageGetSpecialPointer(mdbuf.data))->btpo_cycleid < MAX_BT_CYCLE_ID)
+				{
+					/* assume btree */
+					RmgrTable[RM_BTREE_ID].rm_mask(mdbuf_masked.data, blkno);
+					RmgrTable[RM_BTREE_ID].rm_mask(pageserver_masked, blkno);
+	
+					if (memcmp(mdbuf_masked.data, pageserver_masked, BLCKSZ) != 0)
+					{
+						neon_log(PANIC, "btree buffers differ at blk %u in rel %u/%u/%u fork %u (request LSN %X/%08X):\n------ MD ------\n%s\n------ Page Server ------\n%s\n",
+							 blkno,
+							 RelFileInfoFmt(InfoFromSMgrRel(reln)),
+							 forknum,
+							 (uint32) (request_lsn >> 32), (uint32) request_lsn,
+							 hexdump_page(mdbuf_masked.data),
+							 hexdump_page(pageserver_masked));
+					}
+				}
+			}
+		}
+	}
 #endif
 }
 #endif

proxy/src/bin/proxy.rs

@@ -1,10 +1,6 @@
 #[global_allocator]
 static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
-#[allow(non_upper_case_globals)]
-#[unsafe(export_name = "malloc_conf")]
-pub static malloc_conf: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:21\0";
-
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
     proxy::binary::proxy::run().await

proxy/src/binary/pg_sni_router.rs

@@ -394,7 +394,6 @@ async fn handle_client(
     }
 }
 
-#[allow(clippy::large_enum_variant)]
 enum Connection {
     Raw(tokio::net::TcpStream),
     Tls(tokio_rustls::client::TlsStream<tokio::net::TcpStream>),

proxy/src/binary/proxy.rs

@@ -43,12 +43,11 @@ project_build_tag!(BUILD_TAG);
 use clap::{Parser, ValueEnum};
 
 #[derive(Clone, Debug, ValueEnum)]
-#[clap(rename_all = "kebab-case")]
 enum AuthBackendType {
-    #[clap(alias("cplane-v1"))]
-    ControlPlane,
+    #[value(name("cplane-v1"), alias("control-plane"))]
+    ControlPlaneV1,
 
-    #[clap(alias("link"))]
+    #[value(name("link"), alias("control-redirect"))]
     ConsoleRedirect,
 
     #[cfg(any(test, feature = "testing"))]
@@ -708,7 +707,7 @@ fn build_auth_backend(
     args: &ProxyCliArgs,
 ) -> anyhow::Result<Either<&'static auth::Backend<'static, ()>, &'static ConsoleRedirectBackend>> {
     match &args.auth_backend {
-        AuthBackendType::ControlPlane => {
+        AuthBackendType::ControlPlaneV1 => {
             let wake_compute_cache_config: CacheOptions = args.wake_compute_cache.parse()?;
             let project_info_cache_config: ProjectInfoCacheOptions =
                 args.project_info_cache.parse()?;
@@ -863,7 +862,7 @@ async fn configure_redis(
         ("irsa", _) => match (&args.redis_host, args.redis_port) {
             (Some(host), Some(port)) => Some(
                 ConnectionWithCredentialsProvider::new_with_credentials_provider(
-                    host.clone(),
+                    host.to_string(),
                     port,
                     elasticache::CredentialsProvider::new(
                         args.aws_region.clone(),

proxy/src/cancellation.rs

@@ -56,9 +56,11 @@ pub enum CancelKeyOp {
     },
 }
 
+type Callback = Box<dyn FnOnce(anyhow::Result<&[redis::Value]>) + Send>;
 pub struct Pipeline {
     inner: redis::Pipeline,
-    replies: Vec<CancelReplyOp>,
+    // vec![(number of commands, fn(values))]
+    replies: Vec<(usize, Callback)>,
 }
 
 impl Pipeline {
@@ -70,29 +72,36 @@ impl Pipeline {
     }
 
     async fn execute(&mut self, client: &mut RedisKVClient) {
-        let responses = self.replies.len();
-        let batch_size = self.inner.len();
+        let commands = self.inner.len();
+        let batch_size = self.replies.len();
 
         match client.query(&self.inner).await {
-            // for each reply, we expect that many values.
-            Ok(Value::Array(values)) if values.len() == responses => {
+            Ok(Value::Array(values)) if values.len() == commands => {
                 debug!(
-                    batch_size,
-                    responses, "successfully completed cancellation jobs",
+                    commands,
+                    batch_size, "successfully completed cancellation jobs",
                 );
-                for (value, reply) in std::iter::zip(values, self.replies.drain(..)) {
-                    reply.send_value(value);
+                let mut values = &*values;
+                for (n, resp) in self.replies.drain(..) {
+                    let (v, rest) = values.split_at(n);
+                    values = rest;
+                    resp(Ok(v));
                 }
             }
             Ok(value) => {
-                error!(batch_size, ?value, "unexpected redis return value");
-                for reply in self.replies.drain(..) {
-                    reply.send_err(anyhow!("incorrect response type from redis"));
+                error!(
+                    commands,
+                    batch_size,
+                    ?value,
+                    "unexpected redis return value"
+                );
+                for (_n, resp) in self.replies.drain(..) {
+                    resp(Err(anyhow!("incorrect response type from redis")));
                 }
             }
             Err(err) => {
-                for reply in self.replies.drain(..) {
-                    reply.send_err(anyhow!("could not send cmd to redis: {err}"));
+                for (_n, resp) in self.replies.drain(..) {
+                    resp(Err(anyhow!("could not send cmd to redis: {err}")));
                 }
             }
         }
@@ -101,20 +110,27 @@ impl Pipeline {
         self.replies.clear();
     }
 
-    fn add_command_with_reply(&mut self, cmd: Cmd, reply: CancelReplyOp) {
-        self.inner.add_command(cmd);
-        self.replies.push(reply);
-    }
-
-    fn add_command_no_reply(&mut self, cmd: Cmd) {
-        self.inner.add_command(cmd).ignore();
-    }
-
-    fn add_command(&mut self, cmd: Cmd, reply: Option<CancelReplyOp>) {
-        match reply {
-            Some(reply) => self.add_command_with_reply(cmd, reply),
-            None => self.add_command_no_reply(cmd),
+    /// Add a batch of commands to the pipeline, and run the resp fn when they are all done.
+    ///
+    /// If multiple commands are provided, the response should be able to decode
+    /// all of the values. You can provide a tuple in that case.
+    fn add_commands<F, T, const N: usize>(&mut self, cmds: [Cmd; N], resp: F)
+    where
+        F: FnOnce(anyhow::Result<T>) + Send + 'static,
+        T: FromRedisValue,
+    {
+        for cmd in cmds {
+            self.inner.add_command(cmd);
         }
+        let reply = Box::new(move |res: anyhow::Result<&[redis::Value]>| {
+            let res = match res {
+                Ok(v) => T::from_redis_value(&redis::Value::Array(v.to_owned()))
+                    .context("could not parse value"),
+                Err(e) => Err(e),
+            };
+            resp(res);
+        });
+        self.replies.push((N, reply as Box<_>));
     }
 }
 
@@ -130,18 +146,30 @@ impl CancelKeyOp {
                 _guard,
                 expire,
             } => {
-                let reply =
-                    resp_tx.map(|resp_tx| CancelReplyOp::StoreCancelKey { resp_tx, _guard });
-                pipe.add_command(Cmd::hset(&key, field, value), reply);
-                pipe.add_command_no_reply(Cmd::expire(key, expire));
+                pipe.add_commands(
+                    [Cmd::hset(&key, field, value), Cmd::expire(key, expire)],
+                    // ignore all results
+                    move |res: anyhow::Result<()>| {
+                        let _guard = _guard;
+                        if let Some(resp_tx) = resp_tx {
+                            if resp_tx.send(res).is_err() {
+                                tracing::debug!("could not send reply");
+                            }
+                        }
+                    },
+                );
             }
             CancelKeyOp::GetCancelData {
                 key,
                 resp_tx,
                 _guard,
             } => {
-                let reply = CancelReplyOp::GetCancelData { resp_tx, _guard };
-                pipe.add_command_with_reply(Cmd::hgetall(key), reply);
+                pipe.add_commands([Cmd::hgetall(key)], move |res| {
+                    let _guard = _guard;
+                    if resp_tx.send(res).is_err() {
+                        tracing::debug!("could not send reply");
+                    }
+                });
             }
             CancelKeyOp::RemoveCancelKey {
                 key,
@@ -149,79 +177,14 @@ impl CancelKeyOp {
                 resp_tx,
                 _guard,
             } => {
-                let reply =
-                    resp_tx.map(|resp_tx| CancelReplyOp::RemoveCancelKey { resp_tx, _guard });
-                pipe.add_command(Cmd::hdel(key, field), reply);
-            }
-        }
-    }
-}
-
-// Message types for sending through mpsc channel
-pub enum CancelReplyOp {
-    StoreCancelKey {
-        resp_tx: oneshot::Sender<anyhow::Result<()>>,
-        _guard: CancelChannelSizeGuard<'static>,
-    },
-    GetCancelData {
-        resp_tx: oneshot::Sender<anyhow::Result<Vec<(String, String)>>>,
-        _guard: CancelChannelSizeGuard<'static>,
-    },
-    RemoveCancelKey {
-        resp_tx: oneshot::Sender<anyhow::Result<()>>,
-        _guard: CancelChannelSizeGuard<'static>,
-    },
-}
-
-impl CancelReplyOp {
-    fn send_err(self, e: anyhow::Error) {
-        match self {
-            CancelReplyOp::StoreCancelKey { resp_tx, _guard } => {
-                resp_tx
-                    .send(Err(e))
-                    .inspect_err(|_| tracing::debug!("could not send reply"))
-                    .ok();
-            }
-            CancelReplyOp::GetCancelData { resp_tx, _guard } => {
-                resp_tx
-                    .send(Err(e))
-                    .inspect_err(|_| tracing::debug!("could not send reply"))
-                    .ok();
-            }
-            CancelReplyOp::RemoveCancelKey { resp_tx, _guard } => {
-                resp_tx
-                    .send(Err(e))
-                    .inspect_err(|_| tracing::debug!("could not send reply"))
-                    .ok();
-            }
-        }
-    }
-
-    fn send_value(self, v: redis::Value) {
-        match self {
-            CancelReplyOp::StoreCancelKey { resp_tx, _guard } => {
-                let send =
-                    FromRedisValue::from_owned_redis_value(v).context("could not parse value");
-                resp_tx
-                    .send(send)
-                    .inspect_err(|_| tracing::debug!("could not send reply"))
-                    .ok();
-            }
-            CancelReplyOp::GetCancelData { resp_tx, _guard } => {
-                let send =
-                    FromRedisValue::from_owned_redis_value(v).context("could not parse value");
-                resp_tx
-                    .send(send)
-                    .inspect_err(|_| tracing::debug!("could not send reply"))
-                    .ok();
-            }
-            CancelReplyOp::RemoveCancelKey { resp_tx, _guard } => {
-                let send =
-                    FromRedisValue::from_owned_redis_value(v).context("could not parse value");
-                resp_tx
-                    .send(send)
-                    .inspect_err(|_| tracing::debug!("could not send reply"))
-                    .ok();
+                pipe.add_commands([Cmd::hdel(key, field)], move |res| {
+                    let _guard = _guard;
+                    if let Some(resp_tx) = resp_tx {
+                        if resp_tx.send(res).is_err() {
+                            tracing::debug!("could not send reply");
+                        }
+                    }
+                });
             }
         }
     }

proxy/src/context/mod.rs

@@ -78,7 +78,7 @@ struct RequestContextInner {
 
 #[derive(Clone, Debug)]
 pub(crate) enum AuthMethod {
-    // aka link
+    // aka passwordless, fka link
     ConsoleRedirect,
     ScramSha256,
     ScramSha256Plus,

proxy/src/http/health_server.rs

@@ -3,7 +3,7 @@ use std::net::TcpListener;
 use std::sync::{Arc, Mutex};
 
 use anyhow::{anyhow, bail};
-use http_utils::endpoint::{self, profile_cpu_handler, profile_heap_handler, request_span};
+use http_utils::endpoint::{self, request_span};
 use http_utils::error::ApiError;
 use http_utils::json::json_response;
 use http_utils::{RouterBuilder, RouterService};
@@ -33,12 +33,6 @@ fn make_router(metrics: AppMetrics) -> RouterBuilder<hyper0::Body, ApiError> {
             request_span(r, move |b| prometheus_metrics_handler(b, state))
         })
         .get("/v1/status", status_handler)
-        .get("/profile/cpu", move |r| {
-            request_span(r, profile_cpu_handler)
-        })
-        .get("/profile/heap", move |r| {
-            request_span(r, profile_heap_handler)
-        })
 }
 
 pub async fn task_main(

rust-toolchain.toml

@@ -1,5 +1,5 @@
 [toolchain]
-channel = "1.87.0"
+channel = "1.86.0"
 profile = "default"
 # The default profile includes rustc, rust-std, cargo, rust-docs, rustfmt and clippy.
 # https://rust-lang.github.io/rustup/concepts/profiles.html

safekeeper/src/bin/safekeeper.rs

@@ -22,10 +22,9 @@ use safekeeper::defaults::{
     DEFAULT_PARTIAL_BACKUP_TIMEOUT, DEFAULT_PG_LISTEN_ADDR, DEFAULT_SSL_CERT_FILE,
     DEFAULT_SSL_CERT_RELOAD_PERIOD, DEFAULT_SSL_KEY_FILE,
 };
-use safekeeper::wal_backup::WalBackup;
 use safekeeper::{
     BACKGROUND_RUNTIME, BROKER_RUNTIME, GlobalTimelines, HTTP_RUNTIME, SafeKeeperConf,
-    WAL_SERVICE_RUNTIME, broker, control_file, http, wal_service,
+    WAL_SERVICE_RUNTIME, broker, control_file, http, wal_backup, wal_service,
 };
 use sd_notify::NotifyState;
 use storage_broker::{DEFAULT_ENDPOINT, Uri};
@@ -485,15 +484,15 @@ async fn start_safekeeper(conf: Arc<SafeKeeperConf>) -> Result<()> {
         None => None,
     };
 
-    let wal_backup = Arc::new(WalBackup::new(&conf).await?);
-
-    let global_timelines = Arc::new(GlobalTimelines::new(conf.clone(), wal_backup.clone()));
+    let global_timelines = Arc::new(GlobalTimelines::new(conf.clone()));
 
     // Register metrics collector for active timelines. It's important to do this
     // after daemonizing, otherwise process collector will be upset.
     let timeline_collector = safekeeper::metrics::TimelineCollector::new(global_timelines.clone());
     metrics::register_internal(Box::new(timeline_collector))?;
 
+    wal_backup::init_remote_storage(&conf).await;
+
     // Keep handles to main tasks to die if any of them disappears.
     let mut tasks_handles: FuturesUnordered<BoxFuture<(String, JoinTaskRes)>> =
         FuturesUnordered::new();

safekeeper/src/copy_timeline.rs

@@ -3,7 +3,6 @@ use std::sync::Arc;
 use anyhow::{Result, bail};
 use camino::Utf8PathBuf;
 use postgres_ffi::{MAX_SEND_SIZE, WAL_SEGMENT_SIZE};
-use remote_storage::GenericRemoteStorage;
 use safekeeper_api::membership::Configuration;
 use tokio::fs::OpenOptions;
 use tokio::io::{AsyncSeekExt, AsyncWriteExt};
@@ -31,7 +30,6 @@ pub struct Request {
 pub async fn handle_request(
     request: Request,
     global_timelines: Arc<GlobalTimelines>,
-    storage: Arc<GenericRemoteStorage>,
 ) -> Result<()> {
     // TODO: request.until_lsn MUST be a valid LSN, and we cannot check it :(
     //   if LSN will point to the middle of a WAL record, timeline will be in "broken" state
@@ -129,7 +127,6 @@ pub async fn handle_request(
     assert!(first_ondisk_segment >= first_segment);
 
     copy_s3_segments(
-        &storage,
         wal_seg_size,
         &request.source_ttid,
         &request.destination_ttid,

safekeeper/src/http/routes.rs

@@ -258,7 +258,6 @@ async fn timeline_snapshot_handler(request: Request<Body>) -> Result<Response<Bo
 
     let global_timelines = get_global_timelines(&request);
     let tli = global_timelines.get(ttid).map_err(ApiError::from)?;
-    let storage = global_timelines.get_wal_backup().get_storage();
 
     // To stream the body use wrap_stream which wants Stream of Result<Bytes>,
     // so create the chan and write to it in another task.
@@ -270,7 +269,6 @@ async fn timeline_snapshot_handler(request: Request<Body>) -> Result<Response<Bo
         conf.my_id,
         destination,
         tx,
-        storage,
     ));
 
     let rx_stream = ReceiverStream::new(rx);
@@ -392,18 +390,12 @@ async fn timeline_copy_handler(mut request: Request<Body>) -> Result<Response<Bo
     );
 
     let global_timelines = get_global_timelines(&request);
-    let wal_backup = global_timelines.get_wal_backup();
-    let storage = wal_backup
-        .get_storage()
-        .ok_or(ApiError::BadRequest(anyhow::anyhow!(
-            "Remote Storage is not configured"
-        )))?;
 
     copy_timeline::handle_request(copy_timeline::Request{
         source_ttid,
         until_lsn: request_data.until_lsn,
         destination_ttid: TenantTimelineId::new(source_ttid.tenant_id, request_data.target_timeline_id),
-    }, global_timelines, storage)
+    }, global_timelines)
         .instrument(info_span!("copy_timeline", from=%source_ttid, to=%request_data.target_timeline_id, until_lsn=%request_data.until_lsn))
         .await
         .map_err(ApiError::InternalServerError)?;

safekeeper/src/lib.rs

@@ -125,6 +125,12 @@ pub struct SafeKeeperConf {
     pub enable_tls_wal_service_api: bool,
 }
 
+impl SafeKeeperConf {
+    pub fn is_wal_backup_enabled(&self) -> bool {
+        self.remote_storage.is_some() && self.wal_backup_enabled
+    }
+}
+
 impl SafeKeeperConf {
     pub fn dummy() -> Self {
         SafeKeeperConf {

safekeeper/src/pull_timeline.rs

@@ -9,7 +9,6 @@ use chrono::{DateTime, Utc};
 use futures::{SinkExt, StreamExt, TryStreamExt};
 use http_utils::error::ApiError;
 use postgres_ffi::{PG_TLI, XLogFileName, XLogSegNo};
-use remote_storage::GenericRemoteStorage;
 use reqwest::Certificate;
 use safekeeper_api::Term;
 use safekeeper_api::models::{PullTimelineRequest, PullTimelineResponse, TimelineStatus};
@@ -44,7 +43,6 @@ pub async fn stream_snapshot(
     source: NodeId,
     destination: NodeId,
     tx: mpsc::Sender<Result<Bytes>>,
-    storage: Option<Arc<GenericRemoteStorage>>,
 ) {
     match tli.try_wal_residence_guard().await {
         Err(e) => {
@@ -55,32 +53,10 @@ pub async fn stream_snapshot(
         Ok(maybe_resident_tli) => {
             if let Err(e) = match maybe_resident_tli {
                 Some(resident_tli) => {
-                    stream_snapshot_resident_guts(
-                        resident_tli,
-                        source,
-                        destination,
-                        tx.clone(),
-                        storage,
-                    )
-                    .await
-                }
-                None => {
-                    if let Some(storage) = storage {
-                        stream_snapshot_offloaded_guts(
-                            tli,
-                            source,
-                            destination,
-                            tx.clone(),
-                            &storage,
-                        )
+                    stream_snapshot_resident_guts(resident_tli, source, destination, tx.clone())
                         .await
-                    } else {
-                        tx.send(Err(anyhow!("remote storage not configured")))
-                            .await
-                            .ok();
-                        return;
-                    }
                 }
+                None => stream_snapshot_offloaded_guts(tli, source, destination, tx.clone()).await,
             } {
                 // Error type/contents don't matter as they won't can't reach the client
                 // (hyper likely doesn't do anything with it), but http stream will be
@@ -147,12 +123,10 @@ pub(crate) async fn stream_snapshot_offloaded_guts(
     source: NodeId,
     destination: NodeId,
     tx: mpsc::Sender<Result<Bytes>>,
-    storage: &GenericRemoteStorage,
 ) -> Result<()> {
     let mut ar = prepare_tar_stream(tx);
 
-    tli.snapshot_offloaded(&mut ar, source, destination, storage)
-        .await?;
+    tli.snapshot_offloaded(&mut ar, source, destination).await?;
 
     ar.finish().await?;
 
@@ -165,13 +139,10 @@ pub async fn stream_snapshot_resident_guts(
     source: NodeId,
     destination: NodeId,
     tx: mpsc::Sender<Result<Bytes>>,
-    storage: Option<Arc<GenericRemoteStorage>>,
 ) -> Result<()> {
     let mut ar = prepare_tar_stream(tx);
 
-    let bctx = tli
-        .start_snapshot(&mut ar, source, destination, storage)
-        .await?;
+    let bctx = tli.start_snapshot(&mut ar, source, destination).await?;
     pausable_failpoint!("sk-snapshot-after-list-pausable");
 
     let tli_dir = tli.get_timeline_dir();
@@ -211,7 +182,6 @@ impl Timeline {
         ar: &mut tokio_tar::Builder<W>,
         source: NodeId,
         destination: NodeId,
-        storage: &GenericRemoteStorage,
     ) -> Result<()> {
         // Take initial copy of control file, then release state lock
         let mut control_file = {
@@ -246,7 +216,6 @@ impl Timeline {
         // can fail if the timeline was un-evicted and modified in the background.
         let remote_timeline_path = &self.remote_path;
         wal_backup::copy_partial_segment(
-            storage,
             &replace.previous.remote_path(remote_timeline_path),
             &replace.current.remote_path(remote_timeline_path),
         )
@@ -293,7 +262,6 @@ impl WalResidentTimeline {
         ar: &mut tokio_tar::Builder<W>,
         source: NodeId,
         destination: NodeId,
-        storage: Option<Arc<GenericRemoteStorage>>,
     ) -> Result<SnapshotContext> {
         let mut shared_state = self.write_shared_state().await;
         let wal_seg_size = shared_state.get_wal_seg_size();
@@ -315,7 +283,6 @@ impl WalResidentTimeline {
 
             let remote_timeline_path = &self.tli.remote_path;
             wal_backup::copy_partial_segment(
-                &*storage.context("remote storage not configured")?,
                 &replace.previous.remote_path(remote_timeline_path),
                 &replace.current.remote_path(remote_timeline_path),
             )

safekeeper/src/send_wal.rs

@@ -513,7 +513,7 @@ impl SafekeeperPostgresHandler {
         let end_pos = end_watch.get();
 
         if end_pos < start_pos {
-            info!(
+            warn!(
                 "requested start_pos {} is ahead of available WAL end_pos {}",
                 start_pos, end_pos
             );

safekeeper/src/test_utils.rs

@@ -18,7 +18,7 @@ use crate::send_wal::EndWatch;
 use crate::state::{TimelinePersistentState, TimelineState};
 use crate::timeline::{SharedState, StateSK, Timeline, get_timeline_dir};
 use crate::timelines_set::TimelinesSet;
-use crate::wal_backup::{WalBackup, remote_timeline_path};
+use crate::wal_backup::remote_timeline_path;
 use crate::{SafeKeeperConf, control_file, receive_wal, wal_storage};
 
 /// A Safekeeper testing or benchmarking environment. Uses a tempdir for storage, removed on drop.
@@ -101,22 +101,18 @@ impl Env {
         let safekeeper = self.make_safekeeper(node_id, ttid, start_lsn).await?;
         let shared_state = SharedState::new(StateSK::Loaded(safekeeper));
 
-        let wal_backup = Arc::new(WalBackup::new(&conf).await?);
-
         let timeline = Timeline::new(
             ttid,
             &timeline_dir,
             &remote_path,
             shared_state,
             conf.clone(),
-            wal_backup.clone(),
         );
         timeline.bootstrap(
             &mut timeline.write_shared_state().await,
             &conf,
             Arc::new(TimelinesSet::default()), // ignored for now
             RateLimiter::new(0, 0),
-            wal_backup,
         );
         Ok(timeline)
     }

safekeeper/src/timeline.rs

@@ -35,8 +35,7 @@ use crate::state::{EvictionState, TimelineMemState, TimelinePersistentState, Tim
 use crate::timeline_guard::ResidenceGuard;
 use crate::timeline_manager::{AtomicStatus, ManagerCtl};
 use crate::timelines_set::TimelinesSet;
-use crate::wal_backup;
-use crate::wal_backup::{WalBackup, remote_timeline_path};
+use crate::wal_backup::{self, remote_timeline_path};
 use crate::wal_backup_partial::PartialRemoteSegment;
 use crate::wal_storage::{Storage as wal_storage_iface, WalReader};
 use crate::{SafeKeeperConf, control_file, debug_dump, timeline_manager, wal_storage};
@@ -453,8 +452,6 @@ pub struct Timeline {
     manager_ctl: ManagerCtl,
     conf: Arc<SafeKeeperConf>,
 
-    pub(crate) wal_backup: Arc<WalBackup>,
-
     remote_deletion: std::sync::Mutex<Option<RemoteDeletionReceiver>>,
 
     /// Hold this gate from code that depends on the Timeline's non-shut-down state.  While holding
@@ -479,7 +476,6 @@ impl Timeline {
         remote_path: &RemotePath,
         shared_state: SharedState,
         conf: Arc<SafeKeeperConf>,
-        wal_backup: Arc<WalBackup>,
     ) -> Arc<Self> {
         let (commit_lsn_watch_tx, commit_lsn_watch_rx) =
             watch::channel(shared_state.sk.state().commit_lsn);
@@ -513,7 +509,6 @@ impl Timeline {
             wal_backup_active: AtomicBool::new(false),
             last_removed_segno: AtomicU64::new(0),
             mgr_status: AtomicStatus::new(),
-            wal_backup,
         })
     }
 
@@ -521,7 +516,6 @@ impl Timeline {
     pub fn load_timeline(
         conf: Arc<SafeKeeperConf>,
         ttid: TenantTimelineId,
-        wal_backup: Arc<WalBackup>,
     ) -> Result<Arc<Timeline>> {
         let _enter = info_span!("load_timeline", timeline = %ttid.timeline_id).entered();
 
@@ -535,7 +529,6 @@ impl Timeline {
             &remote_path,
             shared_state,
             conf,
-            wal_backup,
         ))
     }
 
@@ -546,7 +539,6 @@ impl Timeline {
         conf: &SafeKeeperConf,
         broker_active_set: Arc<TimelinesSet>,
         partial_backup_rate_limiter: RateLimiter,
-        wal_backup: Arc<WalBackup>,
     ) {
         let (tx, rx) = self.manager_ctl.bootstrap_manager();
 
@@ -569,7 +561,6 @@ impl Timeline {
                     tx,
                     rx,
                     partial_backup_rate_limiter,
-                    wal_backup,
                 )
                 .await
             }
@@ -615,10 +606,9 @@ impl Timeline {
         // it is cancelled, so WAL storage won't be opened again.
         shared_state.sk.close_wal_store();
 
-        if !only_local {
+        if !only_local && self.conf.is_wal_backup_enabled() {
             self.remote_delete().await?;
         }
-
         let dir_existed = delete_dir(&self.timeline_dir).await?;
         Ok(dir_existed)
     }
@@ -685,20 +675,11 @@ impl Timeline {
         guard: &mut std::sync::MutexGuard<Option<RemoteDeletionReceiver>>,
     ) -> RemoteDeletionReceiver {
         tracing::info!("starting remote deletion");
-        let storage = self.wal_backup.get_storage().clone();
         let (result_tx, result_rx) = tokio::sync::watch::channel(None);
         let ttid = self.ttid;
         tokio::task::spawn(
             async move {
-                let r = if let Some(storage) = storage {
-                    wal_backup::delete_timeline(&storage, &ttid).await
-                } else {
-                    tracing::info!(
-                        "skipping remote deletion because no remote storage is configured; this effectively leaks the objects in remote storage"
-                    );
-                    Ok(())
-                };
-
+                let r = wal_backup::delete_timeline(&ttid).await;
                 if let Err(e) = &r {
                     // Log error here in case nobody ever listens for our result (e.g. dropped API request)
                     tracing::error!("remote deletion failed: {e}");
@@ -1065,13 +1046,14 @@ impl WalResidentTimeline {
 
     pub async fn get_walreader(&self, start_lsn: Lsn) -> Result<WalReader> {
         let (_, persisted_state) = self.get_state().await;
+        let enable_remote_read = self.conf.is_wal_backup_enabled();
 
         WalReader::new(
             &self.ttid,
             self.timeline_dir.clone(),
             &persisted_state,
             start_lsn,
-            self.wal_backup.clone(),
+            enable_remote_read,
         )
     }
 

safekeeper/src/timeline_eviction.rs

@@ -6,7 +6,7 @@
 
 use anyhow::Context;
 use camino::Utf8PathBuf;
-use remote_storage::{GenericRemoteStorage, RemotePath};
+use remote_storage::RemotePath;
 use tokio::fs::File;
 use tokio::io::{AsyncRead, AsyncWriteExt};
 use tracing::{debug, info, instrument, warn};
@@ -68,10 +68,6 @@ impl Manager {
     #[instrument(name = "evict_timeline", skip_all)]
     pub(crate) async fn evict_timeline(&mut self) -> bool {
         assert!(!self.is_offloaded);
-        let Some(storage) = self.wal_backup.get_storage() else {
-            warn!("no remote storage configured, skipping uneviction");
-            return false;
-        };
         let partial_backup_uploaded = match &self.partial_backup_uploaded {
             Some(p) => p.clone(),
             None => {
@@ -91,7 +87,7 @@ impl Manager {
                 .inc();
         });
 
-        if let Err(e) = do_eviction(self, &partial_backup_uploaded, &storage).await {
+        if let Err(e) = do_eviction(self, &partial_backup_uploaded).await {
             warn!("failed to evict timeline: {:?}", e);
             return false;
         }
@@ -106,10 +102,6 @@ impl Manager {
     #[instrument(name = "unevict_timeline", skip_all)]
     pub(crate) async fn unevict_timeline(&mut self) {
         assert!(self.is_offloaded);
-        let Some(storage) = self.wal_backup.get_storage() else {
-            warn!("no remote storage configured, skipping uneviction");
-            return;
-        };
         let partial_backup_uploaded = match &self.partial_backup_uploaded {
             Some(p) => p.clone(),
             None => {
@@ -129,7 +121,7 @@ impl Manager {
                 .inc();
         });
 
-        if let Err(e) = do_uneviction(self, &partial_backup_uploaded, &storage).await {
+        if let Err(e) = do_uneviction(self, &partial_backup_uploaded).await {
             warn!("failed to unevict timeline: {:?}", e);
             return;
         }
@@ -145,12 +137,8 @@ impl Manager {
 /// Ensure that content matches the remote partial backup, if local segment exists.
 /// Then change state in control file and in-memory. If `delete_offloaded_wal` is set,
 /// delete the local segment.
-async fn do_eviction(
-    mgr: &mut Manager,
-    partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
-) -> anyhow::Result<()> {
-    compare_local_segment_with_remote(mgr, partial, storage).await?;
+async fn do_eviction(mgr: &mut Manager, partial: &PartialRemoteSegment) -> anyhow::Result<()> {
+    compare_local_segment_with_remote(mgr, partial).await?;
 
     mgr.tli.switch_to_offloaded(partial).await?;
     // switch manager state as soon as possible
@@ -165,16 +153,12 @@ async fn do_eviction(
 
 /// Ensure that content matches the remote partial backup, if local segment exists.
 /// Then download segment to local disk and change state in control file and in-memory.
-async fn do_uneviction(
-    mgr: &mut Manager,
-    partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
-) -> anyhow::Result<()> {
+async fn do_uneviction(mgr: &mut Manager, partial: &PartialRemoteSegment) -> anyhow::Result<()> {
     // if the local segment is present, validate it
-    compare_local_segment_with_remote(mgr, partial, storage).await?;
+    compare_local_segment_with_remote(mgr, partial).await?;
 
     // atomically download the partial segment
-    redownload_partial_segment(mgr, partial, storage).await?;
+    redownload_partial_segment(mgr, partial).await?;
 
     mgr.tli.switch_to_present().await?;
     // switch manager state as soon as possible
@@ -197,7 +181,6 @@ async fn delete_local_segment(mgr: &Manager, partial: &PartialRemoteSegment) ->
 async fn redownload_partial_segment(
     mgr: &Manager,
     partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
 ) -> anyhow::Result<()> {
     let tmp_file = mgr.tli.timeline_dir().join("remote_partial.tmp");
     let remote_segfile = remote_segment_path(mgr, partial);
@@ -207,7 +190,7 @@ async fn redownload_partial_segment(
         remote_segfile, tmp_file
     );
 
-    let mut reader = wal_backup::read_object(storage, &remote_segfile, 0).await?;
+    let mut reader = wal_backup::read_object(&remote_segfile, 0).await?;
     let mut file = File::create(&tmp_file).await?;
 
     let actual_len = tokio::io::copy(&mut reader, &mut file).await?;
@@ -251,16 +234,13 @@ async fn redownload_partial_segment(
 async fn compare_local_segment_with_remote(
     mgr: &Manager,
     partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
 ) -> anyhow::Result<()> {
     let local_path = local_segment_path(mgr, partial);
 
     match File::open(&local_path).await {
-        Ok(mut local_file) => {
-            do_validation(mgr, &mut local_file, mgr.wal_seg_size, partial, storage)
-                .await
-                .context("validation failed")
-        }
+        Ok(mut local_file) => do_validation(mgr, &mut local_file, mgr.wal_seg_size, partial)
+            .await
+            .context("validation failed"),
         Err(_) => {
             info!(
                 "local WAL file {} is not present, skipping validation",
@@ -278,7 +258,6 @@ async fn do_validation(
     file: &mut File,
     wal_seg_size: usize,
     partial: &PartialRemoteSegment,
-    storage: &GenericRemoteStorage,
 ) -> anyhow::Result<()> {
     let local_size = file.metadata().await?.len() as usize;
     if local_size != wal_seg_size {
@@ -291,7 +270,7 @@ async fn do_validation(
 
     let remote_segfile = remote_segment_path(mgr, partial);
     let mut remote_reader: std::pin::Pin<Box<dyn AsyncRead + Send + Sync>> =
-        wal_backup::read_object(storage, &remote_segfile, 0).await?;
+        wal_backup::read_object(&remote_segfile, 0).await?;
 
     // remote segment should have bytes excatly up to `flush_lsn`
     let expected_remote_size = partial.flush_lsn.segment_offset(mgr.wal_seg_size);

safekeeper/src/timeline_manager.rs

@@ -35,7 +35,7 @@ use crate::state::TimelineState;
 use crate::timeline::{ManagerTimeline, ReadGuardSharedState, StateSK, WalResidentTimeline};
 use crate::timeline_guard::{AccessService, GuardId, ResidenceGuard};
 use crate::timelines_set::{TimelineSetGuard, TimelinesSet};
-use crate::wal_backup::{self, WalBackup, WalBackupTaskHandle};
+use crate::wal_backup::{self, WalBackupTaskHandle};
 use crate::wal_backup_partial::{self, PartialBackup, PartialRemoteSegment};
 
 pub(crate) struct StateSnapshot {
@@ -200,7 +200,6 @@ pub(crate) struct Manager {
     pub(crate) conf: SafeKeeperConf,
     pub(crate) wal_seg_size: usize,
     pub(crate) walsenders: Arc<WalSenders>,
-    pub(crate) wal_backup: Arc<WalBackup>,
 
     // current state
     pub(crate) state_version_rx: tokio::sync::watch::Receiver<usize>,
@@ -239,7 +238,6 @@ pub async fn main_task(
     manager_tx: tokio::sync::mpsc::UnboundedSender<ManagerCtlMessage>,
     mut manager_rx: tokio::sync::mpsc::UnboundedReceiver<ManagerCtlMessage>,
     global_rate_limiter: RateLimiter,
-    wal_backup: Arc<WalBackup>,
 ) {
     tli.set_status(Status::Started);
 
@@ -258,7 +256,6 @@ pub async fn main_task(
         broker_active_set,
         manager_tx,
         global_rate_limiter,
-        wal_backup,
     )
     .await;
 
@@ -374,7 +371,7 @@ pub async fn main_task(
     mgr.tli_broker_active.set(false);
 
     // shutdown background tasks
-    if let Some(storage) = mgr.wal_backup.get_storage() {
+    if mgr.conf.is_wal_backup_enabled() {
         if let Some(backup_task) = mgr.backup_task.take() {
             // If we fell through here, then the timeline is shutting down. This is important
             // because otherwise joining on the wal_backup handle might hang.
@@ -382,7 +379,7 @@ pub async fn main_task(
 
             backup_task.join().await;
         }
-        wal_backup::update_task(&mut mgr, storage, false, &last_state).await;
+        wal_backup::update_task(&mut mgr, false, &last_state).await;
     }
 
     if let Some(recovery_task) = &mut mgr.recovery_task {
@@ -418,13 +415,11 @@ impl Manager {
         broker_active_set: Arc<TimelinesSet>,
         manager_tx: tokio::sync::mpsc::UnboundedSender<ManagerCtlMessage>,
         global_rate_limiter: RateLimiter,
-        wal_backup: Arc<WalBackup>,
     ) -> Manager {
         let (is_offloaded, partial_backup_uploaded) = tli.bootstrap_mgr().await;
         Manager {
             wal_seg_size: tli.get_wal_seg_size().await,
             walsenders: tli.get_walsenders().clone(),
-            wal_backup,
             state_version_rx: tli.get_state_version_rx(),
             num_computes_rx: tli.get_walreceivers().get_num_rx(),
             tli_broker_active: broker_active_set.guard(tli.clone()),
@@ -482,8 +477,8 @@ impl Manager {
         let is_wal_backup_required =
             wal_backup::is_wal_backup_required(self.wal_seg_size, num_computes, state);
 
-        if let Some(storage) = self.wal_backup.get_storage() {
-            wal_backup::update_task(self, storage, is_wal_backup_required, state).await;
+        if self.conf.is_wal_backup_enabled() {
+            wal_backup::update_task(self, is_wal_backup_required, state).await;
         }
 
         // update the state in Arc<Timeline>
@@ -629,9 +624,9 @@ impl Manager {
     /// Spawns partial WAL backup task if needed.
     async fn update_partial_backup(&mut self, state: &StateSnapshot) {
         // check if WAL backup is enabled and should be started
-        let Some(storage) = self.wal_backup.get_storage() else {
+        if !self.conf.is_wal_backup_enabled() {
             return;
-        };
+        }
 
         if self.partial_backup_task.is_some() {
             // partial backup is already running
@@ -655,7 +650,6 @@ impl Manager {
             self.conf.clone(),
             self.global_rate_limiter.clone(),
             cancel.clone(),
-            storage,
         ));
         self.partial_backup_task = Some((handle, cancel));
     }
@@ -675,10 +669,6 @@ impl Manager {
     /// Reset partial backup state and remove its remote storage data. Since it
     /// might concurrently uploading something, cancel the task first.
     async fn backup_partial_reset(&mut self) -> anyhow::Result<Vec<String>> {
-        let Some(storage) = self.wal_backup.get_storage() else {
-            anyhow::bail!("remote storage is not enabled");
-        };
-
         info!("resetting partial backup state");
         // Force unevict timeline if it is evicted before erasing partial backup
         // state. The intended use of this function is to drop corrupted remote
@@ -699,7 +689,7 @@ impl Manager {
         }
 
         let tli = self.wal_resident_timeline()?;
-        let mut partial_backup = PartialBackup::new(tli, self.conf.clone(), storage).await;
+        let mut partial_backup = PartialBackup::new(tli, self.conf.clone()).await;
         // Reset might fail e.g. when cfile is already reset but s3 removal
         // failed, so set manager state to None beforehand. In any case caller
         // is expected to retry until success.

safekeeper/src/timelines_global_map.rs

@@ -25,7 +25,6 @@ use crate::rate_limit::RateLimiter;
 use crate::state::TimelinePersistentState;
 use crate::timeline::{Timeline, TimelineError, delete_dir, get_tenant_dir, get_timeline_dir};
 use crate::timelines_set::TimelinesSet;
-use crate::wal_backup::WalBackup;
 use crate::wal_storage::Storage;
 use crate::{SafeKeeperConf, control_file, wal_storage};
 
@@ -48,24 +47,15 @@ struct GlobalTimelinesState {
     conf: Arc<SafeKeeperConf>,
     broker_active_set: Arc<TimelinesSet>,
     global_rate_limiter: RateLimiter,
-    wal_backup: Arc<WalBackup>,
 }
 
 impl GlobalTimelinesState {
     /// Get dependencies for a timeline constructor.
-    fn get_dependencies(
-        &self,
-    ) -> (
-        Arc<SafeKeeperConf>,
-        Arc<TimelinesSet>,
-        RateLimiter,
-        Arc<WalBackup>,
-    ) {
+    fn get_dependencies(&self) -> (Arc<SafeKeeperConf>, Arc<TimelinesSet>, RateLimiter) {
         (
             self.conf.clone(),
             self.broker_active_set.clone(),
             self.global_rate_limiter.clone(),
-            self.wal_backup.clone(),
         )
     }
 
@@ -94,7 +84,7 @@ pub struct GlobalTimelines {
 
 impl GlobalTimelines {
     /// Create a new instance of the global timelines map.
-    pub fn new(conf: Arc<SafeKeeperConf>, wal_backup: Arc<WalBackup>) -> Self {
+    pub fn new(conf: Arc<SafeKeeperConf>) -> Self {
         Self {
             state: Mutex::new(GlobalTimelinesState {
                 timelines: HashMap::new(),
@@ -102,7 +92,6 @@ impl GlobalTimelines {
                 conf,
                 broker_active_set: Arc::new(TimelinesSet::default()),
                 global_rate_limiter: RateLimiter::new(1, 1),
-                wal_backup,
             }),
         }
     }
@@ -158,7 +147,7 @@ impl GlobalTimelines {
     /// just lock and unlock it for each timeline -- this function is called
     /// during init when nothing else is running, so this is fine.
     async fn load_tenant_timelines(&self, tenant_id: TenantId) -> Result<()> {
-        let (conf, broker_active_set, partial_backup_rate_limiter, wal_backup) = {
+        let (conf, broker_active_set, partial_backup_rate_limiter) = {
             let state = self.state.lock().unwrap();
             state.get_dependencies()
         };
@@ -173,7 +162,7 @@ impl GlobalTimelines {
                         TimelineId::from_str(timeline_dir_entry.file_name().to_str().unwrap_or(""))
                     {
                         let ttid = TenantTimelineId::new(tenant_id, timeline_id);
-                        match Timeline::load_timeline(conf.clone(), ttid, wal_backup.clone()) {
+                        match Timeline::load_timeline(conf.clone(), ttid) {
                             Ok(tli) => {
                                 let mut shared_state = tli.write_shared_state().await;
                                 self.state
@@ -186,7 +175,6 @@ impl GlobalTimelines {
                                     &conf,
                                     broker_active_set.clone(),
                                     partial_backup_rate_limiter.clone(),
-                                    wal_backup.clone(),
                                 );
                             }
                             // If we can't load a timeline, it's most likely because of a corrupted
@@ -224,10 +212,6 @@ impl GlobalTimelines {
         self.state.lock().unwrap().broker_active_set.clone()
     }
 
-    pub fn get_wal_backup(&self) -> Arc<WalBackup> {
-        self.state.lock().unwrap().wal_backup.clone()
-    }
-
     /// Create a new timeline with the given id. If the timeline already exists, returns
     /// an existing timeline.
     pub(crate) async fn create(
@@ -238,7 +222,7 @@ impl GlobalTimelines {
         start_lsn: Lsn,
         commit_lsn: Lsn,
     ) -> Result<Arc<Timeline>> {
-        let (conf, _, _, _) = {
+        let (conf, _, _) = {
             let state = self.state.lock().unwrap();
             if let Ok(timeline) = state.get(&ttid) {
                 // Timeline already exists, return it.
@@ -283,7 +267,7 @@ impl GlobalTimelines {
         check_tombstone: bool,
     ) -> Result<Arc<Timeline>> {
         // Check for existence and mark that we're creating it.
-        let (conf, broker_active_set, partial_backup_rate_limiter, wal_backup) = {
+        let (conf, broker_active_set, partial_backup_rate_limiter) = {
             let mut state = self.state.lock().unwrap();
             match state.timelines.get(&ttid) {
                 Some(GlobalMapTimeline::CreationInProgress) => {
@@ -312,14 +296,7 @@ impl GlobalTimelines {
         };
 
         // Do the actual move and reflect the result in the map.
-        match GlobalTimelines::install_temp_timeline(
-            ttid,
-            tmp_path,
-            conf.clone(),
-            wal_backup.clone(),
-        )
-        .await
-        {
+        match GlobalTimelines::install_temp_timeline(ttid, tmp_path, conf.clone()).await {
             Ok(timeline) => {
                 let mut timeline_shared_state = timeline.write_shared_state().await;
                 let mut state = self.state.lock().unwrap();
@@ -337,7 +314,6 @@ impl GlobalTimelines {
                     &conf,
                     broker_active_set,
                     partial_backup_rate_limiter,
-                    wal_backup,
                 );
                 drop(timeline_shared_state);
                 Ok(timeline)
@@ -360,7 +336,6 @@ impl GlobalTimelines {
         ttid: TenantTimelineId,
         tmp_path: &Utf8PathBuf,
         conf: Arc<SafeKeeperConf>,
-        wal_backup: Arc<WalBackup>,
     ) -> Result<Arc<Timeline>> {
         let tenant_path = get_tenant_dir(conf.as_ref(), &ttid.tenant_id);
         let timeline_path = get_timeline_dir(conf.as_ref(), &ttid);
@@ -402,7 +377,7 @@ impl GlobalTimelines {
         // Do the move.
         durable_rename(tmp_path, &timeline_path, !conf.no_sync).await?;
 
-        Timeline::load_timeline(conf, ttid, wal_backup)
+        Timeline::load_timeline(conf, ttid)
     }
 
     /// Get a timeline from the global map. If it's not present, it doesn't exist on disk,

safekeeper/src/wal_backup.rs

@@ -2,7 +2,6 @@ use std::cmp::min;
 use std::collections::HashSet;
 use std::num::NonZeroU32;
 use std::pin::Pin;
-use std::sync::Arc;
 use std::time::Duration;
 
 use anyhow::{Context, Result};
@@ -18,7 +17,7 @@ use safekeeper_api::models::PeerInfo;
 use tokio::fs::File;
 use tokio::select;
 use tokio::sync::mpsc::{self, Receiver, Sender};
-use tokio::sync::watch;
+use tokio::sync::{OnceCell, watch};
 use tokio::task::JoinHandle;
 use tokio_util::sync::CancellationToken;
 use tracing::*;
@@ -64,12 +63,7 @@ pub(crate) fn is_wal_backup_required(
 /// Based on peer information determine which safekeeper should offload; if it
 /// is me, run (per timeline) task, if not yet. OTOH, if it is not me and task
 /// is running, kill it.
-pub(crate) async fn update_task(
-    mgr: &mut Manager,
-    storage: Arc<GenericRemoteStorage>,
-    need_backup: bool,
-    state: &StateSnapshot,
-) {
+pub(crate) async fn update_task(mgr: &mut Manager, need_backup: bool, state: &StateSnapshot) {
     let (offloader, election_dbg_str) =
         determine_offloader(&state.peers, state.backup_lsn, mgr.tli.ttid, &mgr.conf);
     let elected_me = Some(mgr.conf.my_id) == offloader;
@@ -88,12 +82,7 @@ pub(crate) async fn update_task(
                 return;
             };
 
-            let async_task = backup_task_main(
-                resident,
-                storage,
-                mgr.conf.backup_parallel_jobs,
-                shutdown_rx,
-            );
+            let async_task = backup_task_main(resident, mgr.conf.backup_parallel_jobs, shutdown_rx);
 
             let handle = if mgr.conf.current_thread_runtime {
                 tokio::spawn(async_task)
@@ -180,31 +169,33 @@ fn determine_offloader(
     }
 }
 
-pub struct WalBackup {
-    storage: Option<Arc<GenericRemoteStorage>>,
+static REMOTE_STORAGE: OnceCell<Option<GenericRemoteStorage>> = OnceCell::const_new();
+
+// Storage must be configured and initialized when this is called.
+fn get_configured_remote_storage() -> &'static GenericRemoteStorage {
+    REMOTE_STORAGE
+        .get()
+        .expect("failed to get remote storage")
+        .as_ref()
+        .unwrap()
 }
 
-impl WalBackup {
-    /// Create a new WalBackup instance.
-    pub async fn new(conf: &SafeKeeperConf) -> Result<Self> {
-        if !conf.wal_backup_enabled {
-            return Ok(Self { storage: None });
-        }
-
-        match conf.remote_storage.as_ref() {
-            Some(config) => {
-                let storage = GenericRemoteStorage::from_config(config).await?;
-                Ok(Self {
-                    storage: Some(Arc::new(storage)),
-                })
+pub async fn init_remote_storage(conf: &SafeKeeperConf) {
+    // TODO: refactor REMOTE_STORAGE to avoid using global variables, and provide
+    // dependencies to all tasks instead.
+    REMOTE_STORAGE
+        .get_or_init(|| async {
+            if let Some(conf) = conf.remote_storage.as_ref() {
+                Some(
+                    GenericRemoteStorage::from_config(conf)
+                        .await
+                        .expect("failed to create remote storage"),
+                )
+            } else {
+                None
             }
-            None => Ok(Self { storage: None }),
-        }
-    }
-
-    pub fn get_storage(&self) -> Option<Arc<GenericRemoteStorage>> {
-        self.storage.clone()
-    }
+        })
+        .await;
 }
 
 struct WalBackupTask {
@@ -213,14 +204,12 @@ struct WalBackupTask {
     wal_seg_size: usize,
     parallel_jobs: usize,
     commit_lsn_watch_rx: watch::Receiver<Lsn>,
-    storage: Arc<GenericRemoteStorage>,
 }
 
 /// Offload single timeline.
 #[instrument(name = "wal_backup", skip_all, fields(ttid = %tli.ttid))]
 async fn backup_task_main(
     tli: WalResidentTimeline,
-    storage: Arc<GenericRemoteStorage>,
     parallel_jobs: usize,
     mut shutdown_rx: Receiver<()>,
 ) {
@@ -234,7 +223,6 @@ async fn backup_task_main(
         timeline_dir: tli.get_timeline_dir(),
         timeline: tli,
         parallel_jobs,
-        storage,
     };
 
     // task is spinned up only when wal_seg_size already initialized
@@ -305,7 +293,6 @@ impl WalBackupTask {
 
             match backup_lsn_range(
                 &self.timeline,
-                self.storage.clone(),
                 &mut backup_lsn,
                 commit_lsn,
                 self.wal_seg_size,
@@ -335,7 +322,6 @@ impl WalBackupTask {
 
 async fn backup_lsn_range(
     timeline: &WalResidentTimeline,
-    storage: Arc<GenericRemoteStorage>,
     backup_lsn: &mut Lsn,
     end_lsn: Lsn,
     wal_seg_size: usize,
@@ -366,12 +352,7 @@ async fn backup_lsn_range(
     loop {
         let added_task = match iter.next() {
             Some(s) => {
-                uploads.push_back(backup_single_segment(
-                    &storage,
-                    s,
-                    timeline_dir,
-                    remote_timeline_path,
-                ));
+                uploads.push_back(backup_single_segment(s, timeline_dir, remote_timeline_path));
                 true
             }
             None => false,
@@ -407,7 +388,6 @@ async fn backup_lsn_range(
 }
 
 async fn backup_single_segment(
-    storage: &GenericRemoteStorage,
     seg: &Segment,
     timeline_dir: &Utf8Path,
     remote_timeline_path: &RemotePath,
@@ -415,13 +395,7 @@ async fn backup_single_segment(
     let segment_file_path = seg.file_path(timeline_dir)?;
     let remote_segment_path = seg.remote_path(remote_timeline_path);
 
-    let res = backup_object(
-        storage,
-        &segment_file_path,
-        &remote_segment_path,
-        seg.size(),
-    )
-    .await;
+    let res = backup_object(&segment_file_path, &remote_segment_path, seg.size()).await;
     if res.is_ok() {
         BACKED_UP_SEGMENTS.inc();
     } else {
@@ -481,11 +455,12 @@ fn get_segments(start: Lsn, end: Lsn, seg_size: usize) -> Vec<Segment> {
 }
 
 async fn backup_object(
-    storage: &GenericRemoteStorage,
     source_file: &Utf8Path,
     target_file: &RemotePath,
     size: usize,
 ) -> Result<()> {
+    let storage = get_configured_remote_storage();
+
     let file = File::open(&source_file)
         .await
         .with_context(|| format!("Failed to open file {source_file:?} for wal backup"))?;
@@ -500,11 +475,12 @@ async fn backup_object(
 }
 
 pub(crate) async fn backup_partial_segment(
-    storage: &GenericRemoteStorage,
     source_file: &Utf8Path,
     target_file: &RemotePath,
     size: usize,
 ) -> Result<()> {
+    let storage = get_configured_remote_storage();
+
     let file = File::open(&source_file)
         .await
         .with_context(|| format!("Failed to open file {source_file:?} for wal backup"))?;
@@ -528,20 +504,25 @@ pub(crate) async fn backup_partial_segment(
 }
 
 pub(crate) async fn copy_partial_segment(
-    storage: &GenericRemoteStorage,
     source: &RemotePath,
     destination: &RemotePath,
 ) -> Result<()> {
+    let storage = get_configured_remote_storage();
     let cancel = CancellationToken::new();
 
     storage.copy_object(source, destination, &cancel).await
 }
 
 pub async fn read_object(
-    storage: &GenericRemoteStorage,
     file_path: &RemotePath,
     offset: u64,
 ) -> anyhow::Result<Pin<Box<dyn tokio::io::AsyncRead + Send + Sync>>> {
+    let storage = REMOTE_STORAGE
+        .get()
+        .context("Failed to get remote storage")?
+        .as_ref()
+        .context("No remote storage configured")?;
+
     info!("segment download about to start from remote path {file_path:?} at offset {offset}");
 
     let cancel = CancellationToken::new();
@@ -566,10 +547,8 @@ pub async fn read_object(
 
 /// Delete WAL files for the given timeline. Remote storage must be configured
 /// when called.
-pub async fn delete_timeline(
-    storage: &GenericRemoteStorage,
-    ttid: &TenantTimelineId,
-) -> Result<()> {
+pub async fn delete_timeline(ttid: &TenantTimelineId) -> Result<()> {
+    let storage = get_configured_remote_storage();
     let remote_path = remote_timeline_path(ttid)?;
 
     // see DEFAULT_MAX_KEYS_PER_LIST_RESPONSE
@@ -639,14 +618,14 @@ pub async fn delete_timeline(
 }
 
 /// Used by wal_backup_partial.
-pub async fn delete_objects(storage: &GenericRemoteStorage, paths: &[RemotePath]) -> Result<()> {
+pub async fn delete_objects(paths: &[RemotePath]) -> Result<()> {
     let cancel = CancellationToken::new(); // not really used
+    let storage = get_configured_remote_storage();
     storage.delete_objects(paths, &cancel).await
 }
 
 /// Copy segments from one timeline to another. Used in copy_timeline.
 pub async fn copy_s3_segments(
-    storage: &GenericRemoteStorage,
     wal_seg_size: usize,
     src_ttid: &TenantTimelineId,
     dst_ttid: &TenantTimelineId,
@@ -655,6 +634,12 @@ pub async fn copy_s3_segments(
 ) -> Result<()> {
     const SEGMENTS_PROGRESS_REPORT_INTERVAL: u64 = 1024;
 
+    let storage = REMOTE_STORAGE
+        .get()
+        .expect("failed to get remote storage")
+        .as_ref()
+        .unwrap();
+
     let remote_dst_path = remote_timeline_path(dst_ttid)?;
 
     let cancel = CancellationToken::new();

safekeeper/src/wal_backup_partial.rs

@@ -19,11 +19,9 @@
 //! file. Code updates state in the control file before doing any S3 operations.
 //! This way control file stores information about all potentially existing
 //! remote partial segments and can clean them up after uploading a newer version.
-use std::sync::Arc;
-
 use camino::Utf8PathBuf;
 use postgres_ffi::{PG_TLI, XLogFileName, XLogSegNo};
-use remote_storage::{GenericRemoteStorage, RemotePath};
+use remote_storage::RemotePath;
 use safekeeper_api::Term;
 use serde::{Deserialize, Serialize};
 use tokio_util::sync::CancellationToken;
@@ -156,16 +154,12 @@ pub struct PartialBackup {
     conf: SafeKeeperConf,
     local_prefix: Utf8PathBuf,
     remote_timeline_path: RemotePath,
-    storage: Arc<GenericRemoteStorage>,
+
     state: State,
 }
 
 impl PartialBackup {
-    pub async fn new(
-        tli: WalResidentTimeline,
-        conf: SafeKeeperConf,
-        storage: Arc<GenericRemoteStorage>,
-    ) -> PartialBackup {
+    pub async fn new(tli: WalResidentTimeline, conf: SafeKeeperConf) -> PartialBackup {
         let (_, persistent_state) = tli.get_state().await;
         let wal_seg_size = tli.get_wal_seg_size().await;
 
@@ -179,7 +173,6 @@ impl PartialBackup {
             conf,
             local_prefix,
             remote_timeline_path,
-            storage,
         }
     }
 
@@ -247,8 +240,7 @@ impl PartialBackup {
         let remote_path = prepared.remote_path(&self.remote_timeline_path);
 
         // Upload first `backup_bytes` bytes of the segment to the remote storage.
-        wal_backup::backup_partial_segment(&self.storage, &local_path, &remote_path, backup_bytes)
-            .await?;
+        wal_backup::backup_partial_segment(&local_path, &remote_path, backup_bytes).await?;
         PARTIAL_BACKUP_UPLOADED_BYTES.inc_by(backup_bytes as u64);
 
         // We uploaded the segment, now let's verify that the data is still actual.
@@ -334,7 +326,7 @@ impl PartialBackup {
             let remote_path = self.remote_timeline_path.join(seg);
             objects_to_delete.push(remote_path);
         }
-        wal_backup::delete_objects(&self.storage, &objects_to_delete).await
+        wal_backup::delete_objects(&objects_to_delete).await
     }
 
     /// Delete all non-Uploaded segments from the remote storage. There should be only one
@@ -432,7 +424,6 @@ pub async fn main_task(
     conf: SafeKeeperConf,
     limiter: RateLimiter,
     cancel: CancellationToken,
-    storage: Arc<GenericRemoteStorage>,
 ) -> Option<PartialRemoteSegment> {
     debug!("started");
     let await_duration = conf.partial_backup_timeout;
@@ -441,7 +432,7 @@ pub async fn main_task(
     let mut commit_lsn_rx = tli.get_commit_lsn_watch_rx();
     let mut flush_lsn_rx = tli.get_term_flush_lsn_watch_rx();
 
-    let mut backup = PartialBackup::new(tli, conf, storage).await;
+    let mut backup = PartialBackup::new(tli, conf).await;
 
     debug!("state: {:?}", backup.state);
 

safekeeper/src/wal_storage.rs

@@ -21,7 +21,6 @@ use postgres_ffi::waldecoder::WalStreamDecoder;
 use postgres_ffi::{PG_TLI, XLogFileName, XLogSegNo, dispatch_pgversion};
 use pq_proto::SystemId;
 use remote_storage::RemotePath;
-use std::sync::Arc;
 use tokio::fs::{self, File, OpenOptions, remove_file};
 use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt, AsyncWriteExt};
 use tracing::*;
@@ -33,7 +32,7 @@ use crate::metrics::{
     REMOVED_WAL_SEGMENTS, WAL_STORAGE_OPERATION_SECONDS, WalStorageMetrics, time_io_closure,
 };
 use crate::state::TimelinePersistentState;
-use crate::wal_backup::{WalBackup, read_object, remote_timeline_path};
+use crate::wal_backup::{read_object, remote_timeline_path};
 
 pub trait Storage {
     // Last written LSN.
@@ -646,7 +645,7 @@ pub struct WalReader {
     wal_segment: Option<Pin<Box<dyn AsyncRead + Send + Sync>>>,
 
     // S3 will be used to read WAL if LSN is not available locally
-    wal_backup: Arc<WalBackup>,
+    enable_remote_read: bool,
 
     // We don't have WAL locally if LSN is less than local_start_lsn
     local_start_lsn: Lsn,
@@ -665,7 +664,7 @@ impl WalReader {
         timeline_dir: Utf8PathBuf,
         state: &TimelinePersistentState,
         start_pos: Lsn,
-        wal_backup: Arc<WalBackup>,
+        enable_remote_read: bool,
     ) -> Result<Self> {
         if state.server.wal_seg_size == 0 || state.local_start_lsn == Lsn(0) {
             bail!("state uninitialized, no data to read");
@@ -694,7 +693,7 @@ impl WalReader {
             wal_seg_size: state.server.wal_seg_size as usize,
             pos: start_pos,
             wal_segment: None,
-            wal_backup,
+            enable_remote_read,
             local_start_lsn: state.local_start_lsn,
             timeline_start_lsn: state.timeline_start_lsn,
             pg_version: state.server.pg_version / 10000,
@@ -813,9 +812,9 @@ impl WalReader {
         }
 
         // Try to open remote file, if remote reads are enabled
-        if let Some(storage) = self.wal_backup.get_storage() {
+        if self.enable_remote_read {
             let remote_wal_file_path = self.remote_path.join(&wal_file_name);
-            return read_object(&storage, &remote_wal_file_path, xlogoff as u64).await;
+            return read_object(&remote_wal_file_path, xlogoff as u64).await;
         }
 
         bail!("WAL segment is not found")

storage_controller/src/http.rs

@@ -31,7 +31,7 @@ use pageserver_api::models::{
 };
 use pageserver_api::shard::TenantShardId;
 use pageserver_api::upcall_api::{
-    PutTimelineImportStatusRequest, ReAttachRequest, TimelineImportStatusRequest, ValidateRequest,
+    PutTimelineImportStatusRequest, ReAttachRequest, ValidateRequest,
 };
 use pageserver_client::{BlockUnblock, mgmt_api};
 use routerify::Middleware;
@@ -157,29 +157,6 @@ async fn handle_validate(req: Request<Body>) -> Result<Response<Body>, ApiError>
     json_response(StatusCode::OK, state.service.validate(validate_req).await?)
 }
 
-async fn handle_get_timeline_import_status(req: Request<Body>) -> Result<Response<Body>, ApiError> {
-    check_permissions(&req, Scope::GenerationsApi)?;
-
-    let mut req = match maybe_forward(req).await {
-        ForwardOutcome::Forwarded(res) => {
-            return res;
-        }
-        ForwardOutcome::NotForwarded(req) => req,
-    };
-
-    let get_req = json_request::<TimelineImportStatusRequest>(&mut req).await?;
-
-    let state = get_state(&req);
-
-    json_response(
-        StatusCode::OK,
-        state
-            .service
-            .handle_timeline_shard_import_progress(get_req)
-            .await?,
-    )
-}
-
 async fn handle_put_timeline_import_status(req: Request<Body>) -> Result<Response<Body>, ApiError> {
     check_permissions(&req, Scope::GenerationsApi)?;
 
@@ -2031,13 +2008,6 @@ pub fn make_router(
         .post("/upcall/v1/validate", |r| {
             named_request_span(r, handle_validate, RequestName("upcall_v1_validate"))
         })
-        .get("/upcall/v1/timeline_import_status", |r| {
-            named_request_span(
-                r,
-                handle_get_timeline_import_status,
-                RequestName("upcall_v1_timeline_import_status"),
-            )
-        })
         .post("/upcall/v1/timeline_import_status", |r| {
             named_request_span(
                 r,

storage_controller/src/pageserver_client.rs

@@ -1,5 +1,3 @@
-use std::time::Duration;
-
 use pageserver_api::models::detach_ancestor::AncestorDetached;
 use pageserver_api::models::{
     DetachBehavior, LocationConfig, LocationConfigListResponse, LsnLease, PageserverUtilization,
@@ -214,7 +212,6 @@ impl PageserverClient {
         )
     }
 
-    #[allow(unused)]
     pub(crate) async fn timeline_detail(
         &self,
         tenant_shard_id: TenantShardId,
@@ -360,20 +357,4 @@ impl PageserverClient {
             self.inner.wait_lsn(tenant_shard_id, request).await
         )
     }
-
-    pub(crate) async fn activate_post_import(
-        &self,
-        tenant_shard_id: TenantShardId,
-        timeline_id: TimelineId,
-        timeline_activate_timeout: Duration,
-    ) -> Result<TimelineInfo> {
-        measured_request!(
-            "activate_post_import",
-            crate::metrics::Method::Put,
-            &self.node_id_label,
-            self.inner
-                .activate_post_import(tenant_shard_id, timeline_id, timeline_activate_timeout)
-                .await
-        )
-    }
 }

storage_controller/src/persistence.rs

@@ -1666,39 +1666,6 @@ impl Persistence {
         }
     }
 
-    pub(crate) async fn get_timeline_import(
-        &self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-    ) -> DatabaseResult<Option<TimelineImport>> {
-        use crate::schema::timeline_imports::dsl;
-        let persistent_import = self
-            .with_measured_conn(DatabaseOperation::ListTimelineImports, move |conn| {
-                Box::pin(async move {
-                    let mut from_db: Vec<TimelineImportPersistence> = dsl::timeline_imports
-                        .filter(dsl::tenant_id.eq(tenant_id.to_string()))
-                        .filter(dsl::timeline_id.eq(timeline_id.to_string()))
-                        .load(conn)
-                        .await?;
-
-                    if from_db.len() > 1 {
-                        return Err(DatabaseError::Logical(format!(
-                            "unexpected number of rows ({})",
-                            from_db.len()
-                        )));
-                    }
-
-                    Ok(from_db.pop())
-                })
-            })
-            .await?;
-
-        persistent_import
-            .map(TimelineImport::from_persistent)
-            .transpose()
-            .map_err(|err| DatabaseError::Logical(format!("failed to deserialize import: {err}")))
-    }
-
     pub(crate) async fn delete_timeline_import(
         &self,
         tenant_id: TenantId,

storage_controller/src/scheduler.rs

@@ -628,7 +628,11 @@ impl Scheduler {
             tracing::trace!(%node_id, "attached_shard_count={} shard_count={} expected={}", node.attached_shard_count, node.shard_count, expected_attached_shards_per_node);
         }
 
-        expected_attached_shards_per_node.saturating_sub(node.attached_shard_count)
+        if node.attached_shard_count < expected_attached_shards_per_node {
+            expected_attached_shards_per_node - node.attached_shard_count
+        } else {
+            0
+        }
     }
 
     pub(crate) fn expected_attached_shard_count(&self) -> usize {

storage_controller/src/service.rs

@@ -35,19 +35,19 @@ use pageserver_api::controller_api::{
 };
 use pageserver_api::models::{
     self, DetachBehavior, LocationConfig, LocationConfigListResponse, LocationConfigMode, LsnLease,
-    PageserverUtilization, SecondaryProgress, ShardImportStatus, ShardParameters, TenantConfig,
+    PageserverUtilization, SecondaryProgress, ShardParameters, TenantConfig,
     TenantConfigPatchRequest, TenantConfigRequest, TenantLocationConfigRequest,
     TenantLocationConfigResponse, TenantShardLocation, TenantShardSplitRequest,
     TenantShardSplitResponse, TenantSorting, TenantTimeTravelRequest,
     TimelineArchivalConfigRequest, TimelineCreateRequest, TimelineCreateResponseStorcon,
-    TimelineInfo, TopTenantShardItem, TopTenantShardsRequest,
+    TimelineInfo, TimelineState, TopTenantShardItem, TopTenantShardsRequest,
 };
 use pageserver_api::shard::{
     DEFAULT_STRIPE_SIZE, ShardCount, ShardIdentity, ShardNumber, ShardStripeSize, TenantShardId,
 };
 use pageserver_api::upcall_api::{
     PutTimelineImportStatusRequest, ReAttachRequest, ReAttachResponse, ReAttachResponseTenant,
-    TimelineImportStatusRequest, ValidateRequest, ValidateResponse, ValidateResponseTenant,
+    ValidateRequest, ValidateResponse, ValidateResponseTenant,
 };
 use pageserver_client::{BlockUnblock, mgmt_api};
 use reqwest::{Certificate, StatusCode};
@@ -61,7 +61,6 @@ use utils::completion::Barrier;
 use utils::generation::Generation;
 use utils::id::{NodeId, TenantId, TimelineId};
 use utils::lsn::Lsn;
-use utils::shard::ShardIndex;
 use utils::sync::gate::{Gate, GateGuard};
 use utils::{failpoint_support, pausable_failpoint};
 
@@ -99,8 +98,7 @@ use crate::tenant_shard::{
     ScheduleOptimization, ScheduleOptimizationAction, TenantShard,
 };
 use crate::timeline_import::{
-    ImportResult, ShardImportStatuses, TimelineImport, TimelineImportFinalizeError,
-    TimelineImportState, UpcallClient,
+    ShardImportStatuses, TimelineImport, TimelineImportState, UpcallClient,
 };
 
 const WAITER_FILL_DRAIN_POLL_TIMEOUT: Duration = Duration::from_millis(500);
@@ -194,14 +192,6 @@ pub(crate) enum LeadershipStatus {
     Candidate,
 }
 
-enum ShardGenerationValidity {
-    Valid,
-    Mismatched {
-        claimed: Generation,
-        actual: Option<Generation>,
-    },
-}
-
 pub const RECONCILER_CONCURRENCY_DEFAULT: usize = 128;
 pub const PRIORITY_RECONCILER_CONCURRENCY_DEFAULT: usize = 256;
 pub const SAFEKEEPER_RECONCILER_CONCURRENCY_DEFAULT: usize = 32;
@@ -3915,77 +3905,10 @@ impl Service {
         })
     }
 
-    pub(crate) async fn handle_timeline_shard_import_progress(
-        self: &Arc<Self>,
-        req: TimelineImportStatusRequest,
-    ) -> Result<ShardImportStatus, ApiError> {
-        let validity = self
-            .validate_shard_generation(req.tenant_shard_id, req.generation)
-            .await?;
-        match validity {
-            ShardGenerationValidity::Valid => {
-                // fallthrough
-            }
-            ShardGenerationValidity::Mismatched { claimed, actual } => {
-                tracing::info!(
-                    claimed=?claimed.into(),
-                    actual=?actual.and_then(|g| g.into()),
-                    "Rejecting import progress fetch from stale generation"
-                );
-
-                return Err(ApiError::BadRequest(anyhow::anyhow!("Invalid generation")));
-            }
-        }
-
-        let maybe_import = self
-            .persistence
-            .get_timeline_import(req.tenant_shard_id.tenant_id, req.timeline_id)
-            .await?;
-
-        let import = maybe_import.ok_or_else(|| {
-            ApiError::NotFound(
-                format!(
-                    "import for {}/{} not found",
-                    req.tenant_shard_id.tenant_id, req.timeline_id
-                )
-                .into(),
-            )
-        })?;
-
-        import
-            .shard_statuses
-            .0
-            .get(&req.tenant_shard_id.to_index())
-            .cloned()
-            .ok_or_else(|| {
-                ApiError::NotFound(
-                    format!("shard {} not found", req.tenant_shard_id.shard_slug()).into(),
-                )
-            })
-    }
-
     pub(crate) async fn handle_timeline_shard_import_progress_upcall(
         self: &Arc<Self>,
         req: PutTimelineImportStatusRequest,
     ) -> Result<(), ApiError> {
-        let validity = self
-            .validate_shard_generation(req.tenant_shard_id, req.generation)
-            .await?;
-        match validity {
-            ShardGenerationValidity::Valid => {
-                // fallthrough
-            }
-            ShardGenerationValidity::Mismatched { claimed, actual } => {
-                tracing::info!(
-                    claimed=?claimed.into(),
-                    actual=?actual.and_then(|g| g.into()),
-                    "Rejecting import progress update from stale generation"
-                );
-
-                return Err(ApiError::PreconditionFailed("Invalid generation".into()));
-            }
-        }
-
         let res = self
             .persistence
             .update_timeline_import(req.tenant_shard_id, req.timeline_id, req.status)
@@ -4020,147 +3943,66 @@ impl Service {
         Ok(())
     }
 
-    /// Check that a provided generation for some tenant shard is the most recent one.
-    ///
-    /// Validate with the in-mem state first, and, if that passes, validate with the
-    /// database state which is authoritative.
-    async fn validate_shard_generation(
-        self: &Arc<Self>,
-        tenant_shard_id: TenantShardId,
-        generation: Generation,
-    ) -> Result<ShardGenerationValidity, ApiError> {
-        {
-            let locked = self.inner.read().unwrap();
-            let tenant_shard =
-                locked
-                    .tenants
-                    .get(&tenant_shard_id)
-                    .ok_or(ApiError::InternalServerError(anyhow::anyhow!(
-                        "{} shard not found",
-                        tenant_shard_id
-                    )))?;
-
-            if tenant_shard.generation != Some(generation) {
-                return Ok(ShardGenerationValidity::Mismatched {
-                    claimed: generation,
-                    actual: tenant_shard.generation,
-                });
-            }
-        }
-
-        let mut db_generations = self
-            .persistence
-            .shard_generations(std::iter::once(&tenant_shard_id))
-            .await?;
-        let (_tid, db_generation) =
-            db_generations
-                .pop()
-                .ok_or(ApiError::InternalServerError(anyhow::anyhow!(
-                    "{} shard not found",
-                    tenant_shard_id
-                )))?;
-
-        if db_generation != Some(generation) {
-            return Ok(ShardGenerationValidity::Mismatched {
-                claimed: generation,
-                actual: db_generation,
-            });
-        }
-
-        Ok(ShardGenerationValidity::Valid)
-    }
-
-    /// Finalize the import of a timeline
-    ///
-    /// This method should be called once all shards have reported that the import is complete.
-    /// Firstly, it polls the post import timeline activation endpoint exposed by the pageserver.
-    /// Once the timeline is active on all shards, the timeline also gets created on the
-    /// safekeepers. Finally, notify cplane of the import completion (whether failed or
-    /// successful), and remove the import from the database and in-memory.
-    ///
-    /// If this method gets pre-empted by shut down, it will be called again at start-up (on-going
-    /// imports are stored in the database).
     #[instrument(skip_all, fields(
         tenant_id=%import.tenant_id,
-        timeline_id=%import.timeline_id,
+        shard_id=%import.timeline_id,
     ))]
     async fn finalize_timeline_import(
         self: &Arc<Self>,
         import: TimelineImport,
-    ) -> Result<(), TimelineImportFinalizeError> {
+    ) -> anyhow::Result<()> {
         tracing::info!("Finalizing timeline import");
 
         pausable_failpoint!("timeline-import-pre-cplane-notification");
 
-        let tenant_id = import.tenant_id;
-        let timeline_id = import.timeline_id;
+        let import_failed = import.completion_error().is_some();
 
-        let import_error = import.completion_error();
-        match import_error {
-            Some(err) => {
-                self.notify_cplane_and_delete_import(tenant_id, timeline_id, Err(err))
-                    .await?;
-                tracing::warn!("Timeline import completed with shard errors");
-                Ok(())
-            }
-            None => match self.activate_timeline_post_import(&import).await {
-                Ok(timeline_info) => {
-                    tracing::info!("Post import timeline activation complete");
+        if !import_failed {
+            loop {
+                if self.cancel.is_cancelled() {
+                    anyhow::bail!("Shut down requested while finalizing import");
+                }
 
-                    if self.config.timelines_onto_safekeepers {
-                        // Now that we know the start LSN of this timeline, create it on the
-                        // safekeepers.
-                        self.tenant_timeline_create_safekeepers_until_success(
-                            import.tenant_id,
-                            timeline_info,
-                        )
-                        .await?;
+                let active = self.timeline_active_on_all_shards(&import).await?;
+
+                match active {
+                    Some(timeline_info) => {
+                        tracing::info!("Timeline became active on all shards");
+
+                        if self.config.timelines_onto_safekeepers {
+                            // Now that we know the start LSN of this timeline, create it on the
+                            // safekeepers.
+                            self.tenant_timeline_create_safekeepers_until_success(
+                                import.tenant_id,
+                                timeline_info,
+                            )
+                            .await?;
+                        }
+
+                        break;
                     }
+                    None => {
+                        tracing::info!("Timeline not active on all shards yet");
 
-                    self.notify_cplane_and_delete_import(tenant_id, timeline_id, Ok(()))
-                        .await?;
-
-                    tracing::info!("Timeline import completed successfully");
-                    Ok(())
+                        tokio::select! {
+                            _ = self.cancel.cancelled() => {
+                                anyhow::bail!("Shut down requested while finalizing import");
+                            },
+                            _ = tokio::time::sleep(Duration::from_secs(5)) => {}
+                        };
+                    }
                 }
-                Err(TimelineImportFinalizeError::ShuttingDown) => {
-                    // We got pre-empted by shut down and will resume after the restart.
-                    Err(TimelineImportFinalizeError::ShuttingDown)
-                }
-                Err(err) => {
-                    // Any finalize error apart from shut down is permanent and requires us to notify
-                    // cplane such that it can clean up.
-                    tracing::error!("Import finalize failed with permanent error: {err}");
-                    self.notify_cplane_and_delete_import(
-                        tenant_id,
-                        timeline_id,
-                        Err(err.to_string()),
-                    )
-                    .await?;
-                    Err(err)
-                }
-            },
+            }
         }
-    }
 
-    async fn notify_cplane_and_delete_import(
-        self: &Arc<Self>,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        import_result: ImportResult,
-    ) -> Result<(), TimelineImportFinalizeError> {
-        let import_failed = import_result.is_err();
         tracing::info!(%import_failed, "Notifying cplane of import completion");
 
         let client = UpcallClient::new(self.get_config(), self.cancel.child_token());
-        client
-            .notify_import_complete(tenant_id, timeline_id, import_result)
-            .await
-            .map_err(|_err| TimelineImportFinalizeError::ShuttingDown)?;
+        client.notify_import_complete(&import).await?;
 
         if let Err(err) = self
             .persistence
-            .delete_timeline_import(tenant_id, timeline_id)
+            .delete_timeline_import(import.tenant_id, import.timeline_id)
             .await
         {
             tracing::warn!("Failed to delete timeline import entry from database: {err}");
@@ -4170,113 +4012,14 @@ impl Service {
             .write()
             .unwrap()
             .tenants
-            .range_mut(TenantShardId::tenant_range(tenant_id))
+            .range_mut(TenantShardId::tenant_range(import.tenant_id))
             .for_each(|(_id, shard)| shard.importing = TimelineImportState::Idle);
 
+        tracing::info!(%import_failed, "Timeline import complete");
+
         Ok(())
     }
 
-    /// Activate an imported timeline on all shards once the import is complete.
-    /// Returns the [`TimelineInfo`] reported by shard zero.
-    async fn activate_timeline_post_import(
-        self: &Arc<Self>,
-        import: &TimelineImport,
-    ) -> Result<TimelineInfo, TimelineImportFinalizeError> {
-        const TIMELINE_ACTIVATE_TIMEOUT: Duration = Duration::from_millis(128);
-
-        let mut shards_to_activate: HashSet<ShardIndex> =
-            import.shard_statuses.0.keys().cloned().collect();
-        let mut shard_zero_timeline_info = None;
-
-        while !shards_to_activate.is_empty() {
-            if self.cancel.is_cancelled() {
-                return Err(TimelineImportFinalizeError::ShuttingDown);
-            }
-
-            let targets = {
-                let locked = self.inner.read().unwrap();
-                let mut targets = Vec::new();
-
-                for (tenant_shard_id, shard) in locked
-                    .tenants
-                    .range(TenantShardId::tenant_range(import.tenant_id))
-                {
-                    if !import
-                        .shard_statuses
-                        .0
-                        .contains_key(&tenant_shard_id.to_index())
-                    {
-                        return Err(TimelineImportFinalizeError::MismatchedShards(
-                            tenant_shard_id.to_index(),
-                        ));
-                    }
-
-                    if let Some(node_id) = shard.intent.get_attached() {
-                        let node = locked
-                            .nodes
-                            .get(node_id)
-                            .expect("Pageservers may not be deleted while referenced");
-                        targets.push((*tenant_shard_id, node.clone()));
-                    }
-                }
-
-                targets
-            };
-
-            let targeted_tenant_shards: Vec<_> = targets.iter().map(|(tid, _node)| *tid).collect();
-
-            let results = self
-                .tenant_for_shards_api(
-                    targets,
-                    |tenant_shard_id, client| async move {
-                        client
-                            .activate_post_import(
-                                tenant_shard_id,
-                                import.timeline_id,
-                                TIMELINE_ACTIVATE_TIMEOUT,
-                            )
-                            .await
-                    },
-                    1,
-                    1,
-                    SHORT_RECONCILE_TIMEOUT,
-                    &self.cancel,
-                )
-                .await;
-
-            let mut failed = 0;
-            for (tid, result) in targeted_tenant_shards.iter().zip(results.into_iter()) {
-                match result {
-                    Ok(ok) => {
-                        if tid.is_shard_zero() {
-                            shard_zero_timeline_info = Some(ok);
-                        }
-
-                        shards_to_activate.remove(&tid.to_index());
-                    }
-                    Err(_err) => {
-                        failed += 1;
-                    }
-                }
-            }
-
-            if failed > 0 {
-                tracing::info!(
-                    "Failed to activate timeline on {failed} shards post import. Will retry"
-                );
-            }
-
-            tokio::select! {
-                _ = tokio::time::sleep(Duration::from_millis(250)) => {},
-                _ = self.cancel.cancelled() => {
-                    return Err(TimelineImportFinalizeError::ShuttingDown);
-                }
-            }
-        }
-
-        Ok(shard_zero_timeline_info.expect("All shards replied"))
-    }
-
     async fn finalize_timeline_imports(self: &Arc<Self>, imports: Vec<TimelineImport>) {
         futures::future::join_all(
             imports
@@ -4286,6 +4029,78 @@ impl Service {
         .await;
     }
 
+    /// If the timeline is active on all shards, returns the [`TimelineInfo`]
+    /// collected from shard 0.
+    ///
+    /// An error is returned if the shard layout has changed during the import.
+    /// This is guarded against within the storage controller and the pageserver,
+    /// and, therefore, unexpected.
+    async fn timeline_active_on_all_shards(
+        self: &Arc<Self>,
+        import: &TimelineImport,
+    ) -> anyhow::Result<Option<TimelineInfo>> {
+        let targets = {
+            let locked = self.inner.read().unwrap();
+            let mut targets = Vec::new();
+
+            for (tenant_shard_id, shard) in locked
+                .tenants
+                .range(TenantShardId::tenant_range(import.tenant_id))
+            {
+                if !import
+                    .shard_statuses
+                    .0
+                    .contains_key(&tenant_shard_id.to_index())
+                {
+                    anyhow::bail!("Shard layout change detected on completion");
+                }
+
+                if let Some(node_id) = shard.intent.get_attached() {
+                    let node = locked
+                        .nodes
+                        .get(node_id)
+                        .expect("Pageservers may not be deleted while referenced");
+                    targets.push((*tenant_shard_id, node.clone()));
+                } else {
+                    return Ok(None);
+                }
+            }
+
+            targets
+        };
+
+        if targets.is_empty() {
+            anyhow::bail!("No shards found to finalize import for");
+        }
+
+        let results = self
+            .tenant_for_shards_api(
+                targets,
+                |tenant_shard_id, client| async move {
+                    client
+                        .timeline_detail(tenant_shard_id, import.timeline_id)
+                        .await
+                },
+                1,
+                1,
+                SHORT_RECONCILE_TIMEOUT,
+                &self.cancel,
+            )
+            .await;
+
+        let all_active = results.iter().all(|res| match res {
+            Ok(info) => info.state == TimelineState::Active,
+            Err(_) => false,
+        });
+
+        if all_active {
+            // Both unwraps are validated above
+            Ok(Some(results.into_iter().next().unwrap().unwrap()))
+        } else {
+            Ok(None)
+        }
+    }
+
     pub(crate) async fn tenant_timeline_archival_config(
         &self,
         tenant_id: TenantId,

storage_controller/src/service/safekeeper_service.rs

@@ -10,7 +10,6 @@ use crate::persistence::{
     DatabaseError, SafekeeperTimelineOpKind, TimelinePendingOpPersistence, TimelinePersistence,
 };
 use crate::safekeeper::Safekeeper;
-use crate::timeline_import::TimelineImportFinalizeError;
 use anyhow::Context;
 use http_utils::error::ApiError;
 use pageserver_api::controller_api::{
@@ -328,12 +327,12 @@ impl Service {
         self: &Arc<Self>,
         tenant_id: TenantId,
         timeline_info: TimelineInfo,
-    ) -> Result<(), TimelineImportFinalizeError> {
+    ) -> anyhow::Result<()> {
         const BACKOFF: Duration = Duration::from_secs(5);
 
         loop {
             if self.cancel.is_cancelled() {
-                return Err(TimelineImportFinalizeError::ShuttingDown);
+                anyhow::bail!("Shut down requested while finalizing import");
             }
 
             let res = self
@@ -349,7 +348,7 @@ impl Service {
                     tracing::error!("Failed to create timeline on safekeepers: {err}");
                     tokio::select! {
                         _ = self.cancel.cancelled() => {
-                            return Err(TimelineImportFinalizeError::ShuttingDown);
+                            anyhow::bail!("Shut down requested while finalizing import");
                         },
                         _ = tokio::time::sleep(BACKOFF) => {}
                     };

storage_controller/src/timeline_import.rs

@@ -5,7 +5,7 @@ use http_utils::error::ApiError;
 use reqwest::Method;
 use serde::{Deserialize, Serialize};
 
-use pageserver_api::models::{ShardImportProgress, ShardImportStatus};
+use pageserver_api::models::ShardImportStatus;
 use tokio_util::sync::CancellationToken;
 use utils::{
     id::{TenantId, TimelineId},
@@ -28,12 +28,7 @@ impl ShardImportStatuses {
         ShardImportStatuses(
             shards
                 .into_iter()
-                .map(|ts_id| {
-                    (
-                        ts_id,
-                        ShardImportStatus::InProgress(None::<ShardImportProgress>),
-                    )
-                })
+                .map(|ts_id| (ts_id, ShardImportStatus::InProgress))
                 .collect(),
         )
     }
@@ -51,14 +46,6 @@ pub(crate) enum TimelineImportUpdateFollowUp {
     None,
 }
 
-#[derive(thiserror::Error, Debug)]
-pub(crate) enum TimelineImportFinalizeError {
-    #[error("Shut down interrupted import finalize")]
-    ShuttingDown,
-    #[error("Mismatched shard detected during import finalize: {0}")]
-    MismatchedShards(ShardIndex),
-}
-
 pub(crate) enum TimelineImportUpdateError {
     ImportNotFound {
         tenant_id: TenantId,
@@ -164,8 +151,6 @@ impl TimelineImport {
     }
 }
 
-pub(crate) type ImportResult = Result<(), String>;
-
 pub(crate) struct UpcallClient {
     authorization_header: Option<String>,
     client: reqwest::Client,
@@ -213,9 +198,7 @@ impl UpcallClient {
     /// eventual cplane availability. The cplane API is idempotent.
     pub(crate) async fn notify_import_complete(
         &self,
-        tenant_id: TenantId,
-        timeline_id: TimelineId,
-        import_result: ImportResult,
+        import: &TimelineImport,
     ) -> anyhow::Result<()> {
         let endpoint = if self.base_url.ends_with('/') {
             format!("{}import_complete", self.base_url)
@@ -223,13 +206,15 @@ impl UpcallClient {
             format!("{}/import_complete", self.base_url)
         };
 
+        tracing::info!("Endpoint is {endpoint}");
+
         let request = self
             .client
             .request(Method::PUT, endpoint)
             .json(&ImportCompleteRequest {
-                tenant_id,
-                timeline_id,
-                error: import_result.err(),
+                tenant_id: import.tenant_id,
+                timeline_id: import.timeline_id,
+                error: import.completion_error(),
             })
             .timeout(IMPORT_COMPLETE_REQUEST_TIMEOUT);
 

storage_scrubber/src/checks.rs

@@ -13,7 +13,7 @@ use pageserver::tenant::remote_timeline_client::{
 };
 use pageserver::tenant::storage_layer::LayerName;
 use pageserver_api::shard::ShardIndex;
-use remote_storage::{DownloadError, GenericRemoteStorage, ListingObject, RemotePath};
+use remote_storage::{GenericRemoteStorage, ListingObject, RemotePath};
 use tokio_util::sync::CancellationToken;
 use tracing::{info, warn};
 use utils::generation::Generation;
@@ -165,34 +165,23 @@ pub(crate) async fn branch_cleanup_and_check_errors(
                                 .head_object(&path, &CancellationToken::new())
                                 .await;
 
-                            match response {
-                                Ok(_) => {}
-                                Err(DownloadError::NotFound) => {
-                                    // Object is not present.
-                                    let is_l0 =
-                                        LayerMap::is_l0(layer.key_range(), layer.is_delta());
+                            if let Err(e) = response {
+                                // Object is not present.
+                                let is_l0 = LayerMap::is_l0(layer.key_range(), layer.is_delta());
 
-                                    let msg = format!(
-                                        "index_part.json contains a layer {}{} (shard {}) that is not present in remote storage (layer_is_l0: {})",
-                                        layer,
-                                        metadata.generation.get_suffix(),
-                                        metadata.shard,
-                                        is_l0,
-                                    );
+                                let msg = format!(
+                                    "index_part.json contains a layer {}{} (shard {}) that is not present in remote storage (layer_is_l0: {}) with error: {}",
+                                    layer,
+                                    metadata.generation.get_suffix(),
+                                    metadata.shard,
+                                    is_l0,
+                                    e,
+                                );
 
-                                    if is_l0 || ignore_error {
-                                        result.warnings.push(msg);
-                                    } else {
-                                        result.errors.push(msg);
-                                    }
-                                }
-                                Err(e) => {
-                                    tracing::warn!(
-                                        "cannot check if the layer {}{} is present in remote storage (error: {})",
-                                        layer,
-                                        metadata.generation.get_suffix(),
-                                        e,
-                                    );
+                                if is_l0 || ignore_error {
+                                    result.warnings.push(msg);
+                                } else {
+                                    result.errors.push(msg);
                                 }
                             }
                         }

test_runner/bin/neon_local_create_deep_l0_stack.py

@@ -1,59 +0,0 @@
-"""
-Script to creates a stack of L0 deltas each of which should have 1 Value::Delta per page in `data`,
-in your running neon_local setup.
-
-Use this bash setup to reset your neon_local environment.
-The last line of this bash snippet will run this file here.
-```
- export NEON_REPO_DIR=$PWD/.neon
- export NEON_BIN_DIR=$PWD/target/release
- $NEON_BIN_DIR/neon_local stop
- rm -rf $NEON_REPO_DIR
- $NEON_BIN_DIR/neon_local init
- cat >>  $NEON_REPO_DIR/pageserver_1/pageserver.toml <<"EOF"
- # customizations
- virtual_file_io_mode = "direct-rw"
- page_service_pipelining={mode="pipelined", max_batch_size=32, execution="concurrent-futures"}
- get_vectored_concurrent_io={mode="sidecar-task"}
-EOF
- $NEON_BIN_DIR/neon_local start
-
- psql 'postgresql://localhost:1235/storage_controller' -c 'DELETE FROM tenant_shards'
- sed 's/.*get_vectored_concurrent_io.*/get_vectored_concurrent_io={mode="sidecar-task"}/' -i $NEON_REPO_DIR/pageserver_1/pageserver.toml
- $NEON_BIN_DIR/neon_local pageserver restart
- sleep 2
- $NEON_BIN_DIR/neon_local tenant create --set-default
- ./target/debug/neon_local endpoint stop foo
- rm -rf  $NEON_REPO_DIR/endpoints/foo
- ./target/debug/neon_local endpoint create foo
- echo 'full_page_writes=off' >>  $NEON_REPO_DIR/endpoints/foo/postgresql.conf
- ./target/debug/neon_local endpoint start foo
-
-  pushd test_runner; poetry run python3 -m bin.neon_local_create_deep_l0_stack 10; popd
-```
-"""
-
-import sys
-
-import psycopg2
-from fixtures.common_types import TenantShardId, TimelineId
-from fixtures.pageserver.http import PageserverHttpClient
-from fixtures.pageserver.makelayers.l0stack import L0StackShape, make_l0_stack_standalone
-
-ps_http = PageserverHttpClient(port=9898, is_testing_enabled_or_skip=lambda: None)
-vps_http = PageserverHttpClient(port=1234, is_testing_enabled_or_skip=lambda: None)
-
-tenants = ps_http.tenant_list()
-assert len(tenants) == 1
-tenant_shard_id = TenantShardId.parse(tenants[0]["id"])
-
-timlines = ps_http.timeline_list(tenant_shard_id)
-assert len(timlines) == 1
-timeline_id = TimelineId(timlines[0]["timeline_id"])
-
-connstr = "postgresql://cloud_admin@localhost:55432/postgres"
-conn = psycopg2.connect(connstr)
-
-shape = L0StackShape(logical_table_size_mib=50, delta_stack_height=int(sys.argv[1]))
-
-make_l0_stack_standalone(vps_http, ps_http, tenant_shard_id, timeline_id, conn, shape)

test_runner/fixtures/neon_fixtures.py

@@ -1255,12 +1255,6 @@ class NeonEnv:
                 "no_sync": True,
                 # Look for gaps in WAL received from safekeepeers
                 "validate_wal_contiguity": True,
-                # TODO(vlad): make these configurable through the builder
-                "timeline_import_config": {
-                    "import_job_concurrency": 4,
-                    "import_job_soft_size_limit": 512 * 1024,
-                    "import_job_checkpoint_threshold": 4,
-                },
             }
 
             # Batching (https://github.com/neondatabase/neon/issues/9377):
@@ -1377,11 +1371,7 @@ class NeonEnv:
             force=config.config_init_force,
         )
 
-    def start(
-        self,
-        timeout_in_seconds: int | None = None,
-        extra_ps_env_vars: dict[str, str] | None = None,
-    ):
+    def start(self, timeout_in_seconds: int | None = None):
         # Storage controller starts first, so that pageserver /re-attach calls don't
         # bounce through retries on startup
         self.storage_controller.start(timeout_in_seconds=timeout_in_seconds)
@@ -1400,10 +1390,7 @@ class NeonEnv:
             for pageserver in self.pageservers:
                 futs.append(
                     executor.submit(
-                        lambda ps=pageserver: ps.start(  # type: ignore[misc]
-                            extra_env_vars=extra_ps_env_vars or {},
-                            timeout_in_seconds=timeout_in_seconds,
-                        ),
+                        lambda ps=pageserver: ps.start(timeout_in_seconds=timeout_in_seconds)  # type: ignore[misc]
                     )
                 )
 

test_runner/fixtures/pageserver/makelayers/l0stack.py

@@ -1,148 +0,0 @@
-from dataclasses import dataclass
-
-from psycopg2.extensions import connection as PgConnection
-
-from fixtures.common_types import Lsn, TenantShardId, TimelineId
-from fixtures.log_helper import log
-from fixtures.neon_fixtures import Endpoint
-from fixtures.pageserver.http import PageserverHttpClient
-from fixtures.pageserver.utils import wait_for_last_record_lsn
-
-
-@dataclass
-class L0StackShape:
-    logical_table_size_mib: int = 50
-    delta_stack_height: int = 20
-
-
-def make_l0_stack(endpoint: Endpoint, shape: L0StackShape):
-    """
-    Creates stack of L0 deltas each of which should have 1 Value::Delta per page in table `data`.
-    """
-    env = endpoint.env
-
-    # TDOO: wait for storcon to finish any reonciles before jumping to action here?
-    description = env.storage_controller.tenant_describe(endpoint.tenant_id)
-    shards = description["shards"]
-    assert len(shards) == 1, "does not support sharding"
-    tenant_shard_id = TenantShardId.parse(shards[0]["tenant_shard_id"])
-
-    endpoint.config(["full_page_writes=off"])
-    endpoint.reconfigure()
-
-    ps = env.get_pageserver(shards[0]["node_attached"])
-
-    timeline_id = endpoint.show_timeline_id()
-
-    vps_http = env.storage_controller.pageserver_api()
-    ps_http = ps.http_client()
-    endpoint_conn = endpoint.connect()
-    make_l0_stack_standalone(vps_http, ps_http, tenant_shard_id, timeline_id, endpoint_conn, shape)
-
-
-def make_l0_stack_standalone(
-    vps_http: PageserverHttpClient,
-    ps_http: PageserverHttpClient,
-    tenant_shard_id: TenantShardId,
-    timeline_id: TimelineId,
-    endpoint_conn: PgConnection,
-    shape: L0StackShape,
-):
-    """
-    See make_l0_stack for details.
-
-    This function is a standalone version of make_l0_stack, usable from not-test code.
-    """
-
-    assert not tenant_shard_id.shard_index.is_sharded, (
-        "the current implementation only supports unsharded tenants"
-    )
-
-    tenant_id = tenant_shard_id.tenant_id
-    conn = endpoint_conn
-    desired_size = shape.logical_table_size_mib * 1024 * 1024
-
-    config = {
-        "gc_period": "0s",  # disable periodic gc
-        "checkpoint_timeout": "10 years",
-        "compaction_period": "1h",  # doesn't matter, but 0 value will kill walredo every 10s
-        "compaction_threshold": 100000,  # we just want L0s
-        "compaction_target_size": 134217728,
-        "checkpoint_distance": 268435456,
-        "image_creation_threshold": 100000,  # we just want L0s
-    }
-
-    vps_http.set_tenant_config(tenant_id, config)
-
-    conn.autocommit = True
-    cur = conn.cursor()
-
-    # Ensure full_page_writes are disabled so that all Value::Delta in
-    # pageserver are !will_init, and therefore a getpage needs to read
-    # the entire delta stack.
-    cur.execute("SHOW full_page_writes")
-    assert cur.fetchall()[0][0] == "off", "full_page_writes should be off"
-
-    # each tuple is 23 (header) + 100 bytes = 123 bytes
-    # page header si 24 bytes
-    # 8k page size
-    # (8k-24bytes) / 123 bytes = 63 tuples per page
-    # set fillfactor to 10 to have 6 tuples per page
-    cur.execute("DROP TABLE IF EXISTS data")
-    cur.execute("CREATE TABLE data(id bigint, row char(92)) with (fillfactor=10)")
-    need_pages = desired_size // 8192
-    need_rows = need_pages * 6
-    log.info(f"Need {need_pages} pages, {need_rows} rows")
-    cur.execute(f"INSERT INTO data SELECT i,'row'||i FROM generate_series(1, {need_rows}) as i")
-    # Raise fillfactor to 100% so that all updates are HOT updates.
-    # We assert they're hot updates by checking fetch_id_to_page_mapping remains the same.
-    cur.execute("ALTER TABLE data SET (fillfactor=100)")
-
-    def settle_and_flush():
-        cur.execute("SELECT pg_current_wal_flush_lsn()")
-        flush_lsn = Lsn(cur.fetchall()[0][0])
-        wait_for_last_record_lsn(ps_http, tenant_shard_id, timeline_id, flush_lsn)
-        ps_http.timeline_checkpoint(tenant_id, timeline_id)
-
-    # create an L0 for the initial data we just inserted
-    settle_and_flush()
-
-    # assert we wrote what we think we wrote
-    cur.execute("""
-        with ntuples_per_page as (
-            select (ctid::text::point)[0]::bigint pageno,count(*) ntuples from data group by pageno
-        )
-        select ntuples, count(*) npages from ntuples_per_page group by ntuples order by ntuples;
-    """)
-    rows = cur.fetchall()
-    log.info(f"initial table layout: {rows}")
-    assert len(rows) == 1
-    assert rows[0][0] == 6, f"expected 6 tuples per page, got {rows[0][0]}"
-    assert rows[0][1] == need_pages, f"expected {need_pages} pages, got {rows[0][1]}"
-
-    def fetch_id_to_page_mapping():
-        cur.execute("""
-            SELECT id,(ctid::text::point)[0]::bigint pageno FROM data ORDER BY id
-        """)
-        return cur.fetchall()
-
-    initial_mapping = fetch_id_to_page_mapping()
-
-    # every iteration updates one tuple in each page
-    delta_stack_height = shape.delta_stack_height
-    for i in range(0, delta_stack_height):
-        log.info(i)
-        cur.execute(f"UPDATE data set row = row||',u' where id % 6 = {i % 6}")
-        log.info(f"modified rows: {cur.rowcount}")
-        assert cur.rowcount == need_pages
-        settle_and_flush()
-        post_update_mapping = fetch_id_to_page_mapping()
-        assert initial_mapping == post_update_mapping, "Postgres should be doing HOT updates"
-
-    # Assert the layer count is what we expect it is
-    layer_map = vps_http.layer_map_info(tenant_id, timeline_id)
-    assert (
-        len(layer_map.delta_l0_layers()) == delta_stack_height + 1 + 1
-    )  # +1 for the initdb layer + 1 for the table creation & fill
-    assert len(layer_map.delta_l0_layers()) == len(layer_map.delta_layers())  # it's all L0s
-    assert len(layer_map.image_layers()) == 0  # no images

test_runner/performance/README.md

@@ -15,8 +15,7 @@ Some handy pytest flags for local development:
 - `-k` selects a test to run
 - `--timeout=0` disables our default timeout of 300s (see `setup.cfg`)
 - `--preserve-database-files` to skip cleanup
-- `--out-dir` to produce a JSON with the recorded test metrics.
-  There is a post-processing tool at `test_runner/performance/out_dir_to_csv.py`.
+- `--out-dir` to produce a JSON with the recorded test metrics
 
 # What performance tests do we have and how we run them
 

test_runner/performance/out_dir_to_csv.py

@@ -1,57 +0,0 @@
-# Tool to convert the JSON output from running a perf test with `--out-dir` to a CSV that
-# can be easily pasted into a spreadsheet for quick viz & analysis.
-# Check the `./README.md` in this directory for `--out-dir`.
-#
-# TODO: add the pytest.mark.parametrize to the json and make them columns here
-# https://github.com/neondatabase/neon/issues/11878
-
-import csv
-import json
-import os
-import sys
-
-
-def json_to_csv(json_file):
-    with open(json_file) as f:
-        data = json.load(f)
-
-    # Collect all possible metric names to form headers
-    all_metrics = set()
-    for result in data.get("result", []):
-        for metric in result.get("data", []):
-            all_metrics.add(metric["name"])
-
-    # Sort metrics for consistent output
-    metrics = sorted(list(all_metrics))
-
-    # Create headers
-    headers = ["suit"] + metrics
-
-    # Prepare rows
-    rows = []
-    for result in data.get("result", []):
-        row = {"suit": result["suit"]}
-
-        # Initialize all metrics to empty
-        for metric in metrics:
-            row[metric] = ""
-
-        # Fill in available metrics
-        for item in result.get("data", []):
-            row[item["name"]] = item["value"]
-
-        rows.append(row)
-
-    # Write to stdout as CSV
-    writer = csv.DictWriter(sys.stdout, fieldnames=headers)
-    writer.writeheader()
-    writer.writerows(rows)
-
-
-if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print(f"Usage: python {os.path.basename(__file__)} <json_file>")
-        sys.exit(1)
-
-    json_file = sys.argv[1]
-    json_to_csv(json_file)

test_runner/performance/pageserver/test_page_service_batching.py

@@ -10,8 +10,7 @@ from typing import Any
 import pytest
 from fixtures.benchmark_fixture import MetricReport, NeonBenchmarker
 from fixtures.log_helper import log
-from fixtures.neon_fixtures import NeonEnv, NeonEnvBuilder, PgBin
-from fixtures.pageserver.makelayers import l0stack
+from fixtures.neon_fixtures import NeonEnvBuilder, PgBin, wait_for_last_flush_lsn
 from fixtures.utils import humantime_to_ms
 
 TARGET_RUNTIME = 30
@@ -35,18 +34,28 @@ class PageServicePipeliningConfigPipelined(PageServicePipeliningConfig):
     mode: str = "pipelined"
 
 
-PS_IO_CONCURRENCY = ["sidecar-task"]
-PIPELINING_CONFIGS: list[PageServicePipeliningConfig] = []
+EXECUTION = ["concurrent-futures"]
+BATCHING = ["uniform-lsn", "scattered-lsn"]
+
+NON_BATCHABLE: list[PageServicePipeliningConfig] = [PageServicePipeliningConfigSerial()]
+for max_batch_size in [1, 32]:
+    for execution in EXECUTION:
+        for batching in BATCHING:
+            NON_BATCHABLE.append(
+                PageServicePipeliningConfigPipelined(max_batch_size, execution, batching)
+            )
+
+BATCHABLE: list[PageServicePipeliningConfig] = []
 for max_batch_size in [32]:
-    for execution in ["concurrent-futures"]:
-        for batching in ["scattered-lsn"]:
-            PIPELINING_CONFIGS.append(
+    for execution in EXECUTION:
+        for batching in BATCHING:
+            BATCHABLE.append(
                 PageServicePipeliningConfigPipelined(max_batch_size, execution, batching)
             )
 
 
 @pytest.mark.parametrize(
-    "tablesize_mib, pipelining_config, target_runtime, ps_io_concurrency, effective_io_concurrency, readhead_buffer_size, name",
+    "tablesize_mib, pipelining_config, target_runtime, effective_io_concurrency, readhead_buffer_size, name",
     [
         # batchable workloads should show throughput and CPU efficiency improvements
         *[
@@ -54,23 +63,20 @@ for max_batch_size in [32]:
                 50,
                 config,
                 TARGET_RUNTIME,
-                ps_io_concurrency,
                 100,
                 128,
                 f"batchable {dataclasses.asdict(config)}",
             )
-            for config in PIPELINING_CONFIGS
-            for ps_io_concurrency in PS_IO_CONCURRENCY
+            for config in BATCHABLE
         ],
     ],
 )
-def test_postgres_seqscan(
+def test_throughput(
     neon_env_builder: NeonEnvBuilder,
     zenbenchmark: NeonBenchmarker,
     tablesize_mib: int,
     pipelining_config: PageServicePipeliningConfig,
     target_runtime: int,
-    ps_io_concurrency: str,
     effective_io_concurrency: int,
     readhead_buffer_size: int,
     name: str,
@@ -91,10 +97,6 @@ def test_postgres_seqscan(
     If the compute provides pipeline depth (effective_io_concurrency=100), then
     pipelining configs, especially with max_batch_size>1 should yield dramatic improvements
     in all performance metrics.
-
-    We advance the LSN from a disruptor thread to simulate the effect of a workload with concurrent writes
-    in another table. The `scattered-lsn` batching mode handles this well whereas the
-    initial implementatin (`uniform-lsn`) would break the batch.
     """
 
     #
@@ -112,19 +114,7 @@ def test_postgres_seqscan(
         }
     )
     # For storing configuration as a metric, insert a fake 0 with labels with actual data
-    params.update(
-        {
-            "config": (
-                0,
-                {
-                    "labels": {
-                        "pipelining_config": dataclasses.asdict(pipelining_config),
-                        "ps_io_concurrency": ps_io_concurrency,
-                    }
-                },
-            )
-        }
-    )
+    params.update({"pipelining_config": (0, {"labels": dataclasses.asdict(pipelining_config)})})
 
     log.info("params: %s", params)
 
@@ -276,10 +266,7 @@ def test_postgres_seqscan(
         return iters
 
     env.pageserver.patch_config_toml_nonrecursive(
-        {
-            "page_service_pipelining": dataclasses.asdict(pipelining_config),
-            "get_vectored_concurrent_io": {"mode": ps_io_concurrency},
-        }
+        {"page_service_pipelining": dataclasses.asdict(pipelining_config)}
     )
 
     # set trace for log analysis below
@@ -331,63 +318,77 @@ def test_postgres_seqscan(
     )
 
 
+PRECISION_CONFIGS: list[PageServicePipeliningConfig] = [PageServicePipeliningConfigSerial()]
+for max_batch_size in [1, 32]:
+    for execution in EXECUTION:
+        for batching in BATCHING:
+            PRECISION_CONFIGS.append(
+                PageServicePipeliningConfigPipelined(max_batch_size, execution, batching)
+            )
+
+
 @pytest.mark.parametrize(
-    "pipelining_config,ps_io_concurrency,l0_stack_height,queue_depth,name",
-    [
-        (config, ps_io_concurrency, l0_stack_height, queue_depth, f"{dataclasses.asdict(config)}")
-        for config in PIPELINING_CONFIGS
-        for ps_io_concurrency in PS_IO_CONCURRENCY
-        for queue_depth in [1, 2, 32]
-        for l0_stack_height in [0, 20]
-    ],
+    "pipelining_config,name",
+    [(config, f"{dataclasses.asdict(config)}") for config in PRECISION_CONFIGS],
 )
-def test_random_reads(
+def test_latency(
     neon_env_builder: NeonEnvBuilder,
     zenbenchmark: NeonBenchmarker,
     pg_bin: PgBin,
     pipelining_config: PageServicePipeliningConfig,
-    ps_io_concurrency: str,
-    l0_stack_height: int,
-    queue_depth: int,
     name: str,
 ):
     """
-    Throw pagebench random getpage at latest lsn workload from a single client against pageserver.
+    Measure the latency impact of pipelining in an un-batchable workloads.
+
+    An ideal implementation should not increase average or tail latencies for such workloads.
+
+    We don't have support in pagebench to create queue depth yet.
+    => https://github.com/neondatabase/neon/issues/9837
     """
 
     #
     # Setup
     #
 
-    def build_snapshot_cb(neon_env_builder: NeonEnvBuilder) -> NeonEnv:
-        env = neon_env_builder.init_start()
-        endpoint = env.endpoints.create_start("main")
-        l0stack.make_l0_stack(
-            endpoint,
-            l0stack.L0StackShape(logical_table_size_mib=50, delta_stack_height=l0_stack_height),
-        )
-        return env
-
-    env = neon_env_builder.build_and_use_snapshot(
-        f"test_page_service_batching--test_pagebench-{l0_stack_height}", build_snapshot_cb
-    )
-
     def patch_ps_config(ps_config):
-        ps_config["page_service_pipelining"] = dataclasses.asdict(pipelining_config)
-        ps_config["get_vectored_concurrent_io"] = {"mode": ps_io_concurrency}
+        if pipelining_config is not None:
+            ps_config["page_service_pipelining"] = dataclasses.asdict(pipelining_config)
 
-    env.pageserver.edit_config_toml(patch_ps_config)
+    neon_env_builder.pageserver_config_override = patch_ps_config
 
-    env.start()
+    env = neon_env_builder.init_start()
+    endpoint = env.endpoints.create_start("main")
+    conn = endpoint.connect()
+    cur = conn.cursor()
 
-    lsn = env.safekeepers[0].get_commit_lsn(env.initial_tenant, env.initial_timeline)
-    ep = env.endpoints.create_start("main", lsn=lsn)
-    data_table_relnode_oid = ep.safe_psql_scalar("SELECT 'data'::regclass::oid")
-    ep.stop_and_destroy()
+    cur.execute("SET max_parallel_workers_per_gather=0")  # disable parallel backends
+    cur.execute("SET effective_io_concurrency=1")
+
+    cur.execute("CREATE EXTENSION IF NOT EXISTS neon;")
+    cur.execute("CREATE EXTENSION IF NOT EXISTS neon_test_utils;")
+
+    log.info("Filling the table")
+    cur.execute("CREATE TABLE t (data char(1000)) with (fillfactor=10)")
+    tablesize = 50 * 1024 * 1024
+    npages = tablesize // (8 * 1024)
+    cur.execute("INSERT INTO t SELECT generate_series(1, %s)", (npages,))
+    # TODO: can we force postgres to do sequential scans?
+
+    cur.close()
+    conn.close()
+
+    wait_for_last_flush_lsn(env, endpoint, env.initial_tenant, env.initial_timeline)
+
+    endpoint.stop()
 
     for sk in env.safekeepers:
         sk.stop()
 
+    #
+    # Run single-threaded pagebench (TODO: dedup with other benchmark code)
+    #
+
     env.pageserver.allowed_errors.append(
         # https://github.com/neondatabase/neon/issues/6925
         r".*query handler for.*pagestream.*failed: unexpected message: CopyFail during COPY.*"
@@ -395,8 +396,6 @@ def test_random_reads(
 
     ps_http = env.pageserver.http_client()
 
-    metrics_before = ps_http.get_metrics()
-
     cmd = [
         str(env.neon_binpath / "pagebench"),
         "get-page-latest-lsn",
@@ -406,10 +405,6 @@ def test_random_reads(
         env.pageserver.connstr(password=None),
         "--num-clients",
         "1",
-        "--queue-depth",
-        str(queue_depth),
-        "--only-relnode",
-        str(data_table_relnode_oid),
         "--runtime",
         "10s",
     ]
@@ -418,22 +413,12 @@ def test_random_reads(
     results_path = Path(basepath + ".stdout")
     log.info(f"Benchmark results at: {results_path}")
 
-    metrics_after = ps_http.get_metrics()
-
     with open(results_path) as f:
         results = json.load(f)
     log.info(f"Results:\n{json.dumps(results, sort_keys=True, indent=2)}")
 
     total = results["total"]
 
-    metric = "request_count"
-    zenbenchmark.record(
-        metric,
-        metric_value=total[metric],
-        unit="",
-        report=MetricReport.HIGHER_IS_BETTER,
-    )
-
     metric = "latency_mean"
     zenbenchmark.record(
         metric,
@@ -450,17 +435,3 @@ def test_random_reads(
             unit="ms",
             report=MetricReport.LOWER_IS_BETTER,
         )
-
-    reads_before = metrics_before.query_one(
-        "pageserver_io_operations_seconds_count", filter={"operation": "read"}
-    )
-    reads_after = metrics_after.query_one(
-        "pageserver_io_operations_seconds_count", filter={"operation": "read"}
-    )
-
-    zenbenchmark.record(
-        "virtual_file_reads",
-        metric_value=reads_after.value - reads_before.value,
-        unit="",
-        report=MetricReport.LOWER_IS_BETTER,
-    )

test_runner/regress/test_import_pgdata.py

@@ -130,8 +130,9 @@ def test_pgdata_import_smoke(
     elif rel_block_size == RelBlockSize.TWO_STRPES_PER_SHARD:
         target_relblock_size = (shard_count or 1) * stripe_size * 8192 * 2
     elif rel_block_size == RelBlockSize.MULTIPLE_RELATION_SEGMENTS:
-        segment_size = 16 * 1024 * 1024
-        target_relblock_size = segment_size * 8
+        # Postgres uses a 1GiB segment size, fixed at compile time, so we must use >2GB of data
+        # to exercise multiple segments.
+        target_relblock_size = int(((2.333 * 1024 * 1024 * 1024) // 8192) * 8192)
     else:
         raise ValueError
 
@@ -412,88 +413,6 @@ def test_import_completion_on_restart(
     wait_until(cplane_notified)
 
 
-@run_only_on_default_postgres(reason="PG version is irrelevant here")
-def test_import_respects_tenant_shutdown(
-    neon_env_builder: NeonEnvBuilder, vanilla_pg: VanillaPostgres, make_httpserver: HTTPServer
-):
-    """
-    Validate that importing timelines respect the usual timeline life cycle:
-    1. Shut down on tenant shut-down and resumes upon re-attach
-    2. Deletion on timeline deletion (TODO)
-    """
-    # Set up mock control plane HTTP server to listen for import completions
-    import_completion_signaled = Event()
-
-    def handler(request: Request) -> Response:
-        log.info(f"control plane /import_complete request: {request.json}")
-        import_completion_signaled.set()
-        return Response(json.dumps({}), status=200)
-
-    cplane_mgmt_api_server = make_httpserver
-    cplane_mgmt_api_server.expect_request(
-        "/storage/api/v1/import_complete", method="PUT"
-    ).respond_with_handler(handler)
-
-    # Plug the cplane mock in
-    neon_env_builder.control_plane_hooks_api = (
-        f"http://{cplane_mgmt_api_server.host}:{cplane_mgmt_api_server.port}/storage/api/v1/"
-    )
-
-    # The import will specifiy a local filesystem path mocking remote storage
-    neon_env_builder.enable_pageserver_remote_storage(RemoteStorageKind.LOCAL_FS)
-
-    vanilla_pg.start()
-    vanilla_pg.stop()
-
-    env = neon_env_builder.init_configs()
-    env.start()
-
-    importbucket_path = neon_env_builder.repo_dir / "test_import_completion_bucket"
-    mock_import_bucket(vanilla_pg, importbucket_path)
-
-    tenant_id = TenantId.generate()
-    timeline_id = TimelineId.generate()
-    idempotency = ImportPgdataIdemptencyKey.random()
-
-    # Pause before sending the notification
-    failpoint_name = "import-timeline-pre-execute-pausable"
-    env.pageserver.http_client().configure_failpoints((failpoint_name, "pause"))
-
-    env.storage_controller.tenant_create(tenant_id)
-    env.storage_controller.timeline_create(
-        tenant_id,
-        {
-            "new_timeline_id": str(timeline_id),
-            "import_pgdata": {
-                "idempotency_key": str(idempotency),
-                "location": {"LocalFs": {"path": str(importbucket_path.absolute())}},
-            },
-        },
-    )
-
-    def hit_failpoint():
-        log.info("Checking log for pattern...")
-        try:
-            assert env.pageserver.log_contains(f".*at failpoint {failpoint_name}.*")
-        except Exception:
-            log.exception("Failed to find pattern in log")
-            raise
-
-    wait_until(hit_failpoint)
-    assert not import_completion_signaled.is_set()
-
-    # Restart the pageserver while an import job is in progress.
-    # This clears the failpoint and we expect that the import starts up afresh
-    # after the restart and eventually completes.
-    env.pageserver.stop()
-    env.pageserver.start()
-
-    def cplane_notified():
-        assert import_completion_signaled.is_set()
-
-    wait_until(cplane_notified)
-
-
 def test_fast_import_with_pageserver_ingest(
     test_output_dir,
     vanilla_pg: VanillaPostgres,
@@ -601,9 +520,7 @@ def test_fast_import_with_pageserver_ingest(
     env.neon_cli.mappings_map_branch(import_branch_name, tenant_id, timeline_id)
 
     # Run fast_import
-    fast_import.set_aws_creds(
-        mock_s3_server, {"RUST_LOG": "info,aws_config=debug,aws_sdk_kms=debug"}
-    )
+    fast_import.set_aws_creds(mock_s3_server, {"RUST_LOG": "aws_config=debug,aws_sdk_kms=debug"})
     pg_port = port_distributor.get_port()
     fast_import.run_pgdata(pg_port=pg_port, s3prefix=f"s3://{bucket}/{key_prefix}")