Add walproposer vs vanilla replication test

2026-07-08 06:30:37 +00:00 · 2022-04-02 01:53:34 +04:00
59 changed files with 1444 additions and 5432 deletions
--- a/.circleci/ansible/deploy.yaml
+++ b/.circleci/ansible/deploy.yaml
@@ -116,30 +116,6 @@

  tasks:

-    - name: upload init script
-      when: console_mgmt_base_url is defined
-      ansible.builtin.template:
-        src: scripts/init_safekeeper.sh
-        dest: /tmp/init_safekeeper.sh
-        owner: root
-        group: root
-        mode: '0755'
-      become: true
-      tags:
-      - safekeeper
-
-    - name: init safekeeper
-      shell:
-        cmd: /tmp/init_safekeeper.sh
-      args:
-        creates: "/storage/safekeeper/data/safekeeper.id"
-      environment:
-        ZENITH_REPO_DIR: "/storage/safekeeper/data"
-        LD_LIBRARY_PATH: "/usr/local/lib"
-      become: true
-      tags:
-      - safekeeper
-
    # in the future safekeepers should discover pageservers byself
    # but currently use first pageserver that was discovered
    - name: set first pageserver var for safekeepers
--- a/.circleci/ansible/scripts/init_safekeeper.sh
+++ b/.circleci/ansible/scripts/init_safekeeper.sh
@@ -1,30 +0,0 @@
-#!/bin/sh
-
-# get instance id from meta-data service
-INSTANCE_ID=$(curl -s http://169.254.169.254/latest/meta-data/instance-id)
-
-# store fqdn hostname in var
-HOST=$(hostname -f)
-
-
-cat <<EOF | tee /tmp/payload
-{
-  "version": 1,
-  "host": "${HOST}",
-  "port": 6500,
-  "region_id": {{ console_region_id }},
-  "instance_id": "${INSTANCE_ID}",
-  "http_host": "${HOST}",
-  "http_port": 7676
-}
-EOF
-
-# check if safekeeper already registered or not
-if ! curl -sf -X PATCH -d '{}' {{ console_mgmt_base_url }}/api/v1/safekeepers/${INSTANCE_ID} -o /dev/null; then
-
-    # not registered, so register it now
-    ID=$(curl -sf -X POST {{ console_mgmt_base_url }}/api/v1/safekeepers -d@/tmp/payload | jq -r '.ID')
-
-    # init safekeeper
-    sudo -u safekeeper /usr/local/bin/safekeeper --id ${ID} --init -D /storage/safekeeper/data
-fi
--- a/.circleci/ansible/staging.hosts
+++ b/.circleci/ansible/staging.hosts
@@ -6,7 +6,6 @@ zenith-us-stage-ps-2 console_region_id=27
 zenith-us-stage-sk-1 console_region_id=27
 zenith-us-stage-sk-2 console_region_id=27
 zenith-us-stage-sk-3 console_region_id=27
-zenith-us-stage-sk-4 console_region_id=27

 [storage:children]
 pageservers
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -17,6 +17,12 @@ version = "1.0.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"

+[[package]]
+name = "ahash"
+version = "0.4.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
+
 [[package]]
 name = "ahash"
 version = "0.7.6"
@@ -71,9 +77,9 @@ dependencies = [

 [[package]]
 name = "async-stream"
-version = "0.3.3"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dad5c83079eae9969be7fadefe640a1c566901f05ff91ab221de4b6f68d9507e"
+checksum = "171374e7e3b2504e0e5236e3b59260560f9fe94bfe9ac39ba5e4e929c5590625"
 dependencies = [
 "async-stream-impl",
 "futures-core",
@@ -81,9 +87,9 @@ dependencies = [

 [[package]]
 name = "async-stream-impl"
-version = "0.3.3"
+version = "0.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "10f203db73a71dfa2fb6dd22763990fa26f3d2625a6da2da900d23b87d26be27"
+checksum = "648ed8c8d2ce5409ccd57453d9d1b214b342a0d69376a6feda1fd6cae3299308"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -101,6 +107,23 @@ dependencies = [
 "syn",
 ]

+[[package]]
+name = "attohttpc"
+version = "0.18.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e69e13a99a7e6e070bb114f7ff381e58c7ccc188630121fc4c2fe4bcf24cd072"
+dependencies = [
+ "http",
+ "log",
+ "rustls 0.20.2",
+ "serde",
+ "serde_json",
+ "url",
+ "webpki 0.22.0",
+ "webpki-roots",
+ "wildmatch",
+]
+
 [[package]]
 name = "atty"
 version = "0.2.14"
@@ -118,6 +141,55 @@ version = "1.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"

+[[package]]
+name = "aversion"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41992ab8cfcc3026ef9abceffe0c2b0479c043183fc23825e30d22baab6df334"
+dependencies = [
+ "aversion-macros",
+ "byteorder",
+ "serde",
+ "serde_cbor",
+ "thiserror",
+]
+
+[[package]]
+name = "aversion-macros"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ba5785f953985aa0caca927ba4005880f3b4f53de87f134e810ae3549f744d2"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "aws-creds"
+version = "0.27.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "460a75eac8f3cb7683e0a9a588a83c3ff039331ea7bfbfbfcecf1dacab276e11"
+dependencies = [
+ "anyhow",
+ "attohttpc",
+ "dirs",
+ "rust-ini",
+ "serde",
+ "serde-xml-rs",
+ "serde_derive",
+ "url",
+]
+
+[[package]]
+name = "aws-region"
+version = "0.23.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4e37c2dc2c9047311911ef175e0ffbb3853f17c32b72cf3d562f455e5ff77267"
+dependencies = [
+ "anyhow",
+]
+
 [[package]]
 name = "backtrace"
 version = "0.3.64"
@@ -192,6 +264,17 @@ dependencies = [
 "generic-array",
 ]

+[[package]]
+name = "bookfile"
+version = "0.3.0"
+source = "git+https://github.com/zenithdb/bookfile.git?rev=bf6e43825dfb6e749ae9b80e8372c8fea76cec2f#bf6e43825dfb6e749ae9b80e8372c8fea76cec2f"
+dependencies = [
+ "aversion",
+ "byteorder",
+ "serde",
+ "thiserror",
+]
+
 [[package]]
 name = "boxfnonce"
 version = "0.1.1"
@@ -273,7 +356,6 @@ dependencies = [
 "libc",
 "num-integer",
 "num-traits",
- "serde",
 "time",
 "winapi",
 ]
@@ -391,22 +473,6 @@ dependencies = [
 "zenith_utils",
 ]

-[[package]]
-name = "core-foundation"
-version = "0.9.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
-
-[[package]]
-name = "core-foundation-sys"
-version = "0.8.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
-
 [[package]]
 name = "cpufeatures"
 version = "0.2.1"
@@ -425,15 +491,6 @@ dependencies = [
 "rustc_version",
 ]

-[[package]]
-name = "crc32fast"
-version = "1.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
-dependencies = [
- "cfg-if",
-]
-
 [[package]]
 name = "criterion"
 version = "0.3.5"
@@ -472,9 +529,9 @@ dependencies = [

 [[package]]
 name = "crossbeam-channel"
-version = "0.5.4"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5aaa7bd5fb665c6864b5f963dd9097905c54125909c7aa94c9e18507cdbe6c53"
+checksum = "e54ea8bc3fb1ee042f5aace6e3c6e025d3874866da222930f70ce62aceba0bfa"
 dependencies = [
 "cfg-if",
 "crossbeam-utils",
@@ -493,11 +550,10 @@ dependencies = [

 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.8"
+version = "0.9.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1145cf131a2c6ba0615079ab6a638f7e1973ac9c2634fcbeaaad6114246efe8c"
+checksum = "c00d6d2ea26e8b151d99093005cb442fb9a37aeaca582a03ec70946f49ab5ed9"
 dependencies = [
- "autocfg",
 "cfg-if",
 "crossbeam-utils",
 "lazy_static",
@@ -612,26 +668,34 @@ dependencies = [
 ]

 [[package]]
-name = "dirs-next"
-version = "2.0.0"
+name = "dirs"
+version = "4.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
+checksum = "ca3aa72a6f96ea37bbc5aa912f6788242832f75369bdfdadcb0e38423f100059"
 dependencies = [
- "cfg-if",
- "dirs-sys-next",
+ "dirs-sys",
 ]

 [[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
+name = "dirs-sys"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+checksum = "03d86534ed367a67548dc68113a0f5db55432fdfbb6e6f9d77704397d95d5780"
 dependencies = [
 "libc",
 "redox_users",
 "winapi",
 ]

+[[package]]
+name = "dlv-list"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68df3f2b690c1b86e65ef7830956aededf3cb0a16f898f79b9a6f421a7b6211b"
+dependencies = [
+ "rand",
+]
+
 [[package]]
 name = "either"
 version = "1.6.1"
@@ -725,21 +789,6 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"

-[[package]]
-name = "foreign-types"
-version = "0.3.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
-dependencies = [
- "foreign-types-shared",
-]
-
-[[package]]
-name = "foreign-types-shared"
-version = "0.1.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
-
 [[package]]
 name = "form_urlencoded"
 version = "1.0.1"
@@ -929,13 +978,22 @@ version = "1.8.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "eabb4a44450da02c90444cf74558da904edde8fb4e9035a9a6a4e15445af0bd7"

+[[package]]
+name = "hashbrown"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
+dependencies = [
+ "ahash 0.4.7",
+]
+
 [[package]]
 name = "hashbrown"
 version = "0.11.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
 dependencies = [
- "ahash",
+ "ahash 0.7.6",
 ]

 [[package]]
@@ -1033,9 +1091,9 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"

 [[package]]
 name = "hyper"
-version = "0.14.17"
+version = "0.14.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "043f0e083e9901b6cc658a77d1eb86f4fc650bbb977a4337dd63192826aa85dd"
+checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55"
 dependencies = [
 "bytes",
 "futures-channel",
@@ -1046,7 +1104,7 @@ dependencies = [
 "http-body",
 "httparse",
 "httpdate",
- "itoa 1.0.1",
+ "itoa 0.4.8",
 "pin-project-lite",
 "socket2",
 "tokio",
@@ -1080,19 +1138,6 @@ dependencies = [
 "tokio-io-timeout",
 ]

-[[package]]
-name = "hyper-tls"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
-dependencies = [
- "bytes",
- "hyper",
- "native-tls",
- "tokio",
- "tokio-native-tls",
-]
-
 [[package]]
 name = "ident_case"
 version = "1.0.1"
@@ -1117,7 +1162,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "282a6247722caba404c065016bbfa522806e51714c34f5dfc3e4a3a46fcb4223"
 dependencies = [
 "autocfg",
- "hashbrown",
+ "hashbrown 0.11.2",
 ]

 [[package]]
@@ -1259,6 +1304,17 @@ version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"

+[[package]]
+name = "maybe-async"
+version = "0.2.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6007f9dad048e0a224f27ca599d669fca8cfa0dac804725aab542b2eb032bce6"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "md-5"
 version = "0.9.1"
@@ -1342,24 +1398,6 @@ version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a"

-[[package]]
-name = "native-tls"
-version = "0.2.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48ba9f7719b5a0f42f338907614285fb5fd70e53858141f69898a1fb7203b24d"
-dependencies = [
- "lazy_static",
- "libc",
- "log",
- "openssl",
- "openssl-probe",
- "openssl-sys",
- "schannel",
- "security-framework",
- "security-framework-sys",
- "tempfile",
-]
-
 [[package]]
 name = "nix"
 version = "0.23.1"
@@ -1386,9 +1424,9 @@ dependencies = [

 [[package]]
 name = "ntapi"
-version = "0.3.7"
+version = "0.3.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c28774a7fd2fbb4f0babd8237ce554b73af68021b5f695a3cebd6c59bac0980f"
+checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44"
 dependencies = [
 "winapi",
 ]
@@ -1461,36 +1499,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"

 [[package]]
-name = "openssl"
-version = "0.10.38"
+name = "ordered-multimap"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c7ae222234c30df141154f159066c5093ff73b63204dcda7121eb082fc56a95"
+checksum = "1c672c7ad9ec066e428c00eb917124a06f08db19e2584de982cc34b1f4c12485"
 dependencies = [
- "bitflags",
- "cfg-if",
- "foreign-types",
- "libc",
- "once_cell",
- "openssl-sys",
-]
-
-[[package]]
-name = "openssl-probe"
-version = "0.1.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
-
-[[package]]
-name = "openssl-sys"
-version = "0.9.72"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e46109c383602735fa0a2e48dd2b7c892b048e1bf69e5c3b1d804b7d9c203cb"
-dependencies = [
- "autocfg",
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
+ "dlv-list",
+ "hashbrown 0.9.1",
 ]

 [[package]]
@@ -1509,6 +1524,7 @@ dependencies = [
 "anyhow",
 "async-compression",
 "async-trait",
+ "bookfile",
 "byteorder",
 "bytes",
 "chrono",
@@ -1519,7 +1535,6 @@ dependencies = [
 "daemonize",
 "fail",
 "futures",
- "hex",
 "hex-literal",
 "humantime",
 "hyper",
@@ -1534,8 +1549,7 @@ dependencies = [
 "postgres_ffi",
 "rand",
 "regex",
- "rusoto_core",
- "rusoto_s3",
+ "rust-s3",
 "scopeguard",
 "serde",
 "serde_json",
@@ -1547,7 +1561,6 @@ dependencies = [
 "tokio",
 "tokio-postgres 0.7.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=2949d98df52587d562986aad155dd4e889e408b7)",
 "tokio-stream",
- "tokio-util 0.7.0",
 "toml_edit",
 "tracing",
 "tracing-futures",
@@ -1674,12 +1687,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"

-[[package]]
-name = "pkg-config"
-version = "0.3.24"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
-
 [[package]]
 name = "plotters"
 version = "0.3.1"
@@ -1911,7 +1918,7 @@ dependencies = [
 "clap 3.0.14",
 "fail",
 "futures",
- "hashbrown",
+ "hashbrown 0.11.2",
 "hex",
 "hyper",
 "lazy_static",
@@ -2144,85 +2151,43 @@ dependencies = [
 ]

 [[package]]
-name = "rusoto_core"
-version = "0.47.0"
+name = "rust-ini"
+version = "0.17.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b4f000e8934c1b4f70adde180056812e7ea6b1a247952db8ee98c94cd3116cc"
+checksum = "63471c4aa97a1cf8332a5f97709a79a4234698de6a1f5087faf66f2dae810e22"
 dependencies = [
+ "cfg-if",
+ "ordered-multimap",
+]
+
+[[package]]
+name = "rust-s3"
+version = "0.28.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dc0e521d1084d6950e050d4e2595f0fbdaa2b96bb795bab3d90a282288c5e49"
+dependencies = [
+ "anyhow",
 "async-trait",
+ "aws-creds",
+ "aws-region",
 "base64 0.13.0",
- "bytes",
- "crc32fast",
- "futures",
- "http",
- "hyper",
- "hyper-tls",
- "lazy_static",
- "log",
- "rusoto_credential",
- "rusoto_signature",
- "rustc_version",
- "serde",
- "serde_json",
- "tokio",
- "xml-rs",
-]
-
-[[package]]
-name = "rusoto_credential"
-version = "0.47.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a46b67db7bb66f5541e44db22b0a02fed59c9603e146db3a9e633272d3bac2f"
-dependencies = [
- "async-trait",
+ "cfg-if",
 "chrono",
- "dirs-next",
- "futures",
- "hyper",
- "serde",
- "serde_json",
- "shlex",
- "tokio",
- "zeroize",
-]
-
-[[package]]
-name = "rusoto_s3"
-version = "0.47.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "048c2fe811a823ad5a9acc976e8bf4f1d910df719dcf44b15c3e96c5b7a51027"
-dependencies = [
- "async-trait",
- "bytes",
- "futures",
- "rusoto_core",
- "xml-rs",
-]
-
-[[package]]
-name = "rusoto_signature"
-version = "0.47.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6264e93384b90a747758bcc82079711eacf2e755c3a8b5091687b5349d870bcc"
-dependencies = [
- "base64 0.13.0",
- "bytes",
- "chrono",
- "digest",
- "futures",
 "hex",
 "hmac 0.11.0",
 "http",
- "hyper",
 "log",
- "md-5",
+ "maybe-async",
+ "md5",
 "percent-encoding",
- "pin-project-lite",
- "rusoto_credential",
- "rustc_version",
+ "reqwest",
 "serde",
+ "serde-xml-rs",
+ "serde_derive",
 "sha2",
 "tokio",
+ "tokio-stream",
+ "url",
 ]

 [[package]]
@@ -2310,16 +2275,6 @@ dependencies = [
 "winapi-util",
 ]

-[[package]]
-name = "schannel"
-version = "0.1.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
-dependencies = [
- "lazy_static",
- "winapi",
-]
-
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -2346,29 +2301,6 @@ dependencies = [
 "untrusted",
 ]

-[[package]]
-name = "security-framework"
-version = "2.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2dc14f172faf8a0194a3aded622712b0de276821addc574fa54fc0a1167e10dc"
-dependencies = [
- "bitflags",
- "core-foundation",
- "core-foundation-sys",
- "libc",
- "security-framework-sys",
-]
-
-[[package]]
-name = "security-framework-sys"
-version = "2.6.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0160a13a177a45bfb43ce71c01580998474f556ad854dcbca936dd2841a5c556"
-dependencies = [
- "core-foundation-sys",
- "libc",
-]
-
 [[package]]
 name = "semver"
 version = "1.0.5"
@@ -2384,6 +2316,18 @@ dependencies = [
 "serde_derive",
 ]

+[[package]]
+name = "serde-xml-rs"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65162e9059be2f6a3421ebbb4fef3e74b7d9e7c60c50a0e292c6239f19f1edfa"
+dependencies = [
+ "log",
+ "serde",
+ "thiserror",
+ "xml-rs",
+]
+
 [[package]]
 name = "serde_cbor"
 version = "0.11.2"
@@ -2736,16 +2680,6 @@ dependencies = [
 "syn",
 ]

-[[package]]
-name = "tokio-native-tls"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
-dependencies = [
- "native-tls",
- "tokio",
-]
-
 [[package]]
 name = "tokio-postgres"
 version = "0.7.1"
@@ -3105,12 +3039,6 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "830b7e5d4d90034032940e4ace0d9a9a057e7a45cd94e6c007832e39edb82f6d"

-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
 [[package]]
 name = "vec_map"
 version = "0.8.2"
@@ -3155,8 +3083,7 @@ dependencies = [
 "postgres-protocol 0.6.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=2949d98df52587d562986aad155dd4e889e408b7)",
 "postgres_ffi",
 "regex",
- "rusoto_core",
- "rusoto_s3",
+ "rust-s3",
 "serde",
 "serde_json",
 "serde_with",
@@ -3164,7 +3091,6 @@ dependencies = [
 "tempfile",
 "tokio",
 "tokio-postgres 0.7.1 (git+https://github.com/zenithdb/rust-postgres.git?rev=2949d98df52587d562986aad155dd4e889e408b7)",
- "tokio-util 0.7.0",
 "tracing",
 "url",
 "walkdir",
@@ -3311,6 +3237,12 @@ dependencies = [
 "libc",
 ]

+[[package]]
+name = "wildmatch"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6c48bd20df7e4ced539c12f570f937c6b4884928a87fee70a479d72f031d4e0"
+
 [[package]]
 name = "winapi"
 version = "0.3.9"
@@ -3360,7 +3292,7 @@ dependencies = [
 "cc",
 "clap 2.34.0",
 "either",
- "hashbrown",
+ "hashbrown 0.11.2",
 "libc",
 "log",
 "memchr",
@@ -3467,12 +3399,6 @@ dependencies = [
 "zenith_metrics",
 ]

-[[package]]
-name = "zeroize"
-version = "1.5.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7c88870063c39ee00ec285a2f8d6a966e5b6fb2becc4e8dac77ed0d370ed6006"
-
 [[package]]
 name = "zstd"
 version = "0.10.0+zstd.1.5.2"
--- a/pageserver/Cargo.toml
+++ b/pageserver/Cargo.toml
@@ -4,13 +4,13 @@ version = "0.1.0"
 edition = "2021"

 [dependencies]
+bookfile = { git = "https://github.com/zenithdb/bookfile.git", rev="bf6e43825dfb6e749ae9b80e8372c8fea76cec2f" }
 chrono = "0.4.19"
 rand = "0.8.3"
 regex = "1.4.5"
 bytes = { version = "1.0.1", features = ['serde'] }
 byteorder = "1.4.3"
 futures = "0.3.13"
-hex = "0.4.3"
 hyper = "0.14"
 itertools = "0.10.3"
 lazy_static = "1.4.0"
@@ -18,7 +18,6 @@ log = "0.4.14"
 clap = "3.0"
 daemonize = "0.4.1"
 tokio = { version = "1.17", features = ["process", "sync", "macros", "fs", "rt", "io-util", "time"] }
-tokio-util = { version = "0.7", features = ["io"] }
 postgres-types = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
@@ -35,6 +34,7 @@ serde_with = "1.12.0"

 toml_edit = { version = "0.13", features = ["easy"] }
 scopeguard = "1.1.0"
+async-trait = "0.1"
 const_format = "0.2.21"
 tracing = "0.1.27"
 tracing-futures = "0.2"
@@ -45,9 +45,7 @@ once_cell = "1.8.0"
 crossbeam-utils = "0.8.5"
 fail = "0.5.0"

-rusoto_core = "0.47"
-rusoto_s3 = "0.47"
-async-trait = "0.1"
+rust-s3 = { version = "0.28", default-features = false, features = ["no-verify-ssl", "tokio-rustls-tls"] }
 async-compression = {version = "0.3", features = ["zstd", "tokio"]}

 postgres_ffi = { path = "../postgres_ffi" }
--- a/pageserver/src/bin/dump_layerfile.rs
+++ b/pageserver/src/bin/dump_layerfile.rs
@@ -4,7 +4,6 @@
 use anyhow::Result;
 use clap::{App, Arg};
 use pageserver::layered_repository::dump_layerfile_from_path;
-use pageserver::page_cache;
 use pageserver::virtual_file;
 use std::path::PathBuf;
 use zenith_utils::GIT_VERSION;
@@ -25,7 +24,6 @@ fn main() -> Result<()> {

    // Basic initialization of things that don't change after startup
    virtual_file::init(10);
-    page_cache::init(100);

    dump_layerfile_from_path(&path, true)?;

--- a/pageserver/src/bin/pageserver.rs
+++ b/pageserver/src/bin/pageserver.rs
@@ -185,9 +185,6 @@ fn start_pageserver(conf: &'static PageServerConf, daemonize: bool) -> Result<()
    // Initialize logger
    let log_file = logging::init(LOG_FILE_NAME, daemonize)?;

-    // TODO init only if configured
-    pageserver::wal_metadata::init(conf).expect("wal_metadata init failed");
-
    info!("version: {}", GIT_VERSION);

    // TODO: Check that it looks like a valid repository before going further
--- a/pageserver/src/bin/psbench.rs
+++ b/pageserver/src/bin/psbench.rs
@@ -1,158 +0,0 @@
-//! Pageserver benchmark tool
-//!
-//! Usually it's easier to write python perf tests, but here the performance
-//! of the tester matters, and the API is easier to work with from rust.
-use std::{collections::HashSet, io::{BufRead, BufReader, Cursor}, time::Duration};
-use pageserver::wal_metadata::{Page, WalEntryMetadata};
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use bytes::{BufMut, BytesMut};
-use clap::{App, Arg};
-use std::fs::File;
-use zenith_utils::{GIT_VERSION, lsn::Lsn, pq_proto::{BeMessage, FeMessage}};
-use std::time::Instant;
-
-use anyhow::Result;
-
-const BYTES_IN_PAGE: usize = 8 * 1024;
-
-pub fn read_lines_buffered(file_name: &str) -> impl Iterator<Item = String> {
-    BufReader::new(File::open(file_name).unwrap())
-        .lines()
-        .map(|result| result.unwrap())
-}
-
-pub async fn get_page(
-    pagestream: &mut tokio::net::TcpStream,
-    lsn: &Lsn,
-    page: &Page,
-    latest: bool,
-) -> anyhow::Result<Vec<u8>> {
-    let latest: u8 = if latest {1} else {0};
-    let msg = {
-        let query = {
-            let mut query = BytesMut::new();
-            query.put_u8(2);  // Specifies get_page query
-            query.put_u8(latest);
-            query.put_u64(lsn.0);
-            page.write(&mut query).await?;
-            query.freeze()
-        };
-
-        let mut buf = BytesMut::new();
-        let copy_msg = BeMessage::CopyData(&query);
-        BeMessage::write(&mut buf, &copy_msg)?;
-        buf.freeze()
-    };
-
-    pagestream.write(&msg).await?;
-
-    let response = match FeMessage::read_fut(pagestream).await? {
-        Some(FeMessage::CopyData(page)) => page,
-        r => panic!("Expected CopyData message, got: {:?}", r),
-    };
-
-    let page = {
-        let mut cursor = Cursor::new(response);
-        let tag = AsyncReadExt::read_u8(&mut cursor).await?;
-
-        match tag {
-            102 => {
-                let mut page = Vec::<u8>::new();
-                cursor.read_to_end(&mut page).await?;
-                if page.len() != BYTES_IN_PAGE {
-                    panic!("Expected 8kb page, got: {:?}", page.len());
-                }
-                page
-            },
-            103 => {
-                let mut bytes = Vec::<u8>::new();
-                cursor.read_to_end(&mut bytes).await?;
-                let message = String::from_utf8(bytes)?;
-                panic!("Got error message: {}", message);
-            },
-            _ => panic!("Unhandled tag {:?}", tag)
-        }
-    };
-
-    Ok(page)
-}
-
-#[tokio::main]
-async fn main() -> Result<()> {
-    let arg_matches = App::new("LALALA")
-        .about("lalala")
-        .version(GIT_VERSION)
-        .arg(
-            Arg::new("wal_metadata_file")
-                .help("Path to wal metadata file")
-                .required(true)
-                .index(1),
-        )
-        .arg(
-            Arg::new("tenant_hex")
-                .help("TODO")
-                .required(true)
-                .index(2),
-        )
-        .arg(
-            Arg::new("timeline")
-                .help("TODO")
-                .required(true)
-                .index(3),
-        )
-        .get_matches();
-
-    let metadata_file = arg_matches.value_of("wal_metadata_file").unwrap();
-    let tenant_hex = arg_matches.value_of("tenant_hex").unwrap();
-    let timeline = arg_matches.value_of("timeline").unwrap();
-
-    // Parse log lines
-    let wal_metadata: Vec<WalEntryMetadata> = read_lines_buffered(metadata_file)
-        .map(|line| serde_json::from_str(&line).expect("corrupt metadata file"))
-        .collect();
-
-    // Get raw TCP connection to the pageserver postgres protocol port
-    let mut socket = tokio::net::TcpStream::connect("localhost:15000").await?;
-    let (client, conn) = tokio_postgres::Config::new()
-        .host("127.0.0.1")
-        .port(15000)
-        .dbname("postgres")
-        .user("zenith_admin")
-        .connect_raw(&mut socket, tokio_postgres::NoTls)
-        .await?;
-
-    // Enter pagestream protocol
-    let init_query = format!("pagestream {} {}", tenant_hex, timeline);
-    tokio::select! {
-        _ = conn => panic!("AAAA"),
-        _ = client.query(init_query.as_str(), &[]) => (),
-    };
-
-    // Derive some variables
-    let total_wal_size: usize = wal_metadata.iter().map(|m| m.size).sum();
-    let affected_pages: HashSet<_> = wal_metadata.iter().map(|m| m.affected_pages.clone())
-        .flatten().collect();
-    let latest_lsn = wal_metadata.iter().map(|m| m.lsn).max().unwrap();
-
-    // Get all latest pages
-    let mut durations: Vec<Duration> = vec![];
-    for page in &affected_pages {
-        let start = Instant::now();
-        let _page_bytes = get_page(&mut socket, &latest_lsn, &page, true).await?;
-        let duration = start.elapsed();
-
-        durations.push(duration);
-    }
-
-    durations.sort();
-    // Results are a space separated table of "metric_name value unit", for ease of parsing
-    println!("test_param num_pages {}", affected_pages.len());
-    println!("test_param num_wal_entries {}", wal_metadata.len());
-    println!("test_param total_wal_size {} bytes", total_wal_size);
-    println!("lower_is_better fastest {:?} microseconds", durations.first().unwrap().as_micros());
-    println!("lower_is_better median {:?} microseconds", durations[durations.len() / 2].as_micros());
-    println!("lower_is_better p99 {:?} microseconds", durations[durations.len() - 1 - durations.len() / 100].as_micros());
-    println!("lower_is_better slowest {:?} microseconds", durations.last().unwrap().as_micros());
-
-    Ok(())
-}
--- a/pageserver/src/config.rs
+++ b/pageserver/src/config.rs
@@ -30,13 +30,8 @@ pub mod defaults {
    // FIXME: This current value is very low. I would imagine something like 1 GB or 10 GB
    // would be more appropriate. But a low value forces the code to be exercised more,
    // which is good for now to trigger bugs.
-    // This parameter actually determines L0 layer file size.
    pub const DEFAULT_CHECKPOINT_DISTANCE: u64 = 256 * 1024 * 1024;

-    // Target file size, when creating image and delta layers.
-    // This parameter determines L1 layer file size.
-    pub const DEFAULT_COMPACTION_TARGET_SIZE: u64 = 128 * 1024 * 1024;
-
    pub const DEFAULT_COMPACTION_PERIOD: &str = "1 s";

    pub const DEFAULT_GC_HORIZON: u64 = 64 * 1024 * 1024;
@@ -63,7 +58,6 @@ pub mod defaults {
 #listen_http_addr = '{DEFAULT_HTTP_LISTEN_ADDR}'

 #checkpoint_distance = {DEFAULT_CHECKPOINT_DISTANCE} # in bytes
-#compaction_target_size = {DEFAULT_COMPACTION_TARGET_SIZE} # in bytes
 #compaction_period = '{DEFAULT_COMPACTION_PERIOD}'

 #gc_period = '{DEFAULT_GC_PERIOD}'
@@ -97,13 +91,8 @@ pub struct PageServerConf {
    // Flush out an inmemory layer, if it's holding WAL older than this
    // This puts a backstop on how much WAL needs to be re-digested if the
    // page server crashes.
-    // This parameter actually determines L0 layer file size.
    pub checkpoint_distance: u64,

-    // Target file size, when creating image and delta layers.
-    // This parameter determines L1 layer file size.
-    pub compaction_target_size: u64,
-
    // How often to check if there's compaction work to be done.
    pub compaction_period: Duration,

@@ -160,7 +149,6 @@ struct PageServerConfigBuilder {

    checkpoint_distance: BuilderValue<u64>,

-    compaction_target_size: BuilderValue<u64>,
    compaction_period: BuilderValue<Duration>,

    gc_horizon: BuilderValue<u64>,
@@ -195,7 +183,6 @@ impl Default for PageServerConfigBuilder {
            listen_pg_addr: Set(DEFAULT_PG_LISTEN_ADDR.to_string()),
            listen_http_addr: Set(DEFAULT_HTTP_LISTEN_ADDR.to_string()),
            checkpoint_distance: Set(DEFAULT_CHECKPOINT_DISTANCE),
-            compaction_target_size: Set(DEFAULT_COMPACTION_TARGET_SIZE),
            compaction_period: Set(humantime::parse_duration(DEFAULT_COMPACTION_PERIOD)
                .expect("cannot parse default compaction period")),
            gc_horizon: Set(DEFAULT_GC_HORIZON),
@@ -233,10 +220,6 @@ impl PageServerConfigBuilder {
        self.checkpoint_distance = BuilderValue::Set(checkpoint_distance)
    }

-    pub fn compaction_target_size(&mut self, compaction_target_size: u64) {
-        self.compaction_target_size = BuilderValue::Set(compaction_target_size)
-    }
-
    pub fn compaction_period(&mut self, compaction_period: Duration) {
        self.compaction_period = BuilderValue::Set(compaction_period)
    }
@@ -307,9 +290,6 @@ impl PageServerConfigBuilder {
            checkpoint_distance: self
                .checkpoint_distance
                .ok_or(anyhow::anyhow!("missing checkpoint_distance"))?,
-            compaction_target_size: self
-                .compaction_target_size
-                .ok_or(anyhow::anyhow!("missing compaction_target_size"))?,
            compaction_period: self
                .compaction_period
                .ok_or(anyhow::anyhow!("missing compaction_period"))?,
@@ -449,9 +429,6 @@ impl PageServerConf {
                "listen_pg_addr" => builder.listen_pg_addr(parse_toml_string(key, item)?),
                "listen_http_addr" => builder.listen_http_addr(parse_toml_string(key, item)?),
                "checkpoint_distance" => builder.checkpoint_distance(parse_toml_u64(key, item)?),
-                "compaction_target_size" => {
-                    builder.compaction_target_size(parse_toml_u64(key, item)?)
-                }
                "compaction_period" => builder.compaction_period(parse_toml_duration(key, item)?),
                "gc_horizon" => builder.gc_horizon(parse_toml_u64(key, item)?),
                "gc_period" => builder.gc_period(parse_toml_duration(key, item)?),
@@ -588,7 +565,6 @@ impl PageServerConf {
        PageServerConf {
            id: ZNodeId(0),
            checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
-            compaction_target_size: 4 * 1024 * 1024,
            compaction_period: Duration::from_secs(10),
            gc_horizon: defaults::DEFAULT_GC_HORIZON,
            gc_period: Duration::from_secs(10),
@@ -660,7 +636,6 @@ listen_http_addr = '127.0.0.1:9898'

 checkpoint_distance = 111 # in bytes

-compaction_target_size = 111 # in bytes
 compaction_period = '111 s'

 gc_period = '222 s'
@@ -698,7 +673,6 @@ id = 10
                listen_pg_addr: defaults::DEFAULT_PG_LISTEN_ADDR.to_string(),
                listen_http_addr: defaults::DEFAULT_HTTP_LISTEN_ADDR.to_string(),
                checkpoint_distance: defaults::DEFAULT_CHECKPOINT_DISTANCE,
-                compaction_target_size: defaults::DEFAULT_COMPACTION_TARGET_SIZE,
                compaction_period: humantime::parse_duration(defaults::DEFAULT_COMPACTION_PERIOD)?,
                gc_horizon: defaults::DEFAULT_GC_HORIZON,
                gc_period: humantime::parse_duration(defaults::DEFAULT_GC_PERIOD)?,
@@ -743,7 +717,6 @@ id = 10
                listen_pg_addr: "127.0.0.1:64000".to_string(),
                listen_http_addr: "127.0.0.1:9898".to_string(),
                checkpoint_distance: 111,
-                compaction_target_size: 111,
                compaction_period: Duration::from_secs(111),
                gc_horizon: 222,
                gc_period: Duration::from_secs(222),
--- a/pageserver/src/http/openapi_spec.yml
+++ b/pageserver/src/http/openapi_spec.yml
@@ -18,7 +18,7 @@ paths:
              schema:
                type: object
                required:
-                  - id
+                - id
                properties:
                  id:
                    type: integer
@@ -122,110 +122,6 @@ paths:
            application/json:
              schema:
                $ref: "#/components/schemas/Error"
-
-
-  /v1/tenant/{tenant_id}/timeline/{timeline_id}/attach:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    post:
-      description: Attach remote timeline
-      responses:
-        "200":
-          description: Timeline attaching scheduled
-        "400":
-          description: Error when no tenant id found in path or no timeline id
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "404":
-          description: Timeline not found
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/NotFoundError"
-        "409":
-          description: Timeline download is already in progress
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ConflictError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-
-
-  /v1/tenant/{tenant_id}/timeline/{timeline_id}/detach:
-    parameters:
-      - name: tenant_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-      - name: timeline_id
-        in: path
-        required: true
-        schema:
-          type: string
-          format: hex
-    post:
-      description: Detach local timeline
-      responses:
-        "200":
-          description: Timeline detached
-        "400":
-          description: Error when no tenant id found in path or no timeline id
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-        "401":
-          description: Unauthorized Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/UnauthorizedError"
-        "403":
-          description: Forbidden Error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/ForbiddenError"
-        "500":
-          description: Generic operation error
-          content:
-            application/json:
-              schema:
-                $ref: "#/components/schemas/Error"
-
-
  /v1/tenant/{tenant_id}/timeline/:
    parameters:
      - name: tenant_id
@@ -283,7 +179,7 @@ paths:
          content:
            application/json:
              schema:
-                $ref: "#/components/schemas/ConflictError"
+                $ref: "#/components/schemas/AlreadyExistsError"
        "500":
          description: Generic operation error
          content:
@@ -364,7 +260,7 @@ paths:
          content:
            application/json:
              schema:
-                $ref: "#/components/schemas/ConflictError"
+                $ref: "#/components/schemas/AlreadyExistsError"
        "500":
          description: Generic operation error
          content:
@@ -458,6 +354,13 @@ components:
      properties:
        msg:
          type: string
+    AlreadyExistsError:
+      type: object
+      required:
+        - msg
+      properties:
+        msg:
+          type: string
    ForbiddenError:
      type: object
      required:
@@ -465,20 +368,6 @@ components:
      properties:
        msg:
          type: string
-    NotFoundError:
-      type: object
-      required:
-        - msg
-      properties:
-        msg:
-          type: string
-    ConflictError:
-      type: object
-      required:
-        - msg
-      properties:
-        msg:
-          type: string

 security:
  - JWT: []
--- a/pageserver/src/http/routes.rs
+++ b/pageserver/src/http/routes.rs
@@ -68,7 +68,10 @@ fn get_config(request: &Request<Body>) -> &'static PageServerConf {
 // healthcheck handler
 async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
    let config = get_config(&request);
-    json_response(StatusCode::OK, StatusResponse { id: config.id })
+    Ok(json_response(
+        StatusCode::OK,
+        StatusResponse { id: config.id },
+    )?)
 }

 async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -128,7 +131,7 @@ async fn timeline_list_handler(request: Request<Body>) -> Result<Response<Body>,
        })
    }

-    json_response(StatusCode::OK, response_data)
+    Ok(json_response(StatusCode::OK, response_data)?)
 }

 // Gate non incremental logical size calculation behind a flag
@@ -204,7 +207,7 @@ async fn timeline_detail_handler(request: Request<Body>) -> Result<Response<Body
        remote: remote_timeline_info,
    };

-    json_response(StatusCode::OK, timeline_info)
+    Ok(json_response(StatusCode::OK, timeline_info)?)
 }

 async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -217,7 +220,6 @@ async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body
    let span = tokio::task::spawn_blocking(move || {
        let entered = span.entered();
        if tenant_mgr::get_timeline_for_tenant_load(tenant_id, timeline_id).is_ok() {
-            // TODO: maybe answer with 309 Not Modified here?
            anyhow::bail!("Timeline is already present locally")
        };
        Ok(entered.exit())
@@ -233,10 +235,10 @@ async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body
            tenant_id,
            timeline_id,
        })
-        .ok_or_else(|| ApiError::NotFound("Unknown remote timeline".to_string()))?;
+        .ok_or_else(|| ApiError::BadRequest("Unknown remote timeline".to_string()))?;

    if index_entry.get_awaits_download() {
-        return Err(ApiError::Conflict(
+        return Err(ApiError::NotFound(
            "Timeline download is already in progress".to_string(),
        ));
    }
@@ -244,7 +246,7 @@ async fn timeline_attach_handler(request: Request<Body>) -> Result<Response<Body
    index_entry.set_awaits_download(true);
    schedule_timeline_download(tenant_id, timeline_id);

-    json_response(StatusCode::ACCEPTED, ())
+    Ok(json_response(StatusCode::ACCEPTED, ())?)
 }

 async fn timeline_detach_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -263,7 +265,7 @@ async fn timeline_detach_handler(request: Request<Body>) -> Result<Response<Body
    .await
    .map_err(ApiError::from_err)??;

-    json_response(StatusCode::OK, ())
+    Ok(json_response(StatusCode::OK, ())?)
 }

 async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -277,7 +279,7 @@ async fn tenant_list_handler(request: Request<Body>) -> Result<Response<Body>, A
    .await
    .map_err(ApiError::from_err)??;

-    json_response(StatusCode::OK, response_data)
+    Ok(json_response(StatusCode::OK, response_data)?)
 }

 async fn tenant_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
--- a/pageserver/src/keyspace.rs
+++ b/pageserver/src/keyspace.rs
@@ -2,6 +2,9 @@ use crate::repository::{key_range_size, singleton_range, Key};
 use postgres_ffi::pg_constants;
 use std::ops::Range;

+// Target file size, when creating image and delta layers
+pub const TARGET_FILE_SIZE_BYTES: u64 = 128 * 1024 * 1024; // 128 MB
+
 ///
 /// Represents a set of Keys, in a compact form.
 ///
--- a/pageserver/src/layered_repository.rs
+++ b/pageserver/src/layered_repository.rs
@@ -12,6 +12,7 @@
 //!

 use anyhow::{anyhow, bail, ensure, Context, Result};
+use bookfile::Book;
 use bytes::Bytes;
 use fail::fail_point;
 use itertools::Itertools;
@@ -33,14 +34,13 @@ use std::time::Instant;

 use self::metadata::{metadata_path, TimelineMetadata, METADATA_FILE_NAME};
 use crate::config::PageServerConf;
-use crate::keyspace::KeySpace;
+use crate::keyspace::{KeyPartitioning, KeySpace};
 use crate::page_cache;
 use crate::remote_storage::{schedule_timeline_checkpoint_upload, RemoteIndex};
 use crate::repository::{
    GcResult, Repository, RepositoryTimeline, Timeline, TimelineSyncStatusUpdate, TimelineWriter,
 };
 use crate::repository::{Key, Value};
-use crate::tenant_mgr;
 use crate::thread_mgr;
 use crate::virtual_file::VirtualFile;
 use crate::walreceiver::IS_WAL_RECEIVER;
@@ -55,10 +55,7 @@ use zenith_utils::crashsafe_dir;
 use zenith_utils::lsn::{AtomicLsn, Lsn, RecordLsn};
 use zenith_utils::seqwait::SeqWait;

-mod blob_io;
-pub mod block_io;
 mod delta_layer;
-mod disk_btree;
 pub(crate) mod ephemeral_file;
 mod filename;
 mod image_layer;
@@ -471,20 +468,18 @@ impl LayeredRepository {
        match timelines.get(&timelineid) {
            Some(entry) => match entry {
                LayeredTimelineEntry::Loaded(local_timeline) => {
-                    debug!("timeline {} found loaded into memory", &timelineid);
+                    trace!("timeline {} found loaded", &timelineid);
                    return Ok(Some(Arc::clone(local_timeline)));
                }
-                LayeredTimelineEntry::Unloaded { .. } => {}
+                LayeredTimelineEntry::Unloaded { .. } => {
+                    trace!("timeline {} found unloaded", &timelineid)
+                }
            },
            None => {
-                debug!("timeline {} not found", &timelineid);
+                trace!("timeline {} not found", &timelineid);
                return Ok(None);
            }
        };
-        debug!(
-            "timeline {} found on a local disk, but not loaded into the memory, loading",
-            &timelineid
-        );
        let timeline = self.load_local_timeline(timelineid, timelines)?;
        let was_loaded = timelines.insert(
            timelineid,
@@ -521,7 +516,9 @@ impl LayeredRepository {
            .context("cannot load ancestor timeline")?
            .flatten()
            .map(LayeredTimelineEntry::Loaded);
-        let _enter = info_span!("loading local timeline").entered();
+        let _enter =
+            info_span!("loading timeline", timeline = %timelineid, tenant = %self.tenantid)
+                .entered();
        let timeline = LayeredTimeline::new(
            self.conf,
            metadata,
@@ -633,8 +630,6 @@ impl LayeredRepository {
        horizon: u64,
        checkpoint_before_gc: bool,
    ) -> Result<GcResult> {
-        let _span_guard =
-            info_span!("gc iteration", tenant = %self.tenantid, timeline = ?target_timelineid);
        let mut totals: GcResult = Default::default();
        let now = Instant::now();

@@ -794,6 +789,8 @@ pub struct LayeredTimeline {
    // garbage collecting data that is still needed by the child timelines.
    gc_info: RwLock<GcInfo>,

+    partitioning: RwLock<Option<(KeyPartitioning, Lsn)>>,
+
    // It may change across major versions so for simplicity
    // keep it after running initdb for a timeline.
    // It is needed in checks when we want to error on some operations
@@ -943,6 +940,14 @@ impl Timeline for LayeredTimeline {
        self.disk_consistent_lsn.load()
    }

+    fn hint_partitioning(&self, partitioning: KeyPartitioning, lsn: Lsn) -> Result<()> {
+        self.partitioning
+            .write()
+            .unwrap()
+            .replace((partitioning, lsn));
+        Ok(())
+    }
+
    fn writer<'a>(&'a self) -> Box<dyn TimelineWriter + 'a> {
        Box::new(LayeredTimelineWriter {
            tl: self,
@@ -1029,6 +1034,7 @@ impl LayeredTimeline {
                retain_lsns: Vec::new(),
                cutoff: Lsn(0),
            }),
+            partitioning: RwLock::new(None),

            latest_gc_cutoff_lsn: RwLock::new(metadata.latest_gc_cutoff_lsn()),
            initdb_lsn: metadata.initdb_lsn(),
@@ -1474,7 +1480,8 @@ impl LayeredTimeline {
        //
        // TODO: This perhaps should be done in 'flush_frozen_layers', after flushing
        // *all* the layers, to avoid fsyncing the file multiple times.
-        let disk_consistent_lsn = Lsn(frozen_layer.get_lsn_range().end.0 - 1);
+        let disk_consistent_lsn;
+        disk_consistent_lsn = Lsn(frozen_layer.get_lsn_range().end.0 - 1);

        // If we were able to advance 'disk_consistent_lsn', save it the metadata file.
        // After crash, we will restart WAL streaming and processing from that point.
@@ -1579,15 +1586,20 @@ impl LayeredTimeline {

        let target_file_size = self.conf.checkpoint_distance;

-        // Define partitioning schema if needed
-        if let Ok(pgdir) = tenant_mgr::get_timeline_for_tenant_load(self.tenantid, self.timelineid)
-        {
-            let (partitioning, lsn) =
-                pgdir.repartition(self.get_last_record_lsn(), self.conf.compaction_target_size)?;
+        // 1. The partitioning was already done by the code in
+        // pgdatadir_mapping.rs. We just use it here.
+        let partitioning_guard = self.partitioning.read().unwrap();
+        if let Some((partitioning, lsn)) = partitioning_guard.as_ref() {
            let timer = self.create_images_time_histo.start_timer();
+            // Make a copy of the partitioning, so that we can release
+            // the lock. Otherwise we could block the WAL receiver.
+            let lsn = *lsn;
+            let parts = partitioning.parts.clone();
+            drop(partitioning_guard);
+
            // 2. Create new image layers for partitions that have been modified
            // "enough".
-            for part in partitioning.parts.iter() {
+            for part in parts.iter() {
                if self.time_for_new_image_layer(part, lsn, 3)? {
                    self.create_image_layer(part, lsn)?;
                }
@@ -1602,6 +1614,15 @@ impl LayeredTimeline {
            debug!("Could not compact because no partitioning specified yet");
        }

+        // Call unload() on all frozen layers, to release memory.
+        // This shouldn't be much memory, as only metadata is slurped
+        // into memory.
+        let layers = self.layers.lock().unwrap();
+        for layer in layers.iter_historic_layers() {
+            layer.unload()?;
+        }
+        drop(layers);
+
        Ok(())
    }

@@ -2046,17 +2067,16 @@ impl<'a> TimelineWriter<'_> for LayeredTimelineWriter<'a> {

 /// Dump contents of a layer file to stdout.
 pub fn dump_layerfile_from_path(path: &Path, verbose: bool) -> Result<()> {
-    use std::os::unix::fs::FileExt;
-
-    // All layer files start with a two-byte "magic" value, to identify the kind of
-    // file.
    let file = File::open(path)?;
-    let mut header_buf = [0u8; 2];
-    file.read_exact_at(&mut header_buf, 0)?;
+    let book = Book::new(file)?;

-    match u16::from_be_bytes(header_buf) {
-        crate::IMAGE_FILE_MAGIC => ImageLayer::new_for_path(path, file)?.dump(verbose)?,
-        crate::DELTA_FILE_MAGIC => DeltaLayer::new_for_path(path, file)?.dump(verbose)?,
+    match book.magic() {
+        crate::DELTA_FILE_MAGIC => {
+            DeltaLayer::new_for_path(path, &book)?.dump(verbose)?;
+        }
+        crate::IMAGE_FILE_MAGIC => {
+            ImageLayer::new_for_path(path, &book)?.dump(verbose)?;
+        }
        magic => bail!("unrecognized magic identifier: {:?}", magic),
    }

@@ -2214,6 +2234,12 @@ pub mod tests {
            }

            let cutoff = tline.get_last_record_lsn();
+            let parts = keyspace
+                .clone()
+                .to_keyspace()
+                .partition(TEST_FILE_SIZE as u64);
+            tline.hint_partitioning(parts.clone(), lsn)?;
+
            tline.update_gc_info(Vec::new(), cutoff);
            tline.checkpoint(CheckpointConfig::Forced)?;
            tline.compact()?;
@@ -2256,6 +2282,9 @@ pub mod tests {
            keyspace.add_key(test_key);
        }

+        let parts = keyspace.to_keyspace().partition(TEST_FILE_SIZE as u64);
+        tline.hint_partitioning(parts, lsn)?;
+
        for _ in 0..50 {
            for _ in 0..NUM_KEYS {
                lsn = Lsn(lsn.0 + 0x10);
@@ -2267,6 +2296,7 @@ pub mod tests {
                    lsn,
                    Value::Image(TEST_IMG(&format!("{} at {}", blknum, lsn))),
                )?;
+                println!("updating {} at {}", blknum, lsn);
                writer.finish_write(lsn);
                drop(writer);
                updated[blknum] = lsn;
@@ -2326,6 +2356,9 @@ pub mod tests {
            keyspace.add_key(test_key);
        }

+        let parts = keyspace.to_keyspace().partition(TEST_FILE_SIZE as u64);
+        tline.hint_partitioning(parts, lsn)?;
+
        let mut tline_id = TIMELINE_ID;
        for _ in 0..50 {
            let new_tline_id = ZTimelineId::generate();
--- a/pageserver/src/layered_repository/blob_io.rs
+++ b/pageserver/src/layered_repository/blob_io.rs
@@ -1,139 +0,0 @@
-//!
-//! Functions for reading and writing variable-sized "blobs".
-//!
-//! Each blob begins with a 4-byte length, followed by the actual data.
-//!
-use crate::layered_repository::block_io::{BlockCursor, BlockReader};
-use crate::page_cache::PAGE_SZ;
-use std::cmp::min;
-use std::io::Error;
-
-/// For reading
-pub trait BlobCursor {
-    /// Read a blob into a new buffer.
-    fn read_blob(&mut self, offset: u64) -> Result<Vec<u8>, std::io::Error> {
-        let mut buf = Vec::new();
-        self.read_blob_into_buf(offset, &mut buf)?;
-        Ok(buf)
-    }
-
-    /// Read blob into the given buffer. Any previous contents in the buffer
-    /// are overwritten.
-    fn read_blob_into_buf(
-        &mut self,
-        offset: u64,
-        dstbuf: &mut Vec<u8>,
-    ) -> Result<(), std::io::Error>;
-}
-
-impl<'a, R> BlobCursor for BlockCursor<R>
-where
-    R: BlockReader,
-{
-    fn read_blob_into_buf(
-        &mut self,
-        offset: u64,
-        dstbuf: &mut Vec<u8>,
-    ) -> Result<(), std::io::Error> {
-        let mut blknum = (offset / PAGE_SZ as u64) as u32;
-        let mut off = (offset % PAGE_SZ as u64) as usize;
-
-        let mut buf = self.read_blk(blknum)?;
-
-        // read length
-        let mut len_buf = [0u8; 4];
-        let thislen = PAGE_SZ - off;
-        if thislen < 4 {
-            // it is split across two pages
-            len_buf[..thislen].copy_from_slice(&buf[off..PAGE_SZ]);
-            blknum += 1;
-            buf = self.read_blk(blknum)?;
-            len_buf[thislen..].copy_from_slice(&buf[0..4 - thislen]);
-            off = 4 - thislen;
-        } else {
-            len_buf.copy_from_slice(&buf[off..off + 4]);
-            off += 4;
-        }
-        let len = u32::from_ne_bytes(len_buf) as usize;
-
-        dstbuf.clear();
-
-        // Read the payload
-        let mut remain = len;
-        while remain > 0 {
-            let mut page_remain = PAGE_SZ - off;
-            if page_remain == 0 {
-                // continue on next page
-                blknum += 1;
-                buf = self.read_blk(blknum)?;
-                off = 0;
-                page_remain = PAGE_SZ;
-            }
-            let this_blk_len = min(remain, page_remain);
-            dstbuf.extend_from_slice(&buf[off..off + this_blk_len]);
-            remain -= this_blk_len;
-            off += this_blk_len;
-        }
-        Ok(())
-    }
-}
-
-///
-/// Abstract trait for a data sink that you can write blobs to.
-///
-pub trait BlobWriter {
-    /// Write a blob of data. Returns the offset that it was written to,
-    /// which can be used to retrieve the data later.
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error>;
-}
-
-///
-/// An implementation of BlobWriter to write blobs to anything that
-/// implements std::io::Write.
-///
-pub struct WriteBlobWriter<W>
-where
-    W: std::io::Write,
-{
-    inner: W,
-    offset: u64,
-}
-
-impl<W> WriteBlobWriter<W>
-where
-    W: std::io::Write,
-{
-    pub fn new(inner: W, start_offset: u64) -> Self {
-        WriteBlobWriter {
-            inner,
-            offset: start_offset,
-        }
-    }
-
-    pub fn size(&self) -> u64 {
-        self.offset
-    }
-
-    /// Access the underlying Write object.
-    ///
-    /// NOTE: WriteBlobWriter keeps track of the current write offset. If
-    /// you write something directly to the inner Write object, it makes the
-    /// internally tracked 'offset' to go out of sync. So don't do that.
-    pub fn into_inner(self) -> W {
-        self.inner
-    }
-}
-
-impl<W> BlobWriter for WriteBlobWriter<W>
-where
-    W: std::io::Write,
-{
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
-        let offset = self.offset;
-        self.inner
-            .write_all(&((srcbuf.len()) as u32).to_ne_bytes())?;
-        self.inner.write_all(srcbuf)?;
-        self.offset += 4 + srcbuf.len() as u64;
-        Ok(offset)
-    }
-}
--- a/pageserver/src/layered_repository/block_io.rs
+++ b/pageserver/src/layered_repository/block_io.rs
@@ -1,219 +0,0 @@
-//!
-//! Low-level Block-oriented I/O functions
-//!
-
-use crate::page_cache;
-use crate::page_cache::{ReadBufResult, PAGE_SZ};
-use bytes::Bytes;
-use lazy_static::lazy_static;
-use std::ops::{Deref, DerefMut};
-use std::os::unix::fs::FileExt;
-use std::sync::atomic::AtomicU64;
-
-/// This is implemented by anything that can read 8 kB (PAGE_SZ)
-/// blocks, using the page cache
-///
-/// There are currently two implementations: EphemeralFile, and FileBlockReader
-/// below.
-pub trait BlockReader {
-    type BlockLease: Deref<Target = [u8; PAGE_SZ]> + 'static;
-
-    ///
-    /// Read a block. Returns a "lease" object that can be used to
-    /// access to the contents of the page. (For the page cache, the
-    /// lease object represents a lock on the buffer.)
-    ///
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error>;
-
-    ///
-    /// Create a new "cursor" for reading from this reader.
-    ///
-    /// A cursor caches the last accessed page, allowing for faster
-    /// access if the same block is accessed repeatedly.
-    fn block_cursor(&self) -> BlockCursor<&Self>
-    where
-        Self: Sized,
-    {
-        BlockCursor::new(self)
-    }
-}
-
-impl<B> BlockReader for &B
-where
-    B: BlockReader,
-{
-    type BlockLease = B::BlockLease;
-
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-        (*self).read_blk(blknum)
-    }
-}
-
-///
-/// A "cursor" for efficiently reading multiple pages from a BlockReader
-///
-/// A cursor caches the last accessed page, allowing for faster access if the
-/// same block is accessed repeatedly.
-///
-/// You can access the last page with `*cursor`. 'read_blk' returns 'self', so
-/// that in many cases you can use a BlockCursor as a drop-in replacement for
-/// the underlying BlockReader. For example:
-///
-/// ```no_run
-/// # use pageserver::layered_repository::block_io::{BlockReader, FileBlockReader};
-/// # let reader: FileBlockReader<std::fs::File> = todo!();
-/// let cursor = reader.block_cursor();
-/// let buf = cursor.read_blk(1);
-/// // do stuff with 'buf'
-/// let buf = cursor.read_blk(2);
-/// // do stuff with 'buf'
-/// ```
-///
-pub struct BlockCursor<R>
-where
-    R: BlockReader,
-{
-    reader: R,
-    /// last accessed page
-    cache: Option<(u32, R::BlockLease)>,
-}
-
-impl<R> BlockCursor<R>
-where
-    R: BlockReader,
-{
-    pub fn new(reader: R) -> Self {
-        BlockCursor {
-            reader,
-            cache: None,
-        }
-    }
-
-    pub fn read_blk(&mut self, blknum: u32) -> Result<&Self, std::io::Error> {
-        // Fast return if this is the same block as before
-        if let Some((cached_blk, _buf)) = &self.cache {
-            if *cached_blk == blknum {
-                return Ok(self);
-            }
-        }
-
-        // Read the block from the underlying reader, and cache it
-        self.cache = None;
-        let buf = self.reader.read_blk(blknum)?;
-        self.cache = Some((blknum, buf));
-
-        Ok(self)
-    }
-}
-
-impl<R> Deref for BlockCursor<R>
-where
-    R: BlockReader,
-{
-    type Target = [u8; PAGE_SZ];
-
-    fn deref(&self) -> &<Self as Deref>::Target {
-        &self.cache.as_ref().unwrap().1
-    }
-}
-
-lazy_static! {
-    static ref NEXT_ID: AtomicU64 = AtomicU64::new(1);
-}
-
-/// An adapter for reading a (virtual) file using the page cache.
-///
-/// The file is assumed to be immutable. This doesn't provide any functions
-/// for modifying the file, nor for invalidating the cache if it is modified.
-pub struct FileBlockReader<F> {
-    pub file: F,
-
-    /// Unique ID of this file, used as key in the page cache.
-    file_id: u64,
-}
-
-impl<F> FileBlockReader<F>
-where
-    F: FileExt,
-{
-    pub fn new(file: F) -> Self {
-        let file_id = NEXT_ID.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
-
-        FileBlockReader { file_id, file }
-    }
-
-    /// Read a page from the underlying file into given buffer.
-    fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), std::io::Error> {
-        assert!(buf.len() == PAGE_SZ);
-        self.file.read_exact_at(buf, blkno as u64 * PAGE_SZ as u64)
-    }
-}
-
-impl<F> BlockReader for FileBlockReader<F>
-where
-    F: FileExt,
-{
-    type BlockLease = page_cache::PageReadGuard<'static>;
-
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-        // Look up the right page
-        let cache = page_cache::get();
-        loop {
-            match cache.read_immutable_buf(self.file_id, blknum) {
-                ReadBufResult::Found(guard) => break Ok(guard),
-                ReadBufResult::NotFound(mut write_guard) => {
-                    // Read the page from disk into the buffer
-                    self.fill_buffer(write_guard.deref_mut(), blknum)?;
-                    write_guard.mark_valid();
-
-                    // Swap for read lock
-                    continue;
-                }
-            };
-        }
-    }
-}
-
-///
-/// Trait for block-oriented output
-///
-pub trait BlockWriter {
-    ///
-    /// Write a page to the underlying storage.
-    ///
-    /// 'buf' must be of size PAGE_SZ. Returns the block number the page was
-    /// written to.
-    ///
-    fn write_blk(&mut self, buf: Bytes) -> Result<u32, std::io::Error>;
-}
-
-///
-/// A simple in-memory buffer of blocks.
-///
-pub struct BlockBuf {
-    pub blocks: Vec<Bytes>,
-}
-impl BlockWriter for BlockBuf {
-    fn write_blk(&mut self, buf: Bytes) -> Result<u32, std::io::Error> {
-        assert!(buf.len() == PAGE_SZ);
-        let blknum = self.blocks.len();
-        self.blocks.push(buf);
-        tracing::info!("buffered block {}", blknum);
-        Ok(blknum as u32)
-    }
-}
-
-impl BlockBuf {
-    pub fn new() -> Self {
-        BlockBuf { blocks: Vec::new() }
-    }
-
-    pub fn size(&self) -> u64 {
-        (self.blocks.len() * PAGE_SZ) as u64
-    }
-}
-impl Default for BlockBuf {
-    fn default() -> Self {
-        Self::new()
-    }
-}
--- a/pageserver/src/layered_repository/delta_layer.rs
+++ b/pageserver/src/layered_repository/delta_layer.rs
@@ -1,14 +1,23 @@
 //! A DeltaLayer represents a collection of WAL records or page images in a range of
 //! LSNs, and in a range of Keys. It is stored on a file on disk.
 //!
-//! Usually a delta layer only contains differences, in the form of WAL records
-//! against a base LSN. However, if a relation extended or a whole new relation
-//! is created, there would be no base for the new pages. The entries for them
-//! must be page images or WAL records with the 'will_init' flag set, so that
-//! they can be replayed without referring to an older page version.
+//! Usually a delta layer only contains differences - in the form of WAL records against
+//! a base LSN. However, if a segment is newly created, by creating a new relation or
+//! extending an old one, there might be no base image. In that case, all the entries in
+//! the delta layer must be page images or WAL records with the 'will_init' flag set, so
+//! that they can be replayed without referring to an older page version. Also in some
+//! circumstances, the predecessor layer might actually be another delta layer. That
+//! can happen when you create a new branch in the middle of a delta layer, and the WAL
+//! records on the new branch are put in a new delta layer.
 //!
-//! The delta files are stored in timelines/<timelineid> directory.  Currently,
-//! there are no subdirectories, and each delta file is named like this:
+//! When a delta file needs to be accessed, we slurp the 'index' metadata
+//! into memory, into the DeltaLayerInner struct. See load() and unload() functions.
+//! To access a particular value, we search `index` for the given key.
+//! The byte offset in the index can be used to find the value in
+//! VALUES_CHAPTER.
+//!
+//! On disk, the delta files are stored in timelines/<timelineid> directory.
+//! Currently, there are no subdirectories, and each delta file is named like this:
 //!
 //!    <key start>-<key end>__<start LSN>-<end LSN
 //!
@@ -16,154 +25,72 @@
 //!
 //!    000000067F000032BE0000400000000020B6-000000067F000032BE0000400000000030B6__000000578C6B29-0000000057A50051
 //!
-//! Every delta file consists of three parts: "summary", "index", and
-//! "values". The summary is a fixed size header at the beginning of the file,
-//! and it contains basic information about the layer, and offsets to the other
-//! parts. The "index" is a B-tree, mapping from Key and LSN to an offset in the
-//! "values" part.  The actual page images and WAL records are stored in the
-//! "values" part.
+//!
+//! A delta file is constructed using the 'bookfile' crate. Each file consists of three
+//! parts: the 'index', the values, and a short summary header. They are stored as
+//! separate chapters.
 //!
 use crate::config::PageServerConf;
-use crate::layered_repository::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter};
-use crate::layered_repository::block_io::{BlockBuf, BlockCursor, BlockReader, FileBlockReader};
-use crate::layered_repository::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::layered_repository::filename::{DeltaFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    BlobRef, Layer, ValueReconstructResult, ValueReconstructState,
 };
-use crate::page_cache::{PageReadGuard, PAGE_SZ};
-use crate::repository::{Key, Value, KEY_SIZE};
+use crate::repository::{Key, Value};
 use crate::virtual_file::VirtualFile;
 use crate::walrecord;
+use crate::DELTA_FILE_MAGIC;
 use crate::{ZTenantId, ZTimelineId};
-use crate::{DELTA_FILE_MAGIC, STORAGE_FORMAT_VERSION};
-use anyhow::{bail, ensure, Context, Result};
+use anyhow::{bail, ensure, Result};
 use log::*;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use zenith_utils::vec_map::VecMap;
 // avoid binding to Write (conflicts with std::io::Write)
 // while being able to use std::fmt::Write's methods
 use std::fmt::Write as _;
 use std::fs;
-use std::io::{BufWriter, Write};
-use std::io::{Seek, SeekFrom};
+use std::io::BufWriter;
+use std::io::Write;
 use std::ops::Range;
 use std::os::unix::fs::FileExt;
 use std::path::{Path, PathBuf};
-use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard};
+use std::sync::{RwLock, RwLockReadGuard, RwLockWriteGuard, TryLockError};
+
+use bookfile::{Book, BookWriter, ChapterWriter};

 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;

-///
-/// Header stored in the beginning of the file
-///
-/// After this comes the 'values' part, starting on block 1. After that,
-/// the 'index' starts at the block indicated by 'index_start_blk'
-///
+/// Mapping from (key, lsn) -> page/WAL record
+/// byte ranges in VALUES_CHAPTER
+static INDEX_CHAPTER: u64 = 1;
+
+/// Page/WAL bytes - cannot be interpreted
+/// without the page versions from the INDEX_CHAPTER
+static VALUES_CHAPTER: u64 = 2;
+
+/// Contains the [`Summary`] struct
+static SUMMARY_CHAPTER: u64 = 3;
+
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 struct Summary {
-    /// Magic value to identify this as a zenith delta file. Always DELTA_FILE_MAGIC.
-    magic: u16,
-    format_version: u16,
-
    tenantid: ZTenantId,
    timelineid: ZTimelineId,
    key_range: Range<Key>,
    lsn_range: Range<Lsn>,
-
-    /// Block number where the 'index' part of the file begins.
-    index_start_blk: u32,
-    /// Block within the 'index', where the B-tree root page is stored
-    index_root_blk: u32,
 }

 impl From<&DeltaLayer> for Summary {
    fn from(layer: &DeltaLayer) -> Self {
        Self {
-            magic: DELTA_FILE_MAGIC,
-            format_version: STORAGE_FORMAT_VERSION,
-
            tenantid: layer.tenantid,
            timelineid: layer.timelineid,
            key_range: layer.key_range.clone(),
            lsn_range: layer.lsn_range.clone(),
-
-            index_start_blk: 0,
-            index_root_blk: 0,
        }
    }
 }

-// Flag indicating that this version initialize the page
-const WILL_INIT: u64 = 1;
-
-///
-/// Struct representing reference to BLOB in layers. Reference contains BLOB
-/// offset, and for WAL records it also contains `will_init` flag. The flag
-/// helps to determine the range of records that needs to be applied, without
-/// reading/deserializing records themselves.
-///
-#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
-struct BlobRef(u64);
-
-impl BlobRef {
-    pub fn will_init(&self) -> bool {
-        (self.0 & WILL_INIT) != 0
-    }
-
-    pub fn pos(&self) -> u64 {
-        self.0 >> 1
-    }
-
-    pub fn new(pos: u64, will_init: bool) -> BlobRef {
-        let mut blob_ref = pos << 1;
-        if will_init {
-            blob_ref |= WILL_INIT;
-        }
-        BlobRef(blob_ref)
-    }
-}
-
-const DELTA_KEY_SIZE: usize = KEY_SIZE + 8;
-struct DeltaKey([u8; DELTA_KEY_SIZE]);
-
-///
-/// This is the key of the B-tree index stored in the delta layer. It consists
-/// of the serialized representation of a Key and LSN.
-///
-impl DeltaKey {
-    fn from_slice(buf: &[u8]) -> Self {
-        let mut bytes: [u8; DELTA_KEY_SIZE] = [0u8; DELTA_KEY_SIZE];
-        bytes.copy_from_slice(buf);
-        DeltaKey(bytes)
-    }
-
-    fn from_key_lsn(key: &Key, lsn: Lsn) -> Self {
-        let mut bytes: [u8; DELTA_KEY_SIZE] = [0u8; DELTA_KEY_SIZE];
-        key.write_to_byte_slice(&mut bytes[0..KEY_SIZE]);
-        bytes[KEY_SIZE..].copy_from_slice(&u64::to_be_bytes(lsn.0));
-        DeltaKey(bytes)
-    }
-
-    fn key(&self) -> Key {
-        Key::from_slice(&self.0)
-    }
-
-    fn lsn(&self) -> Lsn {
-        Lsn(u64::from_be_bytes(self.0[KEY_SIZE..].try_into().unwrap()))
-    }
-
-    fn extract_key_from_buf(buf: &[u8]) -> Key {
-        Key::from_slice(&buf[..KEY_SIZE])
-    }
-
-    fn extract_lsn_from_buf(buf: &[u8]) -> Lsn {
-        let mut lsn_buf = [0u8; 8];
-        lsn_buf.copy_from_slice(&buf[KEY_SIZE..]);
-        Lsn(u64::from_be_bytes(lsn_buf))
-    }
-}
-
 ///
 /// DeltaLayer is the in-memory data structure associated with an
 /// on-disk delta file.  We keep a DeltaLayer in memory for each
@@ -184,15 +111,17 @@ pub struct DeltaLayer {
 }

 pub struct DeltaLayerInner {
-    /// If false, the fields below have not been loaded into memory yet.
+    /// If false, the 'index' has not been loaded into memory yet.
    loaded: bool,

-    // values copied from summary
-    index_start_blk: u32,
-    index_root_blk: u32,
+    ///
+    /// All versions of all pages in the layer are kept here.
+    /// Indexed by block number and LSN. The value is an offset into the
+    /// chapter where the page version is stored.
+    ///
+    index: HashMap<Key, VecMap<Lsn, BlobRef>>,

-    /// Reader object for reading blocks from the file. (None if not loaded yet)
-    file: Option<FileBlockReader<VirtualFile>>,
+    book: Option<Book<VirtualFile>>,
 }

 impl Layer for DeltaLayer {
@@ -229,47 +158,45 @@ impl Layer for DeltaLayer {
        {
            // Open the file and lock the metadata in memory
            let inner = self.load()?;
+            let values_reader = inner
+                .book
+                .as_ref()
+                .expect("should be loaded in load call above")
+                .chapter_reader(VALUES_CHAPTER)?;

            // Scan the page versions backwards, starting from `lsn`.
-            let file = inner.file.as_ref().unwrap();
-            let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
-                inner.index_start_blk,
-                inner.index_root_blk,
-                file,
-            );
-            let search_key = DeltaKey::from_key_lsn(&key, Lsn(lsn_range.end.0 - 1));
-
-            let mut offsets: Vec<(Lsn, u64)> = Vec::new();
-
-            tree_reader.visit(&search_key.0, VisitDirection::Backwards, |key, value| {
-                let blob_ref = BlobRef(value);
-                if key[..KEY_SIZE] != search_key.0[..KEY_SIZE] {
-                    return false;
-                }
-                let entry_lsn = DeltaKey::extract_lsn_from_buf(key);
-                offsets.push((entry_lsn, blob_ref.pos()));
-
-                !blob_ref.will_init()
-            })?;
-
-            // Ok, 'offsets' now contains the offsets of all the entries we need to read
-            let mut cursor = file.block_cursor();
-            for (entry_lsn, pos) in offsets {
-                let buf = cursor.read_blob(pos)?;
-                let val = Value::des(&buf)?;
-                match val {
-                    Value::Image(img) => {
-                        reconstruct_state.img = Some((entry_lsn, img));
-                        need_image = false;
+            if let Some(vec_map) = inner.index.get(&key) {
+                let slice = vec_map.slice_range(lsn_range);
+                let mut size = 0usize;
+                let mut first_pos = 0u64;
+                for (_entry_lsn, blob_ref) in slice.iter().rev() {
+                    size += blob_ref.size();
+                    first_pos = blob_ref.pos();
+                    if blob_ref.will_init() {
                        break;
                    }
-                    Value::WalRecord(rec) => {
-                        let will_init = rec.will_init();
-                        reconstruct_state.records.push((entry_lsn, rec));
-                        if will_init {
-                            // This WAL record initializes the page, so no need to go further back
-                            need_image = false;
-                            break;
+                }
+                if size != 0 {
+                    let mut buf = vec![0u8; size];
+                    values_reader.read_exact_at(&mut buf, first_pos)?;
+                    for (entry_lsn, blob_ref) in slice.iter().rev() {
+                        let offs = (blob_ref.pos() - first_pos) as usize;
+                        let val = Value::des(&buf[offs..offs + blob_ref.size()])?;
+                        match val {
+                            Value::Image(img) => {
+                                reconstruct_state.img = Some((*entry_lsn, img));
+                                need_image = false;
+                                break;
+                            }
+                            Value::WalRecord(rec) => {
+                                let will_init = rec.will_init();
+                                reconstruct_state.records.push((*entry_lsn, rec));
+                                if will_init {
+                                    // This WAL record initializes the page, so no need to go further back
+                                    need_image = false;
+                                    break;
+                                }
+                            }
                        }
                    }
                }
@@ -286,7 +213,7 @@ impl Layer for DeltaLayer {
        }
    }

-    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = anyhow::Result<(Key, Lsn, Value)>> + 'a> {
+    fn iter(&self) -> Box<dyn Iterator<Item = anyhow::Result<(Key, Lsn, Value)>> + '_> {
        let inner = self.load().unwrap();

        match DeltaValueIter::new(inner) {
@@ -295,6 +222,36 @@ impl Layer for DeltaLayer {
        }
    }

+    ///
+    /// Release most of the memory used by this layer. If it's accessed again later,
+    /// it will need to be loaded back.
+    ///
+    fn unload(&self) -> Result<()> {
+        // FIXME: In debug mode, loading and unloading the index slows
+        // things down so much that you get timeout errors. At least
+        // with the test_parallel_copy test. So as an even more ad hoc
+        // stopgap fix for that, only unload every on average 10
+        // checkpoint cycles.
+        use rand::RngCore;
+        if rand::thread_rng().next_u32() > (u32::MAX / 10) {
+            return Ok(());
+        }
+
+        let mut inner = match self.inner.try_write() {
+            Ok(inner) => inner,
+            Err(TryLockError::WouldBlock) => return Ok(()),
+            Err(TryLockError::Poisoned(_)) => panic!("DeltaLayer lock was poisoned"),
+        };
+        inner.index = HashMap::default();
+        inner.loaded = false;
+
+        // Note: we keep the Book open. Is that a good idea? The virtual file
+        // machinery has its own rules for closing the file descriptor if it's not
+        // needed, but the Book struct uses up some memory, too.
+
+        Ok(())
+    }
+
    fn delete(&self) -> Result<()> {
        // delete underlying file
        fs::remove_file(self.path())?;
@@ -327,61 +284,42 @@ impl Layer for DeltaLayer {

        let inner = self.load()?;

-        println!(
-            "index_start_blk: {}, root {}",
-            inner.index_start_blk, inner.index_root_blk
-        );
+        let path = self.path();
+        let file = std::fs::File::open(&path)?;
+        let book = Book::new(file)?;
+        let chapter = book.chapter_reader(VALUES_CHAPTER)?;

-        let file = inner.file.as_ref().unwrap();
-        let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
-            inner.index_start_blk,
-            inner.index_root_blk,
-            file,
-        );
-
-        tree_reader.dump()?;
-
-        let mut cursor = file.block_cursor();
-        tree_reader.visit(
-            &[0u8; DELTA_KEY_SIZE],
-            VisitDirection::Forwards,
-            |delta_key, val| {
-                let blob_ref = BlobRef(val);
-                let key = DeltaKey::extract_key_from_buf(delta_key);
-                let lsn = DeltaKey::extract_lsn_from_buf(delta_key);
+        let mut values: Vec<(&Key, &VecMap<Lsn, BlobRef>)> = inner.index.iter().collect();
+        values.sort_by_key(|k| k.0);

+        for (key, versions) in values {
+            for (lsn, blob_ref) in versions.as_slice() {
                let mut desc = String::new();
-                match cursor.read_blob(blob_ref.pos()) {
-                    Ok(buf) => {
-                        let val = Value::des(&buf);
-                        match val {
-                            Ok(Value::Image(img)) => {
-                                write!(&mut desc, " img {} bytes", img.len()).unwrap();
-                            }
-                            Ok(Value::WalRecord(rec)) => {
-                                let wal_desc = walrecord::describe_wal_record(&rec);
-                                write!(
-                                    &mut desc,
-                                    " rec {} bytes will_init: {} {}",
-                                    buf.len(),
-                                    rec.will_init(),
-                                    wal_desc
-                                )
-                                .unwrap();
-                            }
-                            Err(err) => {
-                                write!(&mut desc, " DESERIALIZATION ERROR: {}", err).unwrap();
-                            }
-                        }
+                let mut buf = vec![0u8; blob_ref.size()];
+                chapter.read_exact_at(&mut buf, blob_ref.pos())?;
+                let val = Value::des(&buf);
+
+                match val {
+                    Ok(Value::Image(img)) => {
+                        write!(&mut desc, " img {} bytes", img.len())?;
+                    }
+                    Ok(Value::WalRecord(rec)) => {
+                        let wal_desc = walrecord::describe_wal_record(&rec);
+                        write!(
+                            &mut desc,
+                            " rec {} bytes will_init: {} {}",
+                            buf.len(),
+                            rec.will_init(),
+                            wal_desc
+                        )?;
                    }
                    Err(err) => {
-                        write!(&mut desc, " READ ERROR: {}", err).unwrap();
+                        write!(&mut desc, " DESERIALIZATION ERROR: {}", err)?;
                    }
                }
                println!("  key {} at {}: {}", key, lsn, desc);
-                true
-            },
-        )?;
+            }
+        }

        Ok(())
    }
@@ -437,20 +375,19 @@ impl DeltaLayer {
        let path = self.path();

        // Open the file if it's not open already.
-        if inner.file.is_none() {
-            let file = VirtualFile::open(&path)
-                .with_context(|| format!("Failed to open file '{}'", path.display()))?;
-            inner.file = Some(FileBlockReader::new(file));
+        if inner.book.is_none() {
+            let file = VirtualFile::open(&path)?;
+            inner.book = Some(Book::new(file)?);
        }
-        let file = inner.file.as_mut().unwrap();
-        let summary_blk = file.read_blk(0)?;
-        let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
+        let book = inner.book.as_ref().unwrap();

        match &self.path_or_conf {
            PathOrConf::Conf(_) => {
-                let mut expected_summary = Summary::from(self);
-                expected_summary.index_start_blk = actual_summary.index_start_blk;
-                expected_summary.index_root_blk = actual_summary.index_root_blk;
+                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+                let actual_summary = Summary::des(&chapter)?;
+
+                let expected_summary = Summary::from(self);
+
                if actual_summary != expected_summary {
                    bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
                }
@@ -469,11 +406,12 @@ impl DeltaLayer {
            }
        }

-        inner.index_start_blk = actual_summary.index_start_blk;
-        inner.index_root_blk = actual_summary.index_root_blk;
+        let chapter = book.read_chapter(INDEX_CHAPTER)?;
+        let index = HashMap::des(&chapter)?;

        debug!("loaded from {}", &path.display());

+        inner.index = index;
        inner.loaded = true;
        Ok(())
    }
@@ -493,9 +431,8 @@ impl DeltaLayer {
            lsn_range: filename.lsn_range.clone(),
            inner: RwLock::new(DeltaLayerInner {
                loaded: false,
-                file: None,
-                index_start_blk: 0,
-                index_root_blk: 0,
+                book: None,
+                index: HashMap::default(),
            }),
        }
    }
@@ -503,14 +440,12 @@ impl DeltaLayer {
    /// Create a DeltaLayer struct representing an existing file on disk.
    ///
    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
-    pub fn new_for_path<F>(path: &Path, file: F) -> Result<Self>
+    pub fn new_for_path<F>(path: &Path, book: &Book<F>) -> Result<Self>
    where
        F: FileExt,
    {
-        let mut summary_buf = Vec::new();
-        summary_buf.resize(PAGE_SZ, 0);
-        file.read_exact_at(&mut summary_buf, 0)?;
-        let summary = Summary::des_prefix(&summary_buf)?;
+        let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+        let summary = Summary::des(&chapter)?;

        Ok(DeltaLayer {
            path_or_conf: PathOrConf::Path(path.to_path_buf()),
@@ -520,9 +455,8 @@ impl DeltaLayer {
            lsn_range: summary.lsn_range,
            inner: RwLock::new(DeltaLayerInner {
                loaded: false,
-                file: None,
-                index_start_blk: 0,
-                index_root_blk: 0,
+                book: None,
+                index: HashMap::default(),
            }),
        })
    }
@@ -565,9 +499,10 @@ pub struct DeltaLayerWriter {
    key_start: Key,
    lsn_range: Range<Lsn>,

-    tree: DiskBtreeBuilder<BlockBuf, DELTA_KEY_SIZE>,
+    index: HashMap<Key, VecMap<Lsn, BlobRef>>,

-    blob_writer: WriteBlobWriter<BufWriter<VirtualFile>>,
+    values_writer: ChapterWriter<BufWriter<VirtualFile>>,
+    end_offset: u64,
 }

 impl DeltaLayerWriter {
@@ -593,15 +528,13 @@ impl DeltaLayerWriter {
            u64::from(lsn_range.start),
            u64::from(lsn_range.end)
        ));
-        let mut file = VirtualFile::create(&path)?;
-        // make room for the header block
-        file.seek(SeekFrom::Start(PAGE_SZ as u64))?;
+        let file = VirtualFile::create(&path)?;
        let buf_writer = BufWriter::new(file);
-        let blob_writer = WriteBlobWriter::new(buf_writer, PAGE_SZ as u64);
+        let book = BookWriter::new(buf_writer, DELTA_FILE_MAGIC)?;

-        // Initialize the b-tree index builder
-        let block_buf = BlockBuf::new();
-        let tree_builder = DiskBtreeBuilder::new(block_buf);
+        // Open the page-versions chapter for writing. The calls to
+        // `put_value` will use this to write the contents.
+        let values_writer = book.new_chapter(VALUES_CHAPTER);

        Ok(DeltaLayerWriter {
            conf,
@@ -610,8 +543,9 @@ impl DeltaLayerWriter {
            tenantid,
            key_start,
            lsn_range,
-            tree: tree_builder,
-            blob_writer,
+            index: HashMap::new(),
+            values_writer,
+            end_offset: 0,
        })
    }

@@ -621,56 +555,63 @@ impl DeltaLayerWriter {
    /// The values must be appended in key, lsn order.
    ///
    pub fn put_value(&mut self, key: Key, lsn: Lsn, val: Value) -> Result<()> {
+        //info!("DELTA: key {} at {} on {}", key, lsn, self.path.display());
        assert!(self.lsn_range.start <= lsn);
-
-        let off = self.blob_writer.write_blob(&Value::ser(&val)?)?;
-
-        let blob_ref = BlobRef::new(off, val.will_init());
-
-        let delta_key = DeltaKey::from_key_lsn(&key, lsn);
-        self.tree.append(&delta_key.0, blob_ref.0)?;
+        // Remember the offset and size metadata. The metadata is written
+        // to a separate chapter, in `finish`.
+        let off = self.end_offset;
+        let buf = Value::ser(&val)?;
+        let len = buf.len();
+        self.values_writer.write_all(&buf)?;
+        self.end_offset += len as u64;
+        let vec_map = self.index.entry(key).or_default();
+        let blob_ref = BlobRef::new(off, len, val.will_init());
+        let old = vec_map.append_or_update_last(lsn, blob_ref).unwrap().0;
+        if old.is_some() {
+            // We already had an entry for this LSN. That's odd..
+            bail!(
+                "Value for {} at {} already exists in delta layer being built",
+                key,
+                lsn
+            );
+        }

        Ok(())
    }

    pub fn size(&self) -> u64 {
-        self.blob_writer.size() + self.tree.borrow_writer().size()
+        self.end_offset
    }

    ///
    /// Finish writing the delta layer.
    ///
    pub fn finish(self, key_end: Key) -> anyhow::Result<DeltaLayer> {
-        let index_start_blk =
-            ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32;
-
-        let buf_writer = self.blob_writer.into_inner();
-        let mut file = buf_writer.into_inner()?;
+        // Close the values chapter
+        let book = self.values_writer.close()?;

        // Write out the index
-        let (index_root_blk, block_buf) = self.tree.finish()?;
-        file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))?;
-        for buf in block_buf.blocks {
-            file.write_all(buf.as_ref())?;
-        }
+        let mut chapter = book.new_chapter(INDEX_CHAPTER);
+        let buf = HashMap::ser(&self.index)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;

-        // Fill in the summary on blk 0
+        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
        let summary = Summary {
-            magic: DELTA_FILE_MAGIC,
-            format_version: STORAGE_FORMAT_VERSION,
            tenantid: self.tenantid,
            timelineid: self.timelineid,
            key_range: self.key_start..key_end,
            lsn_range: self.lsn_range.clone(),
-            index_start_blk,
-            index_root_blk,
        };
-        file.seek(SeekFrom::Start(0))?;
-        Summary::ser_into(&summary, &mut file)?;
+        Summary::ser_into(&summary, &mut chapter)?;
+        let book = chapter.close()?;
+
+        // This flushes the underlying 'buf_writer'.
+        book.close()?;

        // Note: Because we opened the file in write-only mode, we cannot
        // reuse the same VirtualFile for reading later. That's why we don't
-        // set inner.file here. The first read will have to re-open it.
+        // set inner.book here. The first read will have to re-open it.
        let layer = DeltaLayer {
            path_or_conf: PathOrConf::Conf(self.conf),
            tenantid: self.tenantid,
@@ -679,9 +620,8 @@ impl DeltaLayerWriter {
            lsn_range: self.lsn_range.clone(),
            inner: RwLock::new(DeltaLayerInner {
                loaded: false,
-                file: None,
-                index_start_blk,
-                index_root_blk,
+                index: HashMap::new(),
+                book: None,
            }),
        };

@@ -704,6 +644,22 @@ impl DeltaLayerWriter {

        Ok(layer)
    }
+
+    pub fn abort(self) {
+        match self.values_writer.close() {
+            Ok(book) => {
+                if let Err(err) = book.close() {
+                    error!("error while closing delta layer file: {}", err);
+                }
+            }
+            Err(err) => {
+                error!("error while closing chapter writer: {}", err);
+            }
+        }
+        if let Err(err) = std::fs::remove_file(self.path) {
+            error!("error removing unfinished delta layer file: {}", err);
+        }
+    }
 }

 ///
@@ -713,23 +669,13 @@ impl DeltaLayerWriter {
 /// That takes up quite a lot of memory. Should do this in a more streaming
 /// fashion.
 ///
-struct DeltaValueIter<'a> {
-    all_offsets: Vec<(DeltaKey, BlobRef)>,
+struct DeltaValueIter {
+    all_offsets: Vec<(Key, Lsn, BlobRef)>,
    next_idx: usize,
-    reader: BlockCursor<Adapter<'a>>,
+    data: Vec<u8>,
 }

-struct Adapter<'a>(RwLockReadGuard<'a, DeltaLayerInner>);
-
-impl<'a> BlockReader for Adapter<'a> {
-    type BlockLease = PageReadGuard<'static>;
-
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-        self.0.file.as_ref().unwrap().read_blk(blknum)
-    }
-}
-
-impl<'a> Iterator for DeltaValueIter<'a> {
+impl Iterator for DeltaValueIter {
    type Item = Result<(Key, Lsn, Value)>;

    fn next(&mut self) -> Option<Self::Item> {
@@ -737,43 +683,40 @@ impl<'a> Iterator for DeltaValueIter<'a> {
    }
 }

-impl<'a> DeltaValueIter<'a> {
-    fn new(inner: RwLockReadGuard<'a, DeltaLayerInner>) -> Result<Self> {
-        let file = inner.file.as_ref().unwrap();
-        let tree_reader = DiskBtreeReader::<_, DELTA_KEY_SIZE>::new(
-            inner.index_start_blk,
-            inner.index_root_blk,
-            file,
-        );
+impl DeltaValueIter {
+    fn new(inner: RwLockReadGuard<DeltaLayerInner>) -> Result<Self> {
+        let mut index: Vec<(&Key, &VecMap<Lsn, BlobRef>)> = inner.index.iter().collect();
+        index.sort_by_key(|x| x.0);

-        let mut all_offsets: Vec<(DeltaKey, BlobRef)> = Vec::new();
-        tree_reader.visit(
-            &[0u8; DELTA_KEY_SIZE],
-            VisitDirection::Forwards,
-            |key, value| {
-                all_offsets.push((DeltaKey::from_slice(key), BlobRef(value)));
-                true
-            },
-        )?;
+        let mut all_offsets: Vec<(Key, Lsn, BlobRef)> = Vec::new();
+        for (key, vec_map) in index.iter() {
+            for (lsn, blob_ref) in vec_map.as_slice().iter() {
+                all_offsets.push((**key, *lsn, *blob_ref));
+            }
+        }

-        let iter = DeltaValueIter {
+        let values_reader = inner
+            .book
+            .as_ref()
+            .expect("should be loaded in load call above")
+            .chapter_reader(VALUES_CHAPTER)?;
+        let file_size = values_reader.len() as usize;
+        let mut layer = DeltaValueIter {
            all_offsets,
            next_idx: 0,
-            reader: BlockCursor::new(Adapter(inner)),
+            data: vec![0u8; file_size],
        };
+        values_reader.read_exact_at(&mut layer.data, 0)?;

-        Ok(iter)
+        Ok(layer)
    }

    fn next_res(&mut self) -> Result<Option<(Key, Lsn, Value)>> {
        if self.next_idx < self.all_offsets.len() {
-            let (delta_key, blob_ref) = &self.all_offsets[self.next_idx];
-
-            let key = delta_key.key();
-            let lsn = delta_key.lsn();
-
-            let buf = self.reader.read_blob(blob_ref.pos())?;
-            let val = Value::des(&buf)?;
+            let (key, lsn, blob_ref) = self.all_offsets[self.next_idx];
+            let offs = blob_ref.pos() as usize;
+            let size = blob_ref.size();
+            let val = Value::des(&self.data[offs..offs + size])?;
            self.next_idx += 1;
            Ok(Some((key, lsn, val)))
        } else {
--- a/pageserver/src/layered_repository/disk_btree.rs
+++ b/pageserver/src/layered_repository/disk_btree.rs
@@ -1,979 +0,0 @@
-//!
-//! Simple on-disk B-tree implementation
-//!
-//! This is used as the index structure within image and delta layers
-//!
-//! Features:
-//! - Fixed-width keys
-//! - Fixed-width values (VALUE_SZ)
-//! - The tree is created in a bulk operation. Insert/deletion after creation
-//!   is not suppported
-//! - page-oriented
-//!
-//! TODO:
-//! - better errors (e.g. with thiserror?)
-//! - maybe something like an Adaptive Radix Tree would be more efficient?
-//! - the values stored by image and delta layers are offsets into the file,
-//!   and they are in monotonically increasing order. Prefix compression would
-//!   be very useful for them, too.
-//! - An Iterator interface would be more convenient for the callers than the
-//!   'visit' function
-//!
-use anyhow;
-use byteorder::{ReadBytesExt, BE};
-use bytes::{BufMut, Bytes, BytesMut};
-use hex;
-use std::cmp::Ordering;
-
-use crate::layered_repository::block_io::{BlockReader, BlockWriter};
-
-// The maximum size of a value stored in the B-tree. 5 bytes is enough currently.
-pub const VALUE_SZ: usize = 5;
-pub const MAX_VALUE: u64 = 0x007f_ffff_ffff;
-
-#[allow(dead_code)]
-pub const PAGE_SZ: usize = 8192;
-
-#[derive(Clone, Copy, Debug)]
-struct Value([u8; VALUE_SZ]);
-
-impl Value {
-    fn from_slice(slice: &[u8]) -> Value {
-        let mut b = [0u8; VALUE_SZ];
-        b.copy_from_slice(slice);
-        Value(b)
-    }
-
-    fn from_u64(x: u64) -> Value {
-        assert!(x <= 0x007f_ffff_ffff);
-        Value([
-            (x >> 32) as u8,
-            (x >> 24) as u8,
-            (x >> 16) as u8,
-            (x >> 8) as u8,
-            x as u8,
-        ])
-    }
-
-    fn from_blknum(x: u32) -> Value {
-        Value([
-            0x80,
-            (x >> 24) as u8,
-            (x >> 16) as u8,
-            (x >> 8) as u8,
-            x as u8,
-        ])
-    }
-
-    #[allow(dead_code)]
-    fn is_offset(self) -> bool {
-        self.0[0] & 0x80 != 0
-    }
-
-    fn to_u64(self) -> u64 {
-        let b = &self.0;
-        (b[0] as u64) << 32
-            | (b[1] as u64) << 24
-            | (b[2] as u64) << 16
-            | (b[3] as u64) << 8
-            | b[4] as u64
-    }
-
-    fn to_blknum(self) -> u32 {
-        let b = &self.0;
-        assert!(b[0] == 0x80);
-        (b[1] as u32) << 24 | (b[2] as u32) << 16 | (b[3] as u32) << 8 | b[4] as u32
-    }
-}
-
-/// This is the on-disk representation.
-struct OnDiskNode<'a, const L: usize> {
-    // Fixed-width fields
-    num_children: u16,
-    level: u8,
-    prefix_len: u8,
-    suffix_len: u8,
-
-    // Variable-length fields. These are stored on-disk after the fixed-width
-    // fields, in this order. In the in-memory representation, these point to
-    // the right parts in the page buffer.
-    prefix: &'a [u8],
-    keys: &'a [u8],
-    values: &'a [u8],
-}
-
-impl<'a, const L: usize> OnDiskNode<'a, L> {
-    ///
-    /// Interpret a PAGE_SZ page as a node.
-    ///
-    fn deparse(buf: &[u8]) -> OnDiskNode<L> {
-        let mut cursor = std::io::Cursor::new(buf);
-        let num_children = cursor.read_u16::<BE>().unwrap();
-        let level = cursor.read_u8().unwrap();
-        let prefix_len = cursor.read_u8().unwrap();
-        let suffix_len = cursor.read_u8().unwrap();
-
-        let mut off = cursor.position();
-        let prefix_off = off as usize;
-        off += prefix_len as u64;
-
-        let keys_off = off as usize;
-        let keys_len = num_children as usize * suffix_len as usize;
-        off += keys_len as u64;
-
-        let values_off = off as usize;
-        let values_len = num_children as usize * VALUE_SZ as usize;
-        //off += values_len as u64;
-
-        let prefix = &buf[prefix_off..prefix_off + prefix_len as usize];
-        let keys = &buf[keys_off..keys_off + keys_len];
-        let values = &buf[values_off..values_off + values_len];
-
-        OnDiskNode {
-            num_children,
-            level,
-            prefix_len,
-            suffix_len,
-            prefix,
-            keys,
-            values,
-        }
-    }
-
-    ///
-    /// Read a value at 'idx'
-    ///
-    fn value(&self, idx: usize) -> Value {
-        let value_off = idx * VALUE_SZ;
-        let value_slice = &self.values[value_off..value_off + VALUE_SZ];
-        Value::from_slice(value_slice)
-    }
-
-    fn binary_search(&self, search_key: &[u8; L], keybuf: &mut [u8]) -> Result<usize, usize> {
-        let mut size = self.num_children as usize;
-        let mut low = 0;
-        let mut high = size;
-        while low < high {
-            let mid = low + size / 2;
-
-            let key_off = mid as usize * self.suffix_len as usize;
-            let suffix = &self.keys[key_off..key_off + self.suffix_len as usize];
-            // Does this match?
-            keybuf[self.prefix_len as usize..].copy_from_slice(suffix);
-
-            let cmp = keybuf[..].cmp(search_key);
-
-            if cmp == Ordering::Less {
-                low = mid + 1;
-            } else if cmp == Ordering::Greater {
-                high = mid;
-            } else {
-                return Ok(mid);
-            }
-            size = high - low;
-        }
-        Err(low)
-    }
-}
-
-///
-/// Public reader object, to search the tree.
-///
-pub struct DiskBtreeReader<R, const L: usize>
-where
-    R: BlockReader,
-{
-    start_blk: u32,
-    root_blk: u32,
-    reader: R,
-}
-
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum VisitDirection {
-    Forwards,
-    Backwards,
-}
-
-impl<R, const L: usize> DiskBtreeReader<R, L>
-where
-    R: BlockReader,
-{
-    pub fn new(start_blk: u32, root_blk: u32, reader: R) -> Self {
-        DiskBtreeReader {
-            start_blk,
-            root_blk,
-            reader,
-        }
-    }
-
-    ///
-    /// Read the value for given key. Returns the value, or None if it doesn't exist.
-    ///
-    pub fn get(&self, search_key: &[u8; L]) -> anyhow::Result<Option<u64>> {
-        let mut result: Option<u64> = None;
-        self.visit(search_key, VisitDirection::Forwards, |key, value| {
-            if key == search_key {
-                result = Some(value);
-            }
-            false
-        })?;
-        Ok(result)
-    }
-
-    ///
-    /// Scan the tree, starting from 'search_key', in the given direction. 'visitor'
-    /// will be called for every key >= 'search_key' (or <= 'search_key', if scanning
-    /// backwards)
-    ///
-    pub fn visit<V>(
-        &self,
-        search_key: &[u8; L],
-        dir: VisitDirection,
-        mut visitor: V,
-    ) -> anyhow::Result<bool>
-    where
-        V: FnMut(&[u8], u64) -> bool,
-    {
-        self.search_recurse(self.root_blk, search_key, dir, &mut visitor)
-    }
-
-    fn search_recurse<V>(
-        &self,
-        node_blknum: u32,
-        search_key: &[u8; L],
-        dir: VisitDirection,
-        visitor: &mut V,
-    ) -> anyhow::Result<bool>
-    where
-        V: FnMut(&[u8], u64) -> bool,
-    {
-        // Locate the node.
-        let blk = self.reader.read_blk(self.start_blk + node_blknum)?;
-
-        // Search all entries on this node
-        self.search_node(blk.as_ref(), search_key, dir, visitor)
-    }
-
-    fn search_node<V>(
-        &self,
-        node_buf: &[u8],
-        search_key: &[u8; L],
-        dir: VisitDirection,
-        visitor: &mut V,
-    ) -> anyhow::Result<bool>
-    where
-        V: FnMut(&[u8], u64) -> bool,
-    {
-        let node = OnDiskNode::deparse(node_buf);
-        let prefix_len = node.prefix_len as usize;
-        let suffix_len = node.suffix_len as usize;
-
-        assert!(node.num_children > 0);
-
-        let mut keybuf = Vec::new();
-        keybuf.extend(node.prefix);
-        keybuf.resize(prefix_len + suffix_len, 0);
-
-        if dir == VisitDirection::Forwards {
-            // Locate the first match
-            let mut idx = match node.binary_search(search_key, keybuf.as_mut_slice()) {
-                Ok(idx) => idx,
-                Err(idx) => {
-                    if node.level == 0 {
-                        // Imagine that the node contains the following keys:
-                        //
-                        // 1
-                        // 3  <-- idx
-                        // 5
-                        //
-                        // If the search key is '2' and there is exact match,
-                        // the binary search would return the index of key
-                        // '3'. That's cool, '3' is the first key to return.
-                        idx
-                    } else {
-                        // This is an internal page, so each key represents a lower
-                        // bound for what's in the child page. If there is no exact
-                        // match, we have to return the *previous* entry.
-                        //
-                        // 1  <-- return this
-                        // 3  <-- idx
-                        // 5
-                        idx.saturating_sub(1)
-                    }
-                }
-            };
-            // idx points to the first match now. Keep going from there
-            let mut key_off = idx * suffix_len;
-            while idx < node.num_children as usize {
-                let suffix = &node.keys[key_off..key_off + suffix_len];
-                keybuf[prefix_len..].copy_from_slice(suffix);
-                let value = node.value(idx as usize);
-                #[allow(clippy::collapsible_if)]
-                if node.level == 0 {
-                    // leaf
-                    if !visitor(&keybuf, value.to_u64()) {
-                        return Ok(false);
-                    }
-                } else {
-                    #[allow(clippy::collapsible_if)]
-                    if !self.search_recurse(value.to_blknum(), search_key, dir, visitor)? {
-                        return Ok(false);
-                    }
-                }
-                idx += 1;
-                key_off += suffix_len;
-            }
-        } else {
-            let mut idx = match node.binary_search(search_key, keybuf.as_mut_slice()) {
-                Ok(idx) => {
-                    // Exact match. That's the first entry to return, and walk
-                    // backwards from there. (The loop below starts from 'idx -
-                    // 1', so add one here to compensate.)
-                    idx + 1
-                }
-                Err(idx) => {
-                    // No exact match. The binary search returned the index of the
-                    // first key that's > search_key. Back off by one, and walk
-                    // backwards from there. (The loop below starts from idx - 1,
-                    // so we don't need to subtract one here)
-                    idx
-                }
-            };
-
-            // idx points to the first match + 1 now. Keep going from there.
-            let mut key_off = idx * suffix_len;
-            while idx > 0 {
-                idx -= 1;
-                key_off -= suffix_len;
-                let suffix = &node.keys[key_off..key_off + suffix_len];
-                keybuf[prefix_len..].copy_from_slice(suffix);
-                let value = node.value(idx as usize);
-                #[allow(clippy::collapsible_if)]
-                if node.level == 0 {
-                    // leaf
-                    if !visitor(&keybuf, value.to_u64()) {
-                        return Ok(false);
-                    }
-                } else {
-                    #[allow(clippy::collapsible_if)]
-                    if !self.search_recurse(value.to_blknum(), search_key, dir, visitor)? {
-                        return Ok(false);
-                    }
-                }
-                if idx == 0 {
-                    break;
-                }
-            }
-        }
-        Ok(true)
-    }
-
-    #[allow(dead_code)]
-    pub fn dump(&self) -> anyhow::Result<()> {
-        self.dump_recurse(self.root_blk, &[], 0)
-    }
-
-    fn dump_recurse(&self, blknum: u32, path: &[u8], depth: usize) -> anyhow::Result<()> {
-        let blk = self.reader.read_blk(self.start_blk + blknum)?;
-        let buf: &[u8] = blk.as_ref();
-
-        let node = OnDiskNode::<L>::deparse(buf);
-
-        print!("{:indent$}", "", indent = depth * 2);
-        println!(
-            "blk #{}: path {}: prefix {}, suffix_len {}",
-            blknum,
-            hex::encode(path),
-            hex::encode(node.prefix),
-            node.suffix_len
-        );
-
-        let mut idx = 0;
-        let mut key_off = 0;
-        while idx < node.num_children {
-            let key = &node.keys[key_off..key_off + node.suffix_len as usize];
-            let val = node.value(idx as usize);
-            print!("{:indent$}", "", indent = depth * 2 + 2);
-            println!("{}: {}", hex::encode(key), hex::encode(val.0));
-
-            if node.level > 0 {
-                let child_path = [path, node.prefix].concat();
-                self.dump_recurse(val.to_blknum(), &child_path, depth + 1)?;
-            }
-            idx += 1;
-            key_off += node.suffix_len as usize;
-        }
-        Ok(())
-    }
-}
-
-///
-/// Public builder object, for creating a new tree.
-///
-/// Usage: Create a builder object by calling 'new', load all the data into the
-/// tree by calling 'append' for each key-value pair, and then call 'finish'
-///
-/// 'L' is the key length in bytes
-pub struct DiskBtreeBuilder<W, const L: usize>
-where
-    W: BlockWriter,
-{
-    writer: W,
-
-    ///
-    /// stack[0] is the current root page, stack.last() is the leaf.
-    ///
-    stack: Vec<BuildNode<L>>,
-
-    /// Last key that was appended to the tree. Used to sanity check that append
-    /// is called in increasing key order.
-    last_key: Option<[u8; L]>,
-}
-
-impl<W, const L: usize> DiskBtreeBuilder<W, L>
-where
-    W: BlockWriter,
-{
-    pub fn new(writer: W) -> Self {
-        DiskBtreeBuilder {
-            writer,
-            last_key: None,
-            stack: vec![BuildNode::new(0)],
-        }
-    }
-
-    pub fn append(&mut self, key: &[u8; L], value: u64) -> Result<(), anyhow::Error> {
-        assert!(value <= MAX_VALUE);
-        if let Some(last_key) = &self.last_key {
-            assert!(key > last_key, "unsorted input");
-        }
-        self.last_key = Some(*key);
-
-        Ok(self.append_internal(key, Value::from_u64(value))?)
-    }
-
-    fn append_internal(&mut self, key: &[u8; L], value: Value) -> Result<(), std::io::Error> {
-        // Try to append to the current leaf buffer
-        let last = self.stack.last_mut().unwrap();
-        let level = last.level;
-        if last.push(key, value) {
-            return Ok(());
-        }
-
-        // It did not fit. Try to compress, and it it succeeds to make some room
-        // on the node, try appending to it again.
-        #[allow(clippy::collapsible_if)]
-        if last.compress() {
-            if last.push(key, value) {
-                return Ok(());
-            }
-        }
-
-        // Could not append to the current leaf. Flush it and create a new one.
-        self.flush_node()?;
-
-        // Replace the node we flushed with an empty one and append the new
-        // key to it.
-        let mut last = BuildNode::new(level);
-        if !last.push(key, value) {
-            panic!("could not push to new leaf node");
-        }
-        self.stack.push(last);
-
-        Ok(())
-    }
-
-    fn flush_node(&mut self) -> Result<(), std::io::Error> {
-        let last = self.stack.pop().unwrap();
-        let buf = last.pack();
-        let downlink_key = last.first_key();
-        let downlink_ptr = self.writer.write_blk(buf)?;
-
-        // Append the downlink to the parent
-        if self.stack.is_empty() {
-            self.stack.push(BuildNode::new(last.level + 1));
-        }
-        self.append_internal(&downlink_key, Value::from_blknum(downlink_ptr))?;
-
-        Ok(())
-    }
-
-    ///
-    /// Flushes everything to disk, and returns the block number of the root page.
-    /// The caller must store the root block number "out-of-band", and pass it
-    /// to the DiskBtreeReader::new() when you want to read the tree again.
-    /// (In the image and delta layers, it is stored in the beginning of the file,
-    /// in the summary header)
-    ///
-    pub fn finish(mut self) -> Result<(u32, W), std::io::Error> {
-        // flush all levels, except the root.
-        while self.stack.len() > 1 {
-            self.flush_node()?;
-        }
-
-        let root = self.stack.first().unwrap();
-        let buf = root.pack();
-        let root_blknum = self.writer.write_blk(buf)?;
-
-        Ok((root_blknum, self.writer))
-    }
-
-    pub fn borrow_writer(&self) -> &W {
-        &self.writer
-    }
-}
-
-///
-/// BuildNode represesnts an incomplete page that we are appending to.
-///
-#[derive(Clone, Debug)]
-struct BuildNode<const L: usize> {
-    num_children: u16,
-    level: u8,
-    prefix: Vec<u8>,
-    suffix_len: usize,
-
-    keys: Vec<u8>,
-    values: Vec<u8>,
-
-    size: usize, // physical size of this node, if it was written to disk like this
-}
-
-const NODE_SIZE: usize = PAGE_SZ;
-
-const NODE_HDR_SIZE: usize = 2 + 1 + 1 + 1;
-
-impl<const L: usize> BuildNode<L> {
-    fn new(level: u8) -> Self {
-        BuildNode {
-            num_children: 0,
-            level,
-            prefix: Vec::new(),
-            suffix_len: 0,
-            keys: Vec::new(),
-            values: Vec::new(),
-            size: NODE_HDR_SIZE,
-        }
-    }
-
-    /// Try to append a key-value pair to this node. Returns 'true' on
-    /// success, 'false' if the page was full or the key was
-    /// incompatible with the prefix of the existing keys.
-    fn push(&mut self, key: &[u8; L], value: Value) -> bool {
-        // If we have already performed prefix-compression on the page,
-        // check that the incoming key has the same prefix.
-        if self.num_children > 0 {
-            // does the prefix allow it?
-            if !key.starts_with(&self.prefix) {
-                return false;
-            }
-        } else {
-            self.suffix_len = key.len();
-        }
-
-        // Is the node too full?
-        if self.size + self.suffix_len + VALUE_SZ >= NODE_SIZE {
-            return false;
-        }
-
-        // All clear
-        self.num_children += 1;
-        self.keys.extend(&key[self.prefix.len()..]);
-        self.values.extend(value.0);
-
-        assert!(self.keys.len() == self.num_children as usize * self.suffix_len as usize);
-        assert!(self.values.len() == self.num_children as usize * VALUE_SZ);
-
-        self.size += self.suffix_len + VALUE_SZ;
-
-        true
-    }
-
-    ///
-    /// Perform prefix-compression.
-    ///
-    /// Returns 'true' on success, 'false' if no compression was possible.
-    ///
-    fn compress(&mut self) -> bool {
-        let first_suffix = self.first_suffix();
-        let last_suffix = self.last_suffix();
-
-        // Find the common prefix among all keys
-        let mut prefix_len = 0;
-        while prefix_len < self.suffix_len {
-            if first_suffix[prefix_len] != last_suffix[prefix_len] {
-                break;
-            }
-            prefix_len += 1;
-        }
-        if prefix_len == 0 {
-            return false;
-        }
-
-        // Can compress. Rewrite the keys without the common prefix.
-        self.prefix.extend(&self.keys[..prefix_len]);
-
-        let mut new_keys = Vec::new();
-        let mut key_off = 0;
-        while key_off < self.keys.len() {
-            let next_key_off = key_off + self.suffix_len;
-            new_keys.extend(&self.keys[key_off + prefix_len..next_key_off]);
-            key_off = next_key_off;
-        }
-        self.keys = new_keys;
-        self.suffix_len -= prefix_len;
-
-        self.size -= prefix_len * self.num_children as usize;
-        self.size += prefix_len;
-
-        assert!(self.keys.len() == self.num_children as usize * self.suffix_len as usize);
-        assert!(self.values.len() == self.num_children as usize * VALUE_SZ);
-
-        true
-    }
-
-    ///
-    /// Serialize the node to on-disk format.
-    ///
-    fn pack(&self) -> Bytes {
-        assert!(self.keys.len() == self.num_children as usize * self.suffix_len as usize);
-        assert!(self.values.len() == self.num_children as usize * VALUE_SZ);
-        assert!(self.num_children > 0);
-
-        let mut buf = BytesMut::new();
-
-        buf.put_u16(self.num_children);
-        buf.put_u8(self.level);
-        buf.put_u8(self.prefix.len() as u8);
-        buf.put_u8(self.suffix_len as u8);
-        buf.put(&self.prefix[..]);
-        buf.put(&self.keys[..]);
-        buf.put(&self.values[..]);
-
-        assert!(buf.len() == self.size);
-
-        assert!(buf.len() <= PAGE_SZ);
-        buf.resize(PAGE_SZ, 0);
-        buf.freeze()
-    }
-
-    fn first_suffix(&self) -> &[u8] {
-        &self.keys[..self.suffix_len]
-    }
-    fn last_suffix(&self) -> &[u8] {
-        &self.keys[self.keys.len() - self.suffix_len..]
-    }
-
-    /// Return the full first key of the page, including the prefix
-    fn first_key(&self) -> [u8; L] {
-        let mut key = [0u8; L];
-        key[..self.prefix.len()].copy_from_slice(&self.prefix);
-        key[self.prefix.len()..].copy_from_slice(self.first_suffix());
-        key
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use rand::Rng;
-    use std::collections::BTreeMap;
-    use std::sync::atomic::{AtomicUsize, Ordering};
-
-    #[derive(Clone, Default)]
-    struct TestDisk {
-        blocks: Vec<Bytes>,
-    }
-    impl TestDisk {
-        fn new() -> Self {
-            Self::default()
-        }
-    }
-    impl BlockReader for TestDisk {
-        type BlockLease = std::rc::Rc<[u8; PAGE_SZ]>;
-
-        fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-            let mut buf = [0u8; PAGE_SZ];
-            buf.copy_from_slice(&self.blocks[blknum as usize]);
-            Ok(std::rc::Rc::new(buf))
-        }
-    }
-    impl BlockWriter for &mut TestDisk {
-        fn write_blk(&mut self, buf: Bytes) -> Result<u32, std::io::Error> {
-            let blknum = self.blocks.len();
-            self.blocks.push(buf);
-            Ok(blknum as u32)
-        }
-    }
-
-    #[test]
-    fn basic() -> anyhow::Result<()> {
-        let mut disk = TestDisk::new();
-        let mut writer = DiskBtreeBuilder::<_, 6>::new(&mut disk);
-
-        let all_keys: Vec<&[u8; 6]> = vec![
-            b"xaaaaa", b"xaaaba", b"xaaaca", b"xabaaa", b"xababa", b"xabaca", b"xabada", b"xabadb",
-        ];
-        let all_data: Vec<(&[u8; 6], u64)> = all_keys
-            .iter()
-            .enumerate()
-            .map(|(idx, key)| (*key, idx as u64))
-            .collect();
-        for (key, val) in all_data.iter() {
-            writer.append(key, *val)?;
-        }
-
-        let (root_offset, _writer) = writer.finish()?;
-
-        let reader = DiskBtreeReader::new(0, root_offset, disk);
-
-        reader.dump()?;
-
-        // Test the `get` function on all the keys.
-        for (key, val) in all_data.iter() {
-            assert_eq!(reader.get(key)?, Some(*val));
-        }
-        // And on some keys that don't exist
-        assert_eq!(reader.get(b"aaaaaa")?, None);
-        assert_eq!(reader.get(b"zzzzzz")?, None);
-        assert_eq!(reader.get(b"xaaabx")?, None);
-
-        // Test search with `visit` function
-        let search_key = b"xabaaa";
-        let expected: Vec<(Vec<u8>, u64)> = all_data
-            .iter()
-            .filter(|(key, _value)| key[..] >= search_key[..])
-            .map(|(key, value)| (key.to_vec(), *value))
-            .collect();
-
-        let mut data = Vec::new();
-        reader.visit(search_key, VisitDirection::Forwards, |key, value| {
-            data.push((key.to_vec(), value));
-            true
-        })?;
-        assert_eq!(data, expected);
-
-        // Test a backwards scan
-        let mut expected: Vec<(Vec<u8>, u64)> = all_data
-            .iter()
-            .filter(|(key, _value)| key[..] <= search_key[..])
-            .map(|(key, value)| (key.to_vec(), *value))
-            .collect();
-        expected.reverse();
-        let mut data = Vec::new();
-        reader.visit(search_key, VisitDirection::Backwards, |key, value| {
-            data.push((key.to_vec(), value));
-            true
-        })?;
-        assert_eq!(data, expected);
-
-        // Backward scan where nothing matches
-        reader.visit(b"aaaaaa", VisitDirection::Backwards, |key, value| {
-            panic!("found unexpected key {}: {}", hex::encode(key), value);
-        })?;
-
-        // Full scan
-        let expected: Vec<(Vec<u8>, u64)> = all_data
-            .iter()
-            .map(|(key, value)| (key.to_vec(), *value))
-            .collect();
-        let mut data = Vec::new();
-        reader.visit(&[0u8; 6], VisitDirection::Forwards, |key, value| {
-            data.push((key.to_vec(), value));
-            true
-        })?;
-        assert_eq!(data, expected);
-
-        Ok(())
-    }
-
-    #[test]
-    fn lots_of_keys() -> anyhow::Result<()> {
-        let mut disk = TestDisk::new();
-        let mut writer = DiskBtreeBuilder::<_, 8>::new(&mut disk);
-
-        const NUM_KEYS: u64 = 1000;
-
-        let mut all_data: BTreeMap<u64, u64> = BTreeMap::new();
-
-        for idx in 0..NUM_KEYS {
-            let key_int: u64 = 1 + idx * 2;
-            let key = u64::to_be_bytes(key_int);
-            writer.append(&key, idx)?;
-
-            all_data.insert(key_int, idx);
-        }
-
-        let (root_offset, _writer) = writer.finish()?;
-
-        let reader = DiskBtreeReader::new(0, root_offset, disk);
-
-        reader.dump()?;
-
-        use std::sync::Mutex;
-
-        let result = Mutex::new(Vec::new());
-        let limit: AtomicUsize = AtomicUsize::new(10);
-        let take_ten = |key: &[u8], value: u64| {
-            let mut keybuf = [0u8; 8];
-            keybuf.copy_from_slice(key);
-            let key_int = u64::from_be_bytes(keybuf);
-
-            let mut result = result.lock().unwrap();
-            result.push((key_int, value));
-
-            // keep going until we have 10 matches
-            result.len() < limit.load(Ordering::Relaxed)
-        };
-
-        for search_key_int in 0..(NUM_KEYS * 2 + 10) {
-            let search_key = u64::to_be_bytes(search_key_int);
-            assert_eq!(
-                reader.get(&search_key)?,
-                all_data.get(&search_key_int).cloned()
-            );
-
-            // Test a forward scan starting with this key
-            result.lock().unwrap().clear();
-            reader.visit(&search_key, VisitDirection::Forwards, take_ten)?;
-            let expected = all_data
-                .range(search_key_int..)
-                .take(10)
-                .map(|(&key, &val)| (key, val))
-                .collect::<Vec<(u64, u64)>>();
-            assert_eq!(*result.lock().unwrap(), expected);
-
-            // And a backwards scan
-            result.lock().unwrap().clear();
-            reader.visit(&search_key, VisitDirection::Backwards, take_ten)?;
-            let expected = all_data
-                .range(..=search_key_int)
-                .rev()
-                .take(10)
-                .map(|(&key, &val)| (key, val))
-                .collect::<Vec<(u64, u64)>>();
-            assert_eq!(*result.lock().unwrap(), expected);
-        }
-
-        // full scan
-        let search_key = u64::to_be_bytes(0);
-        limit.store(usize::MAX, Ordering::Relaxed);
-        result.lock().unwrap().clear();
-        reader.visit(&search_key, VisitDirection::Forwards, take_ten)?;
-        let expected = all_data
-            .iter()
-            .map(|(&key, &val)| (key, val))
-            .collect::<Vec<(u64, u64)>>();
-        assert_eq!(*result.lock().unwrap(), expected);
-
-        // full scan
-        let search_key = u64::to_be_bytes(u64::MAX);
-        limit.store(usize::MAX, Ordering::Relaxed);
-        result.lock().unwrap().clear();
-        reader.visit(&search_key, VisitDirection::Backwards, take_ten)?;
-        let expected = all_data
-            .iter()
-            .rev()
-            .map(|(&key, &val)| (key, val))
-            .collect::<Vec<(u64, u64)>>();
-        assert_eq!(*result.lock().unwrap(), expected);
-
-        Ok(())
-    }
-
-    #[test]
-    fn random_data() -> anyhow::Result<()> {
-        // Generate random keys with exponential distribution, to
-        // exercise the prefix compression
-        const NUM_KEYS: usize = 100000;
-        let mut all_data: BTreeMap<u128, u64> = BTreeMap::new();
-        for idx in 0..NUM_KEYS {
-            let u: f64 = rand::thread_rng().gen_range(0.0..1.0);
-            let t = -(f64::ln(u));
-            let key_int = (t * 1000000.0) as u128;
-
-            all_data.insert(key_int as u128, idx as u64);
-        }
-
-        // Build a tree from it
-        let mut disk = TestDisk::new();
-        let mut writer = DiskBtreeBuilder::<_, 16>::new(&mut disk);
-
-        for (&key, &val) in all_data.iter() {
-            writer.append(&u128::to_be_bytes(key), val)?;
-        }
-        let (root_offset, _writer) = writer.finish()?;
-
-        let reader = DiskBtreeReader::new(0, root_offset, disk);
-
-        // Test get() operation on all the keys
-        for (&key, &val) in all_data.iter() {
-            let search_key = u128::to_be_bytes(key);
-            assert_eq!(reader.get(&search_key)?, Some(val));
-        }
-
-        // Test get() operations on random keys, most of which will not exist
-        for _ in 0..100000 {
-            let key_int = rand::thread_rng().gen::<u128>();
-            let search_key = u128::to_be_bytes(key_int);
-            assert!(reader.get(&search_key)? == all_data.get(&key_int).cloned());
-        }
-
-        // Test boundary cases
-        assert!(reader.get(&u128::to_be_bytes(u128::MIN))? == all_data.get(&u128::MIN).cloned());
-        assert!(reader.get(&u128::to_be_bytes(u128::MAX))? == all_data.get(&u128::MAX).cloned());
-
-        Ok(())
-    }
-
-    #[test]
-    #[should_panic(expected = "unsorted input")]
-    fn unsorted_input() {
-        let mut disk = TestDisk::new();
-        let mut writer = DiskBtreeBuilder::<_, 2>::new(&mut disk);
-
-        let _ = writer.append(b"ba", 1);
-        let _ = writer.append(b"bb", 2);
-        let _ = writer.append(b"aa", 3);
-    }
-
-    ///
-    /// This test contains a particular data set, see disk_btree_test_data.rs
-    ///
-    #[test]
-    fn particular_data() -> anyhow::Result<()> {
-        // Build a tree from it
-        let mut disk = TestDisk::new();
-        let mut writer = DiskBtreeBuilder::<_, 26>::new(&mut disk);
-
-        for (key, val) in disk_btree_test_data::TEST_DATA {
-            writer.append(&key, val)?;
-        }
-        let (root_offset, writer) = writer.finish()?;
-
-        println!("SIZE: {} blocks", writer.blocks.len());
-
-        let reader = DiskBtreeReader::new(0, root_offset, disk);
-
-        // Test get() operation on all the keys
-        for (key, val) in disk_btree_test_data::TEST_DATA {
-            assert_eq!(reader.get(&key)?, Some(val));
-        }
-
-        // Test full scan
-        let mut count = 0;
-        reader.visit(&[0u8; 26], VisitDirection::Forwards, |_key, _value| {
-            count += 1;
-            true
-        })?;
-        assert_eq!(count, disk_btree_test_data::TEST_DATA.len());
-
-        reader.dump()?;
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-#[path = "disk_btree_test_data.rs"]
-mod disk_btree_test_data;
--- a/pageserver/src/layered_repository/disk_btree_test_data.rs
+++ b/pageserver/src/layered_repository/disk_btree_test_data.rs
--- a/pageserver/src/layered_repository/ephemeral_file.rs
+++ b/pageserver/src/layered_repository/ephemeral_file.rs
@@ -2,8 +2,6 @@
 //! used to keep in-memory layers spilled on disk.

 use crate::config::PageServerConf;
-use crate::layered_repository::blob_io::BlobWriter;
-use crate::layered_repository::block_io::BlockReader;
 use crate::page_cache;
 use crate::page_cache::PAGE_SZ;
 use crate::page_cache::{ReadBufResult, WriteBufResult};
@@ -12,7 +10,7 @@ use lazy_static::lazy_static;
 use std::cmp::min;
 use std::collections::HashMap;
 use std::fs::OpenOptions;
-use std::io::{Error, ErrorKind};
+use std::io::{Error, ErrorKind, Seek, SeekFrom, Write};
 use std::ops::DerefMut;
 use std::path::PathBuf;
 use std::sync::{Arc, RwLock};
@@ -43,7 +41,7 @@ pub struct EphemeralFile {
    _timelineid: ZTimelineId,
    file: Arc<VirtualFile>,

-    size: u64,
+    pos: u64,
 }

 impl EphemeralFile {
@@ -72,11 +70,11 @@ impl EphemeralFile {
            _tenantid: tenantid,
            _timelineid: timelineid,
            file: file_rc,
-            size: 0,
+            pos: 0,
        })
    }

-    fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), Error> {
+    pub fn fill_buffer(&self, buf: &mut [u8], blkno: u32) -> Result<(), Error> {
        let mut off = 0;
        while off < PAGE_SZ {
            let n = self
@@ -95,26 +93,6 @@ impl EphemeralFile {
        }
        Ok(())
    }
-
-    fn get_buf_for_write(&self, blkno: u32) -> Result<page_cache::PageWriteGuard, Error> {
-        // Look up the right page
-        let cache = page_cache::get();
-        let mut write_guard = match cache.write_ephemeral_buf(self.file_id, blkno) {
-            WriteBufResult::Found(guard) => guard,
-            WriteBufResult::NotFound(mut guard) => {
-                // Read the page from disk into the buffer
-                // TODO: if we're overwriting the whole page, no need to read it in first
-                self.fill_buffer(guard.deref_mut(), blkno)?;
-                guard.mark_valid();
-
-                // And then fall through to modify it.
-                guard
-            }
-        };
-        write_guard.mark_dirty();
-
-        Ok(write_guard)
-    }
 }

 /// Does the given filename look like an ephemeral file?
@@ -189,49 +167,48 @@ impl FileExt for EphemeralFile {
    }
 }

-impl BlobWriter for EphemeralFile {
-    fn write_blob(&mut self, srcbuf: &[u8]) -> Result<u64, Error> {
-        let pos = self.size;
+impl Write for EphemeralFile {
+    fn write(&mut self, buf: &[u8]) -> Result<usize, Error> {
+        let n = self.write_at(buf, self.pos)?;
+        self.pos += n as u64;
+        Ok(n)
+    }

-        let mut blknum = (self.size / PAGE_SZ as u64) as u32;
-        let mut off = (pos % PAGE_SZ as u64) as usize;
+    fn flush(&mut self) -> Result<(), std::io::Error> {
+        // we don't need to flush data:
+        // * we either write input bytes or not, not keeping any intermediate data buffered
+        // * rust unix file `flush` impl does not flush things either, returning `Ok(())`
+        Ok(())
+    }
+}

-        let mut buf = self.get_buf_for_write(blknum)?;
-
-        // Write the length field
-        let len_buf = u32::to_ne_bytes(srcbuf.len() as u32);
-        let thislen = PAGE_SZ - off;
-        if thislen < 4 {
-            // it needs to be split across pages
-            buf[off..(off + thislen)].copy_from_slice(&len_buf[..thislen]);
-            blknum += 1;
-            buf = self.get_buf_for_write(blknum)?;
-            buf[0..4 - thislen].copy_from_slice(&len_buf[thislen..]);
-            off = 4 - thislen;
-        } else {
-            buf[off..off + 4].copy_from_slice(&len_buf);
-            off += 4;
-        }
-
-        // Write the payload
-        let mut buf_remain = srcbuf;
-        while !buf_remain.is_empty() {
-            let mut page_remain = PAGE_SZ - off;
-            if page_remain == 0 {
-                blknum += 1;
-                buf = self.get_buf_for_write(blknum)?;
-                off = 0;
-                page_remain = PAGE_SZ;
+impl Seek for EphemeralFile {
+    fn seek(&mut self, pos: SeekFrom) -> Result<u64, Error> {
+        match pos {
+            SeekFrom::Start(offset) => {
+                self.pos = offset;
+            }
+            SeekFrom::End(_offset) => {
+                return Err(Error::new(
+                    ErrorKind::Other,
+                    "SeekFrom::End not supported by EphemeralFile",
+                ));
+            }
+            SeekFrom::Current(offset) => {
+                let pos = self.pos as i128 + offset as i128;
+                if pos < 0 {
+                    return Err(Error::new(
+                        ErrorKind::InvalidInput,
+                        "offset would be negative",
+                    ));
+                }
+                if pos > u64::MAX as i128 {
+                    return Err(Error::new(ErrorKind::InvalidInput, "offset overflow"));
+                }
+                self.pos = pos as u64;
            }
-            let this_blk_len = min(page_remain, buf_remain.len());
-            buf[off..(off + this_blk_len)].copy_from_slice(&buf_remain[..this_blk_len]);
-            off += this_blk_len;
-            buf_remain = &buf_remain[this_blk_len..];
        }
-        drop(buf);
-        self.size += 4 + srcbuf.len() as u64;
-
-        Ok(pos)
+        Ok(self.pos)
    }
 }

@@ -262,34 +239,11 @@ pub fn writeback(file_id: u64, blkno: u32, buf: &[u8]) -> Result<(), std::io::Er
    }
 }

-impl BlockReader for EphemeralFile {
-    type BlockLease = page_cache::PageReadGuard<'static>;
-
-    fn read_blk(&self, blknum: u32) -> Result<Self::BlockLease, std::io::Error> {
-        // Look up the right page
-        let cache = page_cache::get();
-        loop {
-            match cache.read_ephemeral_buf(self.file_id, blknum) {
-                ReadBufResult::Found(guard) => return Ok(guard),
-                ReadBufResult::NotFound(mut write_guard) => {
-                    // Read the page from disk into the buffer
-                    self.fill_buffer(write_guard.deref_mut(), blknum)?;
-                    write_guard.mark_valid();
-
-                    // Swap for read lock
-                    continue;
-                }
-            };
-        }
-    }
-}
-
 #[cfg(test)]
 mod tests {
    use super::*;
-    use crate::layered_repository::blob_io::{BlobCursor, BlobWriter};
-    use crate::layered_repository::block_io::BlockCursor;
-    use rand::{seq::SliceRandom, thread_rng, RngCore};
+    use rand::seq::SliceRandom;
+    use rand::thread_rng;
    use std::fs;
    use std::str::FromStr;

@@ -327,19 +281,19 @@ mod tests {
    fn test_ephemeral_files() -> Result<(), Error> {
        let (conf, tenantid, timelineid) = repo_harness("ephemeral_files")?;

-        let file_a = EphemeralFile::create(conf, tenantid, timelineid)?;
+        let mut file_a = EphemeralFile::create(conf, tenantid, timelineid)?;

-        file_a.write_all_at(b"foo", 0)?;
+        file_a.write_all(b"foo")?;
        assert_eq!("foo", read_string(&file_a, 0, 20)?);

-        file_a.write_all_at(b"bar", 3)?;
+        file_a.write_all(b"bar")?;
        assert_eq!("foobar", read_string(&file_a, 0, 20)?);

        // Open a lot of files, enough to cause some page evictions.
        let mut efiles = Vec::new();
        for fileno in 0..100 {
-            let efile = EphemeralFile::create(conf, tenantid, timelineid)?;
-            efile.write_all_at(format!("file {}", fileno).as_bytes(), 0)?;
+            let mut efile = EphemeralFile::create(conf, tenantid, timelineid)?;
+            efile.write_all(format!("file {}", fileno).as_bytes())?;
            assert_eq!(format!("file {}", fileno), read_string(&efile, 0, 10)?);
            efiles.push((fileno, efile));
        }
@@ -353,41 +307,4 @@ mod tests {

        Ok(())
    }
-
-    #[test]
-    fn test_ephemeral_blobs() -> Result<(), Error> {
-        let (conf, tenantid, timelineid) = repo_harness("ephemeral_blobs")?;
-
-        let mut file = EphemeralFile::create(conf, tenantid, timelineid)?;
-
-        let pos_foo = file.write_blob(b"foo")?;
-        assert_eq!(b"foo", file.block_cursor().read_blob(pos_foo)?.as_slice());
-        let pos_bar = file.write_blob(b"bar")?;
-        assert_eq!(b"foo", file.block_cursor().read_blob(pos_foo)?.as_slice());
-        assert_eq!(b"bar", file.block_cursor().read_blob(pos_bar)?.as_slice());
-
-        let mut blobs = Vec::new();
-        for i in 0..10000 {
-            let data = Vec::from(format!("blob{}", i).as_bytes());
-            let pos = file.write_blob(&data)?;
-            blobs.push((pos, data));
-        }
-
-        let mut cursor = BlockCursor::new(&file);
-        for (pos, expected) in blobs {
-            let actual = cursor.read_blob(pos)?;
-            assert_eq!(actual, expected);
-        }
-        drop(cursor);
-
-        // Test a large blob that spans multiple pages
-        let mut large_data = Vec::new();
-        large_data.resize(20000, 0);
-        thread_rng().fill_bytes(&mut large_data);
-        let pos_large = file.write_blob(&large_data)?;
-        let result = file.block_cursor().read_blob(pos_large)?;
-        assert_eq!(result, large_data);
-
-        Ok(())
-    }
 }
--- a/pageserver/src/layered_repository/filename.rs
+++ b/pageserver/src/layered_repository/filename.rs
@@ -25,7 +25,9 @@ impl PartialOrd for DeltaFileName {

 impl Ord for DeltaFileName {
    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.key_range.start.cmp(&other.key_range.start);
+        let mut cmp;
+
+        cmp = self.key_range.start.cmp(&other.key_range.start);
        if cmp != Ordering::Equal {
            return cmp;
        }
@@ -115,7 +117,9 @@ impl PartialOrd for ImageFileName {

 impl Ord for ImageFileName {
    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.key_range.start.cmp(&other.key_range.start);
+        let mut cmp;
+
+        cmp = self.key_range.start.cmp(&other.key_range.start);
        if cmp != Ordering::Equal {
            return cmp;
        }
--- a/pageserver/src/layered_repository/image_layer.rs
+++ b/pageserver/src/layered_repository/image_layer.rs
@@ -13,76 +13,63 @@
 //!
 //!    000000067F000032BE0000400000000070B6-000000067F000032BE0000400000000080B6__00000000346BC568
 //!
-//! Every image layer file consists of three parts: "summary",
-//! "index", and "values".  The summary is a fixed size header at the
-//! beginning of the file, and it contains basic information about the
-//! layer, and offsets to the other parts. The "index" is a B-tree,
-//! mapping from Key to an offset in the "values" part.  The
-//! actual page images are stored in the "values" part.
+//! An image file is constructed using the 'bookfile' crate.
+//!
+//! Only metadata is loaded into memory by the load function.
+//! When images are needed, they are read directly from disk.
+//!
 use crate::config::PageServerConf;
-use crate::layered_repository::blob_io::{BlobCursor, BlobWriter, WriteBlobWriter};
-use crate::layered_repository::block_io::{BlockBuf, BlockReader, FileBlockReader};
-use crate::layered_repository::disk_btree::{DiskBtreeBuilder, DiskBtreeReader, VisitDirection};
 use crate::layered_repository::filename::{ImageFileName, PathOrConf};
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    BlobRef, Layer, ValueReconstructResult, ValueReconstructState,
 };
-use crate::page_cache::PAGE_SZ;
-use crate::repository::{Key, Value, KEY_SIZE};
+use crate::repository::{Key, Value};
 use crate::virtual_file::VirtualFile;
+use crate::IMAGE_FILE_MAGIC;
 use crate::{ZTenantId, ZTimelineId};
-use crate::{IMAGE_FILE_MAGIC, STORAGE_FORMAT_VERSION};
 use anyhow::{bail, ensure, Context, Result};
 use bytes::Bytes;
-use hex;
 use log::*;
 use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
 use std::fs;
-use std::io::Write;
-use std::io::{Seek, SeekFrom};
+use std::io::{BufWriter, Write};
 use std::ops::Range;
 use std::path::{Path, PathBuf};
-use std::sync::{RwLock, RwLockReadGuard};
+use std::sync::{RwLock, RwLockReadGuard, TryLockError};
+
+use bookfile::{Book, BookWriter, ChapterWriter};

 use zenith_utils::bin_ser::BeSer;
 use zenith_utils::lsn::Lsn;

-///
-/// Header stored in the beginning of the file
-///
-/// After this comes the 'values' part, starting on block 1. After that,
-/// the 'index' starts at the block indicated by 'index_start_blk'
-///
+/// Mapping from (key, lsn) -> page/WAL record
+/// byte ranges in VALUES_CHAPTER
+static INDEX_CHAPTER: u64 = 1;
+
+/// Contains each block in block # order
+const VALUES_CHAPTER: u64 = 2;
+
+/// Contains the [`Summary`] struct
+const SUMMARY_CHAPTER: u64 = 3;
+
 #[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
 struct Summary {
-    /// Magic value to identify this as a zenith image file. Always IMAGE_FILE_MAGIC.
-    magic: u16,
-    format_version: u16,
-
    tenantid: ZTenantId,
    timelineid: ZTimelineId,
    key_range: Range<Key>,
-    lsn: Lsn,

-    /// Block number where the 'index' part of the file begins.
-    index_start_blk: u32,
-    /// Block within the 'index', where the B-tree root page is stored
-    index_root_blk: u32,
-    // the 'values' part starts after the summary header, on block 1.
+    lsn: Lsn,
 }

 impl From<&ImageLayer> for Summary {
    fn from(layer: &ImageLayer) -> Self {
        Self {
-            magic: IMAGE_FILE_MAGIC,
-            format_version: STORAGE_FORMAT_VERSION,
            tenantid: layer.tenantid,
            timelineid: layer.timelineid,
            key_range: layer.key_range.clone(),
-            lsn: layer.lsn,

-            index_start_blk: 0,
-            index_root_blk: 0,
+            lsn: layer.lsn,
        }
    }
 }
@@ -110,12 +97,12 @@ pub struct ImageLayerInner {
    /// If false, the 'index' has not been loaded into memory yet.
    loaded: bool,

-    // values copied from summary
-    index_start_blk: u32,
-    index_root_blk: u32,
+    /// The underlying (virtual) file handle. None if the layer hasn't been loaded
+    /// yet.
+    book: Option<Book<VirtualFile>>,

-    /// Reader object for reading blocks from the file. (None if not loaded yet)
-    file: Option<FileBlockReader<VirtualFile>>,
+    /// offset of each value
+    index: HashMap<Key, BlobRef>,
 }

 impl Layer for ImageLayer {
@@ -152,19 +139,24 @@ impl Layer for ImageLayer {

        let inner = self.load()?;

-        let file = inner.file.as_ref().unwrap();
-        let tree_reader = DiskBtreeReader::new(inner.index_start_blk, inner.index_root_blk, file);
+        if let Some(blob_ref) = inner.index.get(&key) {
+            let chapter = inner
+                .book
+                .as_ref()
+                .unwrap()
+                .chapter_reader(VALUES_CHAPTER)?;

-        let mut keybuf: [u8; KEY_SIZE] = [0u8; KEY_SIZE];
-        key.write_to_byte_slice(&mut keybuf);
-        if let Some(offset) = tree_reader.get(&keybuf)? {
-            let blob = file.block_cursor().read_blob(offset).with_context(|| {
-                format!(
-                    "failed to read value from data file {} at offset {}",
-                    self.filename().display(),
-                    offset
-                )
-            })?;
+            let mut blob = vec![0; blob_ref.size()];
+            chapter
+                .read_exact_at(&mut blob, blob_ref.pos())
+                .with_context(|| {
+                    format!(
+                        "failed to read {} bytes from data file {} at offset {}",
+                        blob_ref.size(),
+                        self.filename().display(),
+                        blob_ref.pos()
+                    )
+                })?;
            let value = Bytes::from(blob);

            reconstruct_state.img = Some((self.lsn, value));
@@ -178,6 +170,33 @@ impl Layer for ImageLayer {
        todo!();
    }

+    fn unload(&self) -> Result<()> {
+        // Unload the index.
+        //
+        // TODO: we should access the index directly from pages on the disk,
+        // using the buffer cache. This load/unload mechanism is really ad hoc.
+
+        // FIXME: In debug mode, loading and unloading the index slows
+        // things down so much that you get timeout errors. At least
+        // with the test_parallel_copy test. So as an even more ad hoc
+        // stopgap fix for that, only unload every on average 10
+        // checkpoint cycles.
+        use rand::RngCore;
+        if rand::thread_rng().next_u32() > (u32::MAX / 10) {
+            return Ok(());
+        }
+
+        let mut inner = match self.inner.try_write() {
+            Ok(inner) => inner,
+            Err(TryLockError::WouldBlock) => return Ok(()),
+            Err(TryLockError::Poisoned(_)) => panic!("ImageLayer lock was poisoned"),
+        };
+        inner.index = HashMap::default();
+        inner.loaded = false;
+
+        Ok(())
+    }
+
    fn delete(&self) -> Result<()> {
        // delete underlying file
        fs::remove_file(self.path())?;
@@ -204,16 +223,18 @@ impl Layer for ImageLayer {
        }

        let inner = self.load()?;
-        let file = inner.file.as_ref().unwrap();
-        let tree_reader =
-            DiskBtreeReader::<_, KEY_SIZE>::new(inner.index_start_blk, inner.index_root_blk, file);

-        tree_reader.dump()?;
+        let mut index_vec: Vec<(&Key, &BlobRef)> = inner.index.iter().collect();
+        index_vec.sort_by_key(|x| x.1.pos());

-        tree_reader.visit(&[0u8; KEY_SIZE], VisitDirection::Forwards, |key, value| {
-            println!("key: {} offset {}", hex::encode(key), value);
-            true
-        })?;
+        for (key, blob_ref) in index_vec {
+            println!(
+                "key: {} size {} offset {}",
+                key,
+                blob_ref.size(),
+                blob_ref.pos()
+            );
+        }

        Ok(())
    }
@@ -270,20 +291,21 @@ impl ImageLayer {
        let path = self.path();

        // Open the file if it's not open already.
-        if inner.file.is_none() {
+        if inner.book.is_none() {
            let file = VirtualFile::open(&path)
                .with_context(|| format!("Failed to open file '{}'", path.display()))?;
-            inner.file = Some(FileBlockReader::new(file));
+            inner.book = Some(Book::new(file).with_context(|| {
+                format!("Failed to open file '{}' as a bookfile", path.display())
+            })?);
        }
-        let file = inner.file.as_mut().unwrap();
-        let summary_blk = file.read_blk(0)?;
-        let actual_summary = Summary::des_prefix(summary_blk.as_ref())?;
+        let book = inner.book.as_ref().unwrap();

        match &self.path_or_conf {
            PathOrConf::Conf(_) => {
-                let mut expected_summary = Summary::from(self);
-                expected_summary.index_start_blk = actual_summary.index_start_blk;
-                expected_summary.index_root_blk = actual_summary.index_root_blk;
+                let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+                let actual_summary = Summary::des(&chapter)?;
+
+                let expected_summary = Summary::from(self);

                if actual_summary != expected_summary {
                    bail!("in-file summary does not match expected summary. actual = {:?} expected = {:?}", actual_summary, expected_summary);
@@ -303,9 +325,14 @@ impl ImageLayer {
            }
        }

-        inner.index_start_blk = actual_summary.index_start_blk;
-        inner.index_root_blk = actual_summary.index_root_blk;
+        let chapter = book.read_chapter(INDEX_CHAPTER)?;
+        let index = HashMap::des(&chapter)?;
+
+        info!("loaded from {}", &path.display());
+
+        inner.index = index;
        inner.loaded = true;
+
        Ok(())
    }

@@ -323,10 +350,9 @@ impl ImageLayer {
            key_range: filename.key_range.clone(),
            lsn: filename.lsn,
            inner: RwLock::new(ImageLayerInner {
+                book: None,
+                index: HashMap::new(),
                loaded: false,
-                file: None,
-                index_start_blk: 0,
-                index_root_blk: 0,
            }),
        }
    }
@@ -334,14 +360,12 @@ impl ImageLayer {
    /// Create an ImageLayer struct representing an existing file on disk.
    ///
    /// This variant is only used for debugging purposes, by the 'dump_layerfile' binary.
-    pub fn new_for_path<F>(path: &Path, file: F) -> Result<ImageLayer>
+    pub fn new_for_path<F>(path: &Path, book: &Book<F>) -> Result<ImageLayer>
    where
        F: std::os::unix::prelude::FileExt,
    {
-        let mut summary_buf = Vec::new();
-        summary_buf.resize(PAGE_SZ, 0);
-        file.read_exact_at(&mut summary_buf, 0)?;
-        let summary = Summary::des_prefix(&summary_buf)?;
+        let chapter = book.read_chapter(SUMMARY_CHAPTER)?;
+        let summary = Summary::des(&chapter)?;

        Ok(ImageLayer {
            path_or_conf: PathOrConf::Path(path.to_path_buf()),
@@ -350,10 +374,9 @@ impl ImageLayer {
            key_range: summary.key_range,
            lsn: summary.lsn,
            inner: RwLock::new(ImageLayerInner {
-                file: None,
+                book: None,
+                index: HashMap::new(),
                loaded: false,
-                index_start_blk: 0,
-                index_root_blk: 0,
            }),
        })
    }
@@ -382,21 +405,25 @@ impl ImageLayer {
 ///
 /// 1. Create the ImageLayerWriter by calling ImageLayerWriter::new(...)
 ///
-/// 2. Write the contents by calling `put_page_image` for every key-value
-///    pair in the key range.
+/// 2. Write the contents by calling `put_page_image` for every page
+///    in the segment.
 ///
 /// 3. Call `finish`.
 ///
 pub struct ImageLayerWriter {
    conf: &'static PageServerConf,
-    _path: PathBuf,
+    path: PathBuf,
    timelineid: ZTimelineId,
    tenantid: ZTenantId,
    key_range: Range<Key>,
    lsn: Lsn,

-    blob_writer: WriteBlobWriter<VirtualFile>,
-    tree: DiskBtreeBuilder<BlockBuf, KEY_SIZE>,
+    values_writer: Option<ChapterWriter<BufWriter<VirtualFile>>>,
+    end_offset: u64,
+
+    index: HashMap<Key, BlobRef>,
+
+    finished: bool,
 }

 impl ImageLayerWriter {
@@ -421,24 +448,25 @@ impl ImageLayerWriter {
            },
        );
        info!("new image layer {}", path.display());
-        let mut file = VirtualFile::create(&path)?;
-        // make room for the header block
-        file.seek(SeekFrom::Start(PAGE_SZ as u64))?;
-        let blob_writer = WriteBlobWriter::new(file, PAGE_SZ as u64);
+        let file = VirtualFile::create(&path)?;
+        let buf_writer = BufWriter::new(file);
+        let book = BookWriter::new(buf_writer, IMAGE_FILE_MAGIC)?;

-        // Initialize the b-tree index builder
-        let block_buf = BlockBuf::new();
-        let tree_builder = DiskBtreeBuilder::new(block_buf);
+        // Open the page-images chapter for writing. The calls to
+        // `put_image` will use this to write the contents.
+        let chapter = book.new_chapter(VALUES_CHAPTER);

        let writer = ImageLayerWriter {
            conf,
-            _path: path,
+            path,
            timelineid,
            tenantid,
            key_range: key_range.clone(),
            lsn,
-            tree: tree_builder,
-            blob_writer,
+            values_writer: Some(chapter),
+            index: HashMap::new(),
+            end_offset: 0,
+            finished: false,
        };

        Ok(writer)
@@ -451,45 +479,49 @@ impl ImageLayerWriter {
    ///
    pub fn put_image(&mut self, key: Key, img: &[u8]) -> Result<()> {
        ensure!(self.key_range.contains(&key));
-        let off = self.blob_writer.write_blob(img)?;
+        let off = self.end_offset;

-        let mut keybuf: [u8; KEY_SIZE] = [0u8; KEY_SIZE];
-        key.write_to_byte_slice(&mut keybuf);
-        self.tree.append(&keybuf, off)?;
+        if let Some(writer) = &mut self.values_writer {
+            let len = img.len();
+            writer.write_all(img)?;
+            self.end_offset += len as u64;
+
+            let old = self.index.insert(key, BlobRef::new(off, len, true));
+            assert!(old.is_none());
+        } else {
+            panic!()
+        }

        Ok(())
    }

-    pub fn finish(self) -> anyhow::Result<ImageLayer> {
-        let index_start_blk =
-            ((self.blob_writer.size() + PAGE_SZ as u64 - 1) / PAGE_SZ as u64) as u32;
-
-        let mut file = self.blob_writer.into_inner();
+    pub fn finish(&mut self) -> anyhow::Result<ImageLayer> {
+        // Close the values chapter
+        let book = self.values_writer.take().unwrap().close()?;

        // Write out the index
-        file.seek(SeekFrom::Start(index_start_blk as u64 * PAGE_SZ as u64))?;
-        let (index_root_blk, block_buf) = self.tree.finish()?;
-        for buf in block_buf.blocks {
-            file.write_all(buf.as_ref())?;
-        }
+        let mut chapter = book.new_chapter(INDEX_CHAPTER);
+        let buf = HashMap::ser(&self.index)?;
+        chapter.write_all(&buf)?;
+        let book = chapter.close()?;

-        // Fill in the summary on blk 0
+        // Write out the summary chapter
+        let mut chapter = book.new_chapter(SUMMARY_CHAPTER);
        let summary = Summary {
-            magic: IMAGE_FILE_MAGIC,
-            format_version: STORAGE_FORMAT_VERSION,
            tenantid: self.tenantid,
            timelineid: self.timelineid,
            key_range: self.key_range.clone(),
            lsn: self.lsn,
-            index_start_blk,
-            index_root_blk,
        };
-        file.seek(SeekFrom::Start(0))?;
-        Summary::ser_into(&summary, &mut file)?;
+        Summary::ser_into(&summary, &mut chapter)?;
+        let book = chapter.close()?;
+
+        // This flushes the underlying 'buf_writer'.
+        book.close()?;

        // Note: Because we open the file in write-only mode, we cannot
        // reuse the same VirtualFile for reading later. That's why we don't
-        // set inner.file here. The first read will have to re-open it.
+        // set inner.book here. The first read will have to re-open it.
        let layer = ImageLayer {
            path_or_conf: PathOrConf::Conf(self.conf),
            timelineid: self.timelineid,
@@ -497,14 +529,28 @@ impl ImageLayerWriter {
            key_range: self.key_range.clone(),
            lsn: self.lsn,
            inner: RwLock::new(ImageLayerInner {
+                book: None,
                loaded: false,
-                file: None,
-                index_start_blk,
-                index_root_blk,
+                index: HashMap::new(),
            }),
        };
        trace!("created image layer {}", layer.path().display());

+        self.finished = true;
+
        Ok(layer)
    }
 }
+
+impl Drop for ImageLayerWriter {
+    fn drop(&mut self) {
+        if let Some(page_image_writer) = self.values_writer.take() {
+            if let Ok(book) = page_image_writer.close() {
+                let _ = book.close();
+            }
+        }
+        if !self.finished {
+            let _ = fs::remove_file(&self.path);
+        }
+    }
+}
--- a/pageserver/src/layered_repository/inmemory_layer.rs
+++ b/pageserver/src/layered_repository/inmemory_layer.rs
@@ -5,12 +5,10 @@
 //! its position in the file, is kept in memory, though.
 //!
 use crate::config::PageServerConf;
-use crate::layered_repository::blob_io::{BlobCursor, BlobWriter};
-use crate::layered_repository::block_io::BlockReader;
 use crate::layered_repository::delta_layer::{DeltaLayer, DeltaLayerWriter};
 use crate::layered_repository::ephemeral_file::EphemeralFile;
 use crate::layered_repository::storage_layer::{
-    Layer, ValueReconstructResult, ValueReconstructState,
+    BlobRef, Layer, ValueReconstructResult, ValueReconstructState,
 };
 use crate::repository::{Key, Value};
 use crate::walrecord;
@@ -21,7 +19,9 @@ use std::collections::HashMap;
 // avoid binding to Write (conflicts with std::io::Write)
 // while being able to use std::fmt::Write's methods
 use std::fmt::Write as _;
+use std::io::Write;
 use std::ops::Range;
+use std::os::unix::fs::FileExt;
 use std::path::PathBuf;
 use std::sync::RwLock;
 use zenith_utils::bin_ser::BeSer;
@@ -54,12 +54,14 @@ pub struct InMemoryLayerInner {
    /// by block number and LSN. The value is an offset into the
    /// ephemeral file where the page version is stored.
    ///
-    index: HashMap<Key, VecMap<Lsn, u64>>,
+    index: HashMap<Key, VecMap<Lsn, BlobRef>>,

    /// The values are stored in a serialized format in this file.
    /// Each serialized Value is preceded by a 'u32' length field.
    /// PerSeg::page_versions map stores offsets into this file.
    file: EphemeralFile,
+
+    end_offset: u64,
 }

 impl InMemoryLayerInner {
@@ -118,12 +120,10 @@ impl Layer for InMemoryLayer {

        let inner = self.inner.read().unwrap();

-        let mut reader = inner.file.block_cursor();
-
        // Scan the page versions backwards, starting from `lsn`.
        if let Some(vec_map) = inner.index.get(&key) {
            let slice = vec_map.slice_range(lsn_range);
-            for (entry_lsn, pos) in slice.iter().rev() {
+            for (entry_lsn, blob_ref) in slice.iter().rev() {
                match &reconstruct_state.img {
                    Some((cached_lsn, _)) if entry_lsn <= cached_lsn => {
                        return Ok(ValueReconstructResult::Complete)
@@ -131,7 +131,8 @@ impl Layer for InMemoryLayer {
                    _ => {}
                }

-                let buf = reader.read_blob(*pos)?;
+                let mut buf = vec![0u8; blob_ref.size()];
+                inner.file.read_exact_at(&mut buf, blob_ref.pos())?;
                let value = Value::des(&buf)?;
                match value {
                    Value::Image(img) => {
@@ -166,6 +167,13 @@ impl Layer for InMemoryLayer {
        todo!();
    }

+    /// Cannot unload anything in an in-memory layer, since there's no backing
+    /// store. To release memory used by an in-memory layer, use 'freeze' to turn
+    /// it into an on-disk layer.
+    fn unload(&self) -> Result<()> {
+        Ok(())
+    }
+
    /// Nothing to do here. When you drop the last reference to the layer, it will
    /// be deallocated.
    fn delete(&self) -> Result<()> {
@@ -200,12 +208,12 @@ impl Layer for InMemoryLayer {
            return Ok(());
        }

-        let mut cursor = inner.file.block_cursor();
        let mut buf = Vec::new();
        for (key, vec_map) in inner.index.iter() {
-            for (lsn, pos) in vec_map.as_slice() {
+            for (lsn, blob_ref) in vec_map.as_slice() {
                let mut desc = String::new();
-                cursor.read_blob_into_buf(*pos, &mut buf)?;
+                buf.resize(blob_ref.size(), 0);
+                inner.file.read_exact_at(&mut buf, blob_ref.pos())?;
                let val = Value::des(&buf);
                match val {
                    Ok(Value::Image(img)) => {
@@ -260,6 +268,7 @@ impl InMemoryLayer {
                end_lsn: None,
                index: HashMap::new(),
                file,
+                end_offset: 0,
            }),
        })
    }
@@ -274,10 +283,15 @@ impl InMemoryLayer {

        inner.assert_writeable();

-        let off = inner.file.write_blob(&Value::ser(&val)?)?;
+        let off = inner.end_offset;
+        let buf = Value::ser(&val)?;
+        let len = buf.len();
+        inner.file.write_all(&buf)?;
+        inner.end_offset += len as u64;

        let vec_map = inner.index.entry(key).or_default();
-        let old = vec_map.append_or_update_last(lsn, off).unwrap().0;
+        let blob_ref = BlobRef::new(off, len, val.will_init());
+        let old = vec_map.append_or_update_last(lsn, blob_ref).unwrap().0;
        if old.is_some() {
            // We already had an entry for this LSN. That's odd..
            warn!("Key {} at {} already exists", key, lsn);
@@ -331,21 +345,21 @@ impl InMemoryLayer {
            self.start_lsn..inner.end_lsn.unwrap(),
        )?;

-        let mut buf = Vec::new();
-
-        let mut cursor = inner.file.block_cursor();
-
-        let mut keys: Vec<(&Key, &VecMap<Lsn, u64>)> = inner.index.iter().collect();
-        keys.sort_by_key(|k| k.0);
-
-        for (key, vec_map) in keys.iter() {
-            let key = **key;
-            // Write all page versions
-            for (lsn, pos) in vec_map.as_slice() {
-                cursor.read_blob_into_buf(*pos, &mut buf)?;
-                let val = Value::des(&buf)?;
-                delta_layer_writer.put_value(key, *lsn, val)?;
+        let mut do_steps = || -> Result<()> {
+            for (key, vec_map) in inner.index.iter() {
+                // Write all page versions
+                for (lsn, blob_ref) in vec_map.as_slice() {
+                    let mut buf = vec![0u8; blob_ref.size()];
+                    inner.file.read_exact_at(&mut buf, blob_ref.pos())?;
+                    let val = Value::des(&buf)?;
+                    delta_layer_writer.put_value(*key, *lsn, val)?;
+                }
            }
+            Ok(())
+        };
+        if let Err(err) = do_steps() {
+            delta_layer_writer.abort();
+            return Err(err);
        }

        let delta_layer = delta_layer_writer.finish(Key::MAX)?;
--- a/pageserver/src/layered_repository/layer_map.rs
+++ b/pageserver/src/layered_repository/layer_map.rs
@@ -207,11 +207,11 @@ impl LayerMap {
        NUM_ONDISK_LAYERS.dec();
    }

-    /// Is there a newer image layer for given key-range?
+    /// Is there a newer image layer for given segment?
    ///
    /// This is used for garbage collection, to determine if an old layer can
    /// be deleted.
-    /// We ignore layers newer than disk_consistent_lsn because they will be removed at restart
+    /// We ignore segments newer than disk_consistent_lsn because they will be removed at restart
    /// We also only look at historic layers
    //#[allow(dead_code)]
    pub fn newer_image_layer_exists(
@@ -250,6 +250,22 @@ impl LayerMap {
        }
    }

+    /// Is there any layer for given segment that is alive at the lsn?
+    ///
+    /// This is a public wrapper for SegEntry fucntion,
+    /// used for garbage collection, to determine if some alive layer
+    /// exists at the lsn. If so, we shouldn't delete a newer dropped layer
+    /// to avoid incorrectly making it visible.
+    /*
+        pub fn layer_exists_at_lsn(&self, seg: SegmentTag, lsn: Lsn) -> Result<bool> {
+            Ok(if let Some(segentry) = self.historic_layers.get(&seg) {
+                segentry.exists_at_lsn(seg, lsn)?.unwrap_or(false)
+            } else {
+                false
+            })
+        }
+    */
+
    pub fn iter_historic_layers(&self) -> std::slice::Iter<Arc<dyn Layer>> {
        self.historic_layers.iter()
    }
@@ -296,7 +312,9 @@ impl LayerMap {
        key_range: &Range<Key>,
        lsn: Lsn,
    ) -> Result<Vec<(Range<Key>, Option<Arc<dyn Layer>>)>> {
-        let mut points = vec![key_range.start];
+        let mut points: Vec<Key>;
+
+        points = vec![key_range.start];
        for l in self.historic_layers.iter() {
            if l.get_lsn_range().start > lsn {
                continue;
--- a/pageserver/src/layered_repository/storage_layer.rs
+++ b/pageserver/src/layered_repository/storage_layer.rs
@@ -7,6 +7,7 @@ use crate::walrecord::ZenithWalRecord;
 use crate::{ZTenantId, ZTimelineId};
 use anyhow::Result;
 use bytes::Bytes;
+use serde::{Deserialize, Serialize};
 use std::ops::Range;
 use std::path::PathBuf;

@@ -87,7 +88,7 @@ pub trait Layer: Send + Sync {
    /// Identify the timeline this layer belongs to
    fn get_timeline_id(&self) -> ZTimelineId;

-    /// Range of keys that this layer covers
+    /// Range of segments that this layer covers
    fn get_key_range(&self) -> Range<Key>;

    /// Inclusive start bound of the LSN range that this layer holds
@@ -122,7 +123,7 @@ pub trait Layer: Send + Sync {
        reconstruct_data: &mut ValueReconstructState,
    ) -> Result<ValueReconstructResult>;

-    /// Does this layer only contain some data for the key-range (incremental),
+    /// Does this layer only contain some data for the segment (incremental),
    /// or does it contain a version of every page? This is important to know
    /// for garbage collecting old layers: an incremental layer depends on
    /// the previous non-incremental layer.
@@ -134,9 +135,46 @@ pub trait Layer: Send + Sync {
    /// Iterate through all keys and values stored in the layer
    fn iter(&self) -> Box<dyn Iterator<Item = Result<(Key, Lsn, Value)>> + '_>;

+    /// Release memory used by this layer. There is no corresponding 'load'
+    /// function, that's done implicitly when you call one of the get-functions.
+    fn unload(&self) -> Result<()>;
+
    /// Permanently remove this layer from disk.
    fn delete(&self) -> Result<()>;

    /// Dump summary of the contents of the layer to stdout
    fn dump(&self, verbose: bool) -> Result<()>;
 }
+
+// Flag indicating that this version initialize the page
+const WILL_INIT: u64 = 1;
+
+///
+/// Struct representing reference to BLOB in layers. Reference contains BLOB offset and size.
+/// For WAL records (delta layer) it also contains `will_init` flag which helps to determine range of records
+/// which needs to be applied without reading/deserializing records themselves.
+///
+#[derive(Debug, Serialize, Deserialize, Copy, Clone)]
+pub struct BlobRef(u64);
+
+impl BlobRef {
+    pub fn will_init(&self) -> bool {
+        (self.0 & WILL_INIT) != 0
+    }
+
+    pub fn pos(&self) -> u64 {
+        self.0 >> 32
+    }
+
+    pub fn size(&self) -> usize {
+        ((self.0 & 0xFFFFFFFF) >> 1) as usize
+    }
+
+    pub fn new(pos: u64, size: usize, will_init: bool) -> BlobRef {
+        let mut blob_ref = (pos << 32) | ((size as u64) << 1);
+        if will_init {
+            blob_ref |= WILL_INIT;
+        }
+        BlobRef(blob_ref)
+    }
+}
--- a/pageserver/src/lib.rs
+++ b/pageserver/src/lib.rs
@@ -19,7 +19,6 @@ pub mod walingest;
 pub mod walreceiver;
 pub mod walrecord;
 pub mod walredo;
-pub mod wal_metadata;

 use lazy_static::lazy_static;
 use tracing::info;
@@ -39,11 +38,11 @@ use pgdatadir_mapping::DatadirTimeline;
 /// This is embedded in the metadata file, and also in the header of all the
 /// layer files. If you make any backwards-incompatible changes to the storage
 /// format, bump this!
-pub const STORAGE_FORMAT_VERSION: u16 = 3;
+pub const STORAGE_FORMAT_VERSION: u16 = 1;

 // Magic constants used to identify different kinds of files
-pub const IMAGE_FILE_MAGIC: u16 = 0x5A60;
-pub const DELTA_FILE_MAGIC: u16 = 0x5A61;
+pub const IMAGE_FILE_MAGIC: u32 = 0x5A60_0000 | STORAGE_FORMAT_VERSION as u32;
+pub const DELTA_FILE_MAGIC: u32 = 0x5A61_0000 | STORAGE_FORMAT_VERSION as u32;

 lazy_static! {
    static ref LIVE_CONNECTIONS_COUNT: IntGaugeVec = register_int_gauge_vec!(
--- a/pageserver/src/page_cache.rs
+++ b/pageserver/src/page_cache.rs
@@ -41,7 +41,7 @@ use std::{
    convert::TryInto,
    sync::{
        atomic::{AtomicU8, AtomicUsize, Ordering},
-        RwLock, RwLockReadGuard, RwLockWriteGuard, TryLockError,
+        RwLock, RwLockReadGuard, RwLockWriteGuard,
    },
 };

@@ -52,12 +52,11 @@ use zenith_utils::{
    zid::{ZTenantId, ZTimelineId},
 };

-use crate::config::PageServerConf;
 use crate::layered_repository::writeback_ephemeral_file;
 use crate::repository::Key;

 static PAGE_CACHE: OnceCell<PageCache> = OnceCell::new();
-const TEST_PAGE_CACHE_SIZE: usize = 50;
+const TEST_PAGE_CACHE_SIZE: usize = 10;

 ///
 /// Initialize the page cache. This must be called once at page server startup.
@@ -91,7 +90,6 @@ const MAX_USAGE_COUNT: u8 = 5;
 /// CacheKey uniquely identifies a "thing" to cache in the page cache.
 ///
 #[derive(Debug, PartialEq, Eq, Clone)]
-#[allow(clippy::enum_variant_names)]
 enum CacheKey {
    MaterializedPage {
        hash_key: MaterializedPageHashKey,
@@ -101,10 +99,6 @@ enum CacheKey {
        file_id: u64,
        blkno: u32,
    },
-    ImmutableFilePage {
-        file_id: u64,
-        blkno: u32,
-    },
 }

 #[derive(Debug, PartialEq, Eq, Hash, Clone)]
@@ -179,8 +173,6 @@ pub struct PageCache {

    ephemeral_page_map: RwLock<HashMap<(u64, u32), usize>>,

-    immutable_page_map: RwLock<HashMap<(u64, u32), usize>>,
-
    /// The actual buffers with their metadata.
    slots: Box<[Slot]>,

@@ -203,12 +195,6 @@ impl std::ops::Deref for PageReadGuard<'_> {
    }
 }

-impl AsRef<[u8; PAGE_SZ]> for PageReadGuard<'_> {
-    fn as_ref(&self) -> &[u8; PAGE_SZ] {
-        self.0.buf
-    }
-}
-
 ///
 /// PageWriteGuard is a lease on a buffer for modifying it. The page is kept locked
 /// until the guard is dropped.
@@ -240,12 +226,6 @@ impl std::ops::Deref for PageWriteGuard<'_> {
    }
 }

-impl AsMut<[u8; PAGE_SZ]> for PageWriteGuard<'_> {
-    fn as_mut(&mut self) -> &mut [u8; PAGE_SZ] {
-        self.inner.buf
-    }
-}
-
 impl PageWriteGuard<'_> {
    /// Mark that the buffer contents are now valid.
    pub fn mark_valid(&mut self) {
@@ -401,36 +381,6 @@ impl PageCache {
        }
    }

-    // Section 1.3: Public interface functions for working with immutable file pages.
-
-    pub fn read_immutable_buf(&self, file_id: u64, blkno: u32) -> ReadBufResult {
-        let mut cache_key = CacheKey::ImmutableFilePage { file_id, blkno };
-
-        self.lock_for_read(&mut cache_key)
-    }
-
-    /// Immediately drop all buffers belonging to given file, without writeback
-    pub fn drop_buffers_for_immutable(&self, drop_file_id: u64) {
-        for slot_idx in 0..self.slots.len() {
-            let slot = &self.slots[slot_idx];
-
-            let mut inner = slot.inner.write().unwrap();
-            if let Some(key) = &inner.key {
-                match key {
-                    CacheKey::ImmutableFilePage { file_id, blkno: _ }
-                        if *file_id == drop_file_id =>
-                    {
-                        // remove mapping for old buffer
-                        self.remove_mapping(key);
-                        inner.key = None;
-                        inner.dirty = false;
-                    }
-                    _ => {}
-                }
-            }
-        }
-    }
-
    //
    // Section 2: Internal interface functions for lookup/update.
    //
@@ -628,10 +578,6 @@ impl PageCache {
                let map = self.ephemeral_page_map.read().unwrap();
                Some(*map.get(&(*file_id, *blkno))?)
            }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let map = self.immutable_page_map.read().unwrap();
-                Some(*map.get(&(*file_id, *blkno))?)
-            }
        }
    }

@@ -655,10 +601,6 @@ impl PageCache {
                let map = self.ephemeral_page_map.read().unwrap();
                Some(*map.get(&(*file_id, *blkno))?)
            }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let map = self.immutable_page_map.read().unwrap();
-                Some(*map.get(&(*file_id, *blkno))?)
-            }
        }
    }

@@ -690,11 +632,6 @@ impl PageCache {
                map.remove(&(*file_id, *blkno))
                    .expect("could not find old key in mapping");
            }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let mut map = self.immutable_page_map.write().unwrap();
-                map.remove(&(*file_id, *blkno))
-                    .expect("could not find old key in mapping");
-            }
        }
    }

@@ -735,16 +672,6 @@ impl PageCache {
                    }
                }
            }
-            CacheKey::ImmutableFilePage { file_id, blkno } => {
-                let mut map = self.immutable_page_map.write().unwrap();
-                match map.entry((*file_id, *blkno)) {
-                    Entry::Occupied(entry) => Some(*entry.get()),
-                    Entry::Vacant(entry) => {
-                        entry.insert(slot_idx);
-                        None
-                    }
-                }
-            }
        }
    }

@@ -756,33 +683,16 @@ impl PageCache {
    ///
    /// On return, the slot is empty and write-locked.
    fn find_victim(&self) -> (usize, RwLockWriteGuard<SlotInner>) {
-        let iter_limit = self.slots.len() * 10;
+        let iter_limit = self.slots.len() * 2;
        let mut iters = 0;
        loop {
-            iters += 1;
            let slot_idx = self.next_evict_slot.fetch_add(1, Ordering::Relaxed) % self.slots.len();

            let slot = &self.slots[slot_idx];

-            if slot.dec_usage_count() == 0 {
-                let mut inner = match slot.inner.try_write() {
-                    Ok(inner) => inner,
-                    Err(TryLockError::Poisoned(err)) => {
-                        panic!("buffer lock was poisoned: {:?}", err)
-                    }
-                    Err(TryLockError::WouldBlock) => {
-                        // If we have looped through the whole buffer pool 10 times
-                        // and still haven't found a victim buffer, something's wrong.
-                        // Maybe all the buffers were in locked. That could happen in
-                        // theory, if you have more threads holding buffers locked than
-                        // there are buffers in the pool. In practice, with a reasonably
-                        // large buffer pool it really shouldn't happen.
-                        if iters > iter_limit {
-                            panic!("could not find a victim buffer to evict");
-                        }
-                        continue;
-                    }
-                };
+            if slot.dec_usage_count() == 0 || iters >= iter_limit {
+                let mut inner = slot.inner.write().unwrap();
+
                if let Some(old_key) = &inner.key {
                    if inner.dirty {
                        if let Err(err) = Self::writeback(old_key, inner.buf) {
@@ -807,6 +717,8 @@ impl PageCache {
                }
                return (slot_idx, inner);
            }
+
+            iters += 1;
        }
    }

@@ -822,13 +734,6 @@ impl PageCache {
            CacheKey::EphemeralPage { file_id, blkno } => {
                writeback_ephemeral_file(*file_id, *blkno, buf)
            }
-            CacheKey::ImmutableFilePage {
-                file_id: _,
-                blkno: _,
-            } => Err(std::io::Error::new(
-                std::io::ErrorKind::Other,
-                "unexpected dirty immutable page",
-            )),
        }
    }

@@ -859,7 +764,6 @@ impl PageCache {
        Self {
            materialized_page_map: Default::default(),
            ephemeral_page_map: Default::default(),
-            immutable_page_map: Default::default(),
            slots,
            next_evict_slot: AtomicUsize::new(0),
        }
--- a/pageserver/src/pgdatadir_mapping.rs
+++ b/pageserver/src/pgdatadir_mapping.rs
@@ -6,7 +6,7 @@
 //! walingest.rs handles a few things like implicit relation creation and extension.
 //! Clarify that)
 //!
-use crate::keyspace::{KeyPartitioning, KeySpace, KeySpaceAccum};
+use crate::keyspace::{KeySpace, KeySpaceAccum, TARGET_FILE_SIZE_BYTES};
 use crate::reltag::{RelTag, SlruKind};
 use crate::repository::*;
 use crate::repository::{Repository, Timeline};
@@ -18,9 +18,10 @@ use serde::{Deserialize, Serialize};
 use std::collections::{HashMap, HashSet};
 use std::ops::Range;
 use std::sync::atomic::{AtomicIsize, Ordering};
-use std::sync::{Arc, Mutex, RwLockReadGuard};
+use std::sync::{Arc, RwLockReadGuard};
 use tracing::{debug, error, trace, warn};
 use zenith_utils::bin_ser::BeSer;
+use zenith_utils::lsn::AtomicLsn;
 use zenith_utils::lsn::Lsn;

 /// Block number within a relation or SLRU. This matches PostgreSQL's BlockNumber type.
@@ -37,7 +38,7 @@ where
    pub tline: Arc<R::Timeline>,

    /// When did we last calculate the partitioning?
-    partitioning: Mutex<(KeyPartitioning, Lsn)>,
+    last_partitioning: AtomicLsn,

    /// Configuration: how often should the partitioning be recalculated.
    repartition_threshold: u64,
@@ -50,7 +51,7 @@ impl<R: Repository> DatadirTimeline<R> {
    pub fn new(tline: Arc<R::Timeline>, repartition_threshold: u64) -> Self {
        DatadirTimeline {
            tline,
-            partitioning: Mutex::new((KeyPartitioning::new(), Lsn(0))),
+            last_partitioning: AtomicLsn::new(0),
            current_logical_size: AtomicIsize::new(0),
            repartition_threshold,
        }
@@ -387,19 +388,6 @@ impl<R: Repository> DatadirTimeline<R> {

        Ok(result.to_keyspace())
    }
-
-    pub fn repartition(&self, lsn: Lsn, partition_size: u64) -> Result<(KeyPartitioning, Lsn)> {
-        let mut partitioning_guard = self.partitioning.lock().unwrap();
-        if partitioning_guard.1 == Lsn(0)
-            || lsn.0 - partitioning_guard.1 .0 > self.repartition_threshold
-        {
-            let keyspace = self.collect_keyspace(lsn)?;
-            let partitioning = keyspace.partition(partition_size);
-            *partitioning_guard = (partitioning, lsn);
-            return Ok((partitioning_guard.0.clone(), lsn));
-        }
-        Ok((partitioning_guard.0.clone(), partitioning_guard.1))
-    }
 }

 /// DatadirModification represents an operation to ingest an atomic set of
@@ -779,6 +767,7 @@ impl<'a, R: Repository> DatadirModification<'a, R> {
    pub fn commit(self) -> Result<()> {
        let writer = self.tline.tline.writer();

+        let last_partitioning = self.tline.last_partitioning.load();
        let pending_nblocks = self.pending_nblocks;

        for (key, value) in self.pending_updates {
@@ -790,6 +779,15 @@ impl<'a, R: Repository> DatadirModification<'a, R> {

        writer.finish_write(self.lsn);

+        if last_partitioning == Lsn(0)
+            || self.lsn.0 - last_partitioning.0 > self.tline.repartition_threshold
+        {
+            let keyspace = self.tline.collect_keyspace(self.lsn)?;
+            let partitioning = keyspace.partition(TARGET_FILE_SIZE_BYTES);
+            self.tline.tline.hint_partitioning(partitioning, self.lsn)?;
+            self.tline.last_partitioning.store(self.lsn);
+        }
+
        if pending_nblocks != 0 {
            self.tline.current_logical_size.fetch_add(
                pending_nblocks * pg_constants::BLCKSZ as isize,
@@ -1215,7 +1213,7 @@ pub fn create_test_timeline<R: Repository>(
    timeline_id: zenith_utils::zid::ZTimelineId,
 ) -> Result<Arc<crate::DatadirTimeline<R>>> {
    let tline = repo.create_empty_timeline(timeline_id, Lsn(8))?;
-    let tline = DatadirTimeline::new(tline, 256 * 1024);
+    let tline = DatadirTimeline::new(tline, crate::layered_repository::tests::TEST_FILE_SIZE / 10);
    let mut m = tline.begin_modification(Lsn(8));
    m.init_empty()?;
    m.commit()?;
--- a/pageserver/src/reltag.rs
+++ b/pageserver/src/reltag.rs
@@ -39,7 +39,9 @@ impl PartialOrd for RelTag {

 impl Ord for RelTag {
    fn cmp(&self, other: &Self) -> Ordering {
-        let mut cmp = self.spcnode.cmp(&other.spcnode);
+        let mut cmp;
+
+        cmp = self.spcnode.cmp(&other.spcnode);
        if cmp != Ordering::Equal {
            return cmp;
        }
--- a/pageserver/src/remote_storage.rs
+++ b/pageserver/src/remote_storage.rs
@@ -5,7 +5,7 @@
 //! There are a few components the storage machinery consists of:
 //! * [`RemoteStorage`] trait a CRUD-like generic abstraction to use for adapting external storages with a few implementations:
 //!     * [`local_fs`] allows to use local file system as an external storage
-//!     * [`s3_bucket`] uses AWS S3 bucket as an external storage
+//!     * [`rust_s3`] uses AWS S3 bucket as an external storage
 //!
 //! * synchronization logic at [`storage_sync`] module that keeps pageserver state (both runtime one and the workdir files) and storage state in sync.
 //! Synchronization internals are split into submodules
@@ -82,7 +82,7 @@
 //! The sync queue processing also happens in batches, so the sync tasks can wait in the queue for some time.

 mod local_fs;
-mod s3_bucket;
+mod rust_s3;
 mod storage_sync;

 use std::{
@@ -98,7 +98,7 @@ use zenith_utils::zid::{ZTenantId, ZTenantTimelineId, ZTimelineId};

 pub use self::storage_sync::index::{RemoteIndex, TimelineIndexEntry};
 pub use self::storage_sync::{schedule_timeline_checkpoint_upload, schedule_timeline_download};
-use self::{local_fs::LocalFs, s3_bucket::S3Bucket};
+use self::{local_fs::LocalFs, rust_s3::S3};
 use crate::layered_repository::ephemeral_file::is_ephemeral_file;
 use crate::{
    config::{PageServerConf, RemoteStorageKind},
@@ -151,7 +151,7 @@ pub fn start_local_timeline_sync(
                storage_sync::spawn_storage_sync_thread(
                    config,
                    local_timeline_files,
-                    S3Bucket::new(s3_config, &config.workdir)?,
+                    S3::new(s3_config, &config.workdir)?,
                    storage_config.max_concurrent_sync,
                    storage_config.max_sync_errors,
                )
@@ -325,35 +325,27 @@ trait RemoteStorage: Send + Sync {
        &self,
        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
        to: &Self::StoragePath,
-        metadata: Option<StorageMetadata>,
    ) -> anyhow::Result<()>;

    /// Streams the remote storage entry contents into the buffered writer given, returns the filled writer.
-    /// Returns the metadata, if any was stored with the file previously.
    async fn download(
        &self,
        from: &Self::StoragePath,
        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>>;
+    ) -> anyhow::Result<()>;

    /// Streams a given byte range of the remote storage entry contents into the buffered writer given, returns the filled writer.
-    /// Returns the metadata, if any was stored with the file previously.
    async fn download_range(
        &self,
        from: &Self::StoragePath,
        start_inclusive: u64,
        end_exclusive: Option<u64>,
        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>>;
+    ) -> anyhow::Result<()>;

    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()>;
 }

-/// Extra set of key-value pairs that contain arbitrary metadata about the storage entry.
-/// Immutable, cannot be changed once the file is created.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct StorageMetadata(HashMap<String, String>);
-
 fn strip_path_prefix<'a>(prefix: &'a Path, path: &'a Path) -> anyhow::Result<&'a Path> {
    if prefix == path {
        anyhow::bail!(
--- a/pageserver/src/remote_storage/README.md
+++ b/pageserver/src/remote_storage/README.md
@@ -46,6 +46,18 @@ This could be avoided by a background thread/future storing the serialized index

 No file checksum assertion is done currently, but should be (AWS S3 returns file checksums during the `list` operation)

+* sad rust-s3 api
+
+rust-s3 is not very pleasant to use:
+1. it returns `anyhow::Result` and it's hard to distinguish "missing file" cases from "no connection" one, for instance
+2. at least one function it its API that we need (`get_object_stream`) has `async` keyword and blocks (!), see details [here](https://github.com/zenithdb/zenith/pull/752#discussion_r728373091)
+3. it's a prerelease library with unclear maintenance status
+4. noisy on debug level
+
+But it's already used in the project, so for now it's reused to avoid bloating the dependency tree.
+Based on previous evaluation, even `rusoto-s3` could be a better choice over this library, but needs further benchmarking.
+
+
 * gc is ignored

 So far, we don't adjust the remote storage based on GC thread loop results, only checkpointer loop affects the remote storage.
--- a/pageserver/src/remote_storage/local_fs.rs
+++ b/pageserver/src/remote_storage/local_fs.rs
@@ -5,6 +5,7 @@
 //! volume is mounted to the local FS.

 use std::{
+    ffi::OsString,
    future::Future,
    path::{Path, PathBuf},
    pin::Pin,
@@ -17,7 +18,7 @@ use tokio::{
 };
 use tracing::*;

-use super::{strip_path_prefix, RemoteStorage, StorageMetadata};
+use super::{strip_path_prefix, RemoteStorage};

 pub struct LocalFs {
    pageserver_workdir: &'static Path,
@@ -53,32 +54,6 @@ impl LocalFs {
            )
        }
    }
-
-    async fn read_storage_metadata(
-        &self,
-        file_path: &Path,
-    ) -> anyhow::Result<Option<StorageMetadata>> {
-        let metadata_path = storage_metadata_path(file_path);
-        if metadata_path.exists() && metadata_path.is_file() {
-            let metadata_string = fs::read_to_string(&metadata_path).await.with_context(|| {
-                format!(
-                    "Failed to read metadata from the local storage at '{}'",
-                    metadata_path.display()
-                )
-            })?;
-
-            serde_json::from_str(&metadata_string)
-                .with_context(|| {
-                    format!(
-                        "Failed to deserialize metadata from the local storage at '{}'",
-                        metadata_path.display()
-                    )
-                })
-                .map(|metadata| Some(StorageMetadata(metadata)))
-        } else {
-            Ok(None)
-        }
-    }
 }

 #[async_trait::async_trait]
@@ -106,14 +81,19 @@ impl RemoteStorage for LocalFs {
        &self,
        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
        to: &Self::StoragePath,
-        metadata: Option<StorageMetadata>,
    ) -> anyhow::Result<()> {
        let target_file_path = self.resolve_in_storage(to)?;
        create_target_directory(&target_file_path).await?;
        // We need this dance with sort of durable rename (without fsyncs)
        // to prevent partial uploads. This was really hit when pageserver shutdown
        // cancelled the upload and partial file was left on the fs
-        let temp_file_path = path_with_suffix_extension(&target_file_path, ".temp");
+        let mut temp_extension = target_file_path
+            .extension()
+            .unwrap_or_default()
+            .to_os_string();
+
+        temp_extension.push(OsString::from(".temp"));
+        let temp_file_path = target_file_path.with_extension(temp_extension);
        let mut destination = io::BufWriter::new(
            fs::OpenOptions::new()
                .write(true)
@@ -152,23 +132,6 @@ impl RemoteStorage for LocalFs {
                    target_file_path.display()
                )
            })?;
-
-        if let Some(storage_metadata) = metadata {
-            let storage_metadata_path = storage_metadata_path(&target_file_path);
-            fs::write(
-                &storage_metadata_path,
-                serde_json::to_string(&storage_metadata.0)
-                    .context("Failed to serialize storage metadata as json")?,
-            )
-            .await
-            .with_context(|| {
-                format!(
-                    "Failed to write metadata to the local storage at '{}'",
-                    storage_metadata_path.display()
-                )
-            })?;
-        }
-
        Ok(())
    }

@@ -176,7 +139,7 @@ impl RemoteStorage for LocalFs {
        &self,
        from: &Self::StoragePath,
        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
        let file_path = self.resolve_in_storage(from)?;

        if file_path.exists() && file_path.is_file() {
@@ -199,8 +162,7 @@ impl RemoteStorage for LocalFs {
                )
            })?;
            source.flush().await?;
-
-            self.read_storage_metadata(&file_path).await
+            Ok(())
        } else {
            bail!(
                "File '{}' either does not exist or is not a file",
@@ -215,7 +177,7 @@ impl RemoteStorage for LocalFs {
        start_inclusive: u64,
        end_exclusive: Option<u64>,
        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
        if let Some(end_exclusive) = end_exclusive {
            ensure!(
                end_exclusive > start_inclusive,
@@ -224,7 +186,7 @@ impl RemoteStorage for LocalFs {
                end_exclusive
            );
            if start_inclusive == end_exclusive.saturating_sub(1) {
-                return Ok(None);
+                return Ok(());
            }
        }
        let file_path = self.resolve_in_storage(from)?;
@@ -258,8 +220,7 @@ impl RemoteStorage for LocalFs {
                    file_path.display()
                )
            })?;
-
-            self.read_storage_metadata(&file_path).await
+            Ok(())
        } else {
            bail!(
                "File '{}' either does not exist or is not a file",
@@ -281,17 +242,6 @@ impl RemoteStorage for LocalFs {
    }
 }

-fn path_with_suffix_extension(original_path: &Path, suffix: &str) -> PathBuf {
-    let mut extension_with_suffix = original_path.extension().unwrap_or_default().to_os_string();
-    extension_with_suffix.push(suffix);
-
-    original_path.with_extension(extension_with_suffix)
-}
-
-fn storage_metadata_path(original_path: &Path) -> PathBuf {
-    path_with_suffix_extension(original_path, ".metadata")
-}
-
 fn get_all_files<'a, P>(
    directory_path: P,
 ) -> Pin<Box<dyn Future<Output = anyhow::Result<Vec<PathBuf>>> + Send + Sync + 'a>>
@@ -501,7 +451,7 @@ mod fs_tests {
    use super::*;
    use crate::repository::repo_harness::{RepoHarness, TIMELINE_ID};

-    use std::{collections::HashMap, io::Write};
+    use std::io::Write;
    use tempfile::tempdir;

    #[tokio::test]
@@ -515,7 +465,7 @@ mod fs_tests {
        )
        .await?;
        let target_path = PathBuf::from("/").join("somewhere").join("else");
-        match storage.upload(source, &target_path, None).await {
+        match storage.upload(source, &target_path).await {
            Ok(()) => panic!("Should not allow storing files with wrong target path"),
            Err(e) => {
                let message = format!("{:?}", e);
@@ -525,14 +475,14 @@ mod fs_tests {
        }
        assert!(storage.list().await?.is_empty());

-        let target_path_1 = upload_dummy_file(&repo_harness, &storage, "upload_1", None).await?;
+        let target_path_1 = upload_dummy_file(&repo_harness, &storage, "upload_1").await?;
        assert_eq!(
            storage.list().await?,
            vec![target_path_1.clone()],
            "Should list a single file after first upload"
        );

-        let target_path_2 = upload_dummy_file(&repo_harness, &storage, "upload_2", None).await?;
+        let target_path_2 = upload_dummy_file(&repo_harness, &storage, "upload_2").await?;
        assert_eq!(
            list_files_sorted(&storage).await?,
            vec![target_path_1.clone(), target_path_2.clone()],
@@ -553,16 +503,12 @@ mod fs_tests {
        let repo_harness = RepoHarness::create("download_file")?;
        let storage = create_storage()?;
        let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;

        let mut content_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage.download(&upload_target, &mut content_bytes).await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
+        storage.download(&upload_target, &mut content_bytes).await?;
        content_bytes.flush().await?;
+
        let contents = String::from_utf8(content_bytes.into_inner().into_inner())?;
        assert_eq!(
            dummy_contents(upload_name),
@@ -587,16 +533,12 @@ mod fs_tests {
        let repo_harness = RepoHarness::create("download_file_range_positive")?;
        let storage = create_storage()?;
        let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;

        let mut full_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
+        storage
            .download_range(&upload_target, 0, None, &mut full_range_bytes)
            .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
        full_range_bytes.flush().await?;
        assert_eq!(
            dummy_contents(upload_name),
@@ -606,7 +548,7 @@ mod fs_tests {

        let mut zero_range_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
        let same_byte = 1_000_000_000;
-        let metadata = storage
+        storage
            .download_range(
                &upload_target,
                same_byte,
@@ -614,10 +556,6 @@ mod fs_tests {
                &mut zero_range_bytes,
            )
            .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
        zero_range_bytes.flush().await?;
        assert!(
            zero_range_bytes.into_inner().into_inner().is_empty(),
@@ -628,7 +566,7 @@ mod fs_tests {
        let (first_part_local, second_part_local) = uploaded_bytes.split_at(3);

        let mut first_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
+        storage
            .download_range(
                &upload_target,
                0,
@@ -636,11 +574,6 @@ mod fs_tests {
                &mut first_part_remote,
            )
            .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
        first_part_remote.flush().await?;
        let first_part_remote = first_part_remote.into_inner().into_inner();
        assert_eq!(
@@ -650,7 +583,7 @@ mod fs_tests {
        );

        let mut second_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let metadata = storage
+        storage
            .download_range(
                &upload_target,
                first_part_local.len() as u64,
@@ -658,11 +591,6 @@ mod fs_tests {
                &mut second_part_remote,
            )
            .await?;
-        assert!(
-            metadata.is_none(),
-            "No metadata should be returned for no metadata upload"
-        );
-
        second_part_remote.flush().await?;
        let second_part_remote = second_part_remote.into_inner().into_inner();
        assert_eq!(
@@ -679,7 +607,7 @@ mod fs_tests {
        let repo_harness = RepoHarness::create("download_file_range_negative")?;
        let storage = create_storage()?;
        let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;

        let start = 10000;
        let end = 234;
@@ -717,7 +645,7 @@ mod fs_tests {
        let repo_harness = RepoHarness::create("delete_file")?;
        let storage = create_storage()?;
        let upload_name = "upload_1";
-        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name, None).await?;
+        let upload_target = upload_dummy_file(&repo_harness, &storage, upload_name).await?;

        storage.delete(&upload_target).await?;
        assert!(storage.list().await?.is_empty());
@@ -733,69 +661,10 @@ mod fs_tests {
        Ok(())
    }

-    #[tokio::test]
-    async fn file_with_metadata() -> anyhow::Result<()> {
-        let repo_harness = RepoHarness::create("download_file")?;
-        let storage = create_storage()?;
-        let upload_name = "upload_1";
-        let metadata = StorageMetadata(HashMap::from([
-            ("one".to_string(), "1".to_string()),
-            ("two".to_string(), "2".to_string()),
-        ]));
-        let upload_target =
-            upload_dummy_file(&repo_harness, &storage, upload_name, Some(metadata.clone())).await?;
-
-        let mut content_bytes = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let full_download_metadata = storage.download(&upload_target, &mut content_bytes).await?;
-
-        content_bytes.flush().await?;
-        let contents = String::from_utf8(content_bytes.into_inner().into_inner())?;
-        assert_eq!(
-            dummy_contents(upload_name),
-            contents,
-            "We should upload and download the same contents"
-        );
-
-        assert_eq!(
-            full_download_metadata.as_ref(),
-            Some(&metadata),
-            "We should get the same metadata back for full download"
-        );
-
-        let uploaded_bytes = dummy_contents(upload_name).into_bytes();
-        let (first_part_local, _) = uploaded_bytes.split_at(3);
-
-        let mut first_part_remote = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
-        let partial_download_metadata = storage
-            .download_range(
-                &upload_target,
-                0,
-                Some(first_part_local.len() as u64),
-                &mut first_part_remote,
-            )
-            .await?;
-        first_part_remote.flush().await?;
-        let first_part_remote = first_part_remote.into_inner().into_inner();
-        assert_eq!(
-            first_part_local,
-            first_part_remote.as_slice(),
-            "First part bytes should be returned when requested"
-        );
-
-        assert_eq!(
-            partial_download_metadata.as_ref(),
-            Some(&metadata),
-            "We should get the same metadata back for partial download"
-        );
-
-        Ok(())
-    }
-
    async fn upload_dummy_file(
        harness: &RepoHarness<'_>,
        storage: &LocalFs,
        name: &str,
-        metadata: Option<StorageMetadata>,
    ) -> anyhow::Result<PathBuf> {
        let timeline_path = harness.timeline_path(&TIMELINE_ID);
        let relative_timeline_path = timeline_path.strip_prefix(&harness.conf.workdir)?;
@@ -808,7 +677,6 @@ mod fs_tests {
                )
                .await?,
                &storage_path,
-                metadata,
            )
            .await?;
        Ok(storage_path)
--- a/pageserver/src/remote_storage/s3_bucket.rs
+++ b/pageserver/src/remote_storage/s3_bucket.rs
@@ -1,4 +1,4 @@
-//! AWS S3 storage wrapper around `rusoto` library.
+//! AWS S3 storage wrapper around `rust_s3` library.
 //!
 //! Respects `prefix_in_bucket` property from [`S3Config`],
 //! allowing multiple pageservers to independently work with the same S3 bucket, if
@@ -7,25 +7,15 @@
 use std::path::{Path, PathBuf};

 use anyhow::Context;
-use rusoto_core::{
-    credential::{InstanceMetadataProvider, StaticProvider},
-    HttpClient, Region,
-};
-use rusoto_s3::{
-    DeleteObjectRequest, GetObjectRequest, ListObjectsV2Request, PutObjectRequest, S3Client,
-    StreamingBody, S3,
-};
-use tokio::io;
-use tokio_util::io::ReaderStream;
-use tracing::{debug, trace};
+use s3::{bucket::Bucket, creds::Credentials, region::Region};
+use tokio::io::{self, AsyncWriteExt};
+use tracing::debug;

 use crate::{
    config::S3Config,
    remote_storage::{strip_path_prefix, RemoteStorage},
 };

-use super::StorageMetadata;
-
 const S3_FILE_SEPARATOR: char = '/';

 #[derive(Debug, Eq, PartialEq)]
@@ -60,50 +50,38 @@ impl S3ObjectKey {
 }

 /// AWS S3 storage.
-pub struct S3Bucket {
+pub struct S3 {
    pageserver_workdir: &'static Path,
-    client: S3Client,
-    bucket_name: String,
+    bucket: Bucket,
    prefix_in_bucket: Option<String>,
 }

-impl S3Bucket {
-    /// Creates the S3 storage, errors if incorrect AWS S3 configuration provided.
+impl S3 {
+    /// Creates the storage, errors if incorrect AWS S3 configuration provided.
    pub fn new(aws_config: &S3Config, pageserver_workdir: &'static Path) -> anyhow::Result<Self> {
-        // TODO kb check this
-        // Keeping a single client may cause issues due to timeouts.
-        // https://github.com/rusoto/rusoto/issues/1686
-
        debug!(
-            "Creating s3 remote storage for S3 bucket {}",
+            "Creating s3 remote storage around bucket {}",
            aws_config.bucket_name
        );
        let region = match aws_config.endpoint.clone() {
-            Some(custom_endpoint) => Region::Custom {
-                name: aws_config.bucket_region.clone(),
-                endpoint: custom_endpoint,
+            Some(endpoint) => Region::Custom {
+                endpoint,
+                region: aws_config.bucket_region.clone(),
            },
            None => aws_config
                .bucket_region
                .parse::<Region>()
                .context("Failed to parse the s3 region from config")?,
        };
-        let request_dispatcher = HttpClient::new().context("Failed to create S3 http client")?;
-        let client = if aws_config.access_key_id.is_none() && aws_config.secret_access_key.is_none()
-        {
-            trace!("Using IAM-based AWS access");
-            S3Client::new_with(request_dispatcher, InstanceMetadataProvider::new(), region)
-        } else {
-            trace!("Using credentials-based AWS access");
-            S3Client::new_with(
-                request_dispatcher,
-                StaticProvider::new_minimal(
-                    aws_config.access_key_id.clone().unwrap_or_default(),
-                    aws_config.secret_access_key.clone().unwrap_or_default(),
-                ),
-                region,
-            )
-        };
+
+        let credentials = Credentials::new(
+            aws_config.access_key_id.as_deref(),
+            aws_config.secret_access_key.as_deref(),
+            None,
+            None,
+            None,
+        )
+        .context("Failed to create the s3 credentials")?;

        let prefix_in_bucket = aws_config.prefix_in_bucket.as_deref().map(|prefix| {
            let mut prefix = prefix;
@@ -119,16 +97,20 @@ impl S3Bucket {
        });

        Ok(Self {
-            client,
+            bucket: Bucket::new_with_path_style(
+                aws_config.bucket_name.as_str(),
+                region,
+                credentials,
+            )
+            .context("Failed to create the s3 bucket")?,
            pageserver_workdir,
-            bucket_name: aws_config.bucket_name.clone(),
            prefix_in_bucket,
        })
    }
 }

 #[async_trait::async_trait]
-impl RemoteStorage for S3Bucket {
+impl RemoteStorage for S3 {
    type StoragePath = S3ObjectKey;

    fn storage_path(&self, local_path: &Path) -> anyhow::Result<Self::StoragePath> {
@@ -147,74 +129,74 @@ impl RemoteStorage for S3Bucket {
    }

    async fn list(&self) -> anyhow::Result<Vec<Self::StoragePath>> {
-        let mut document_keys = Vec::new();
+        let list_response = self
+            .bucket
+            .list(self.prefix_in_bucket.clone().unwrap_or_default(), None)
+            .await
+            .context("Failed to list s3 objects")?;

-        let mut continuation_token = None;
-        loop {
-            let fetch_response = self
-                .client
-                .list_objects_v2(ListObjectsV2Request {
-                    bucket: self.bucket_name.clone(),
-                    prefix: self.prefix_in_bucket.clone(),
-                    continuation_token,
-                    ..ListObjectsV2Request::default()
-                })
-                .await?;
-            document_keys.extend(
-                fetch_response
-                    .contents
-                    .unwrap_or_default()
-                    .into_iter()
-                    .filter_map(|o| Some(S3ObjectKey(o.key?))),
-            );
-
-            match fetch_response.continuation_token {
-                Some(new_token) => continuation_token = Some(new_token),
-                None => break,
-            }
-        }
-
-        Ok(document_keys)
+        Ok(list_response
+            .into_iter()
+            .flat_map(|response| response.contents)
+            .map(|s3_object| S3ObjectKey(s3_object.key))
+            .collect())
    }

    async fn upload(
        &self,
-        from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
+        mut from: impl io::AsyncRead + Unpin + Send + Sync + 'static,
        to: &Self::StoragePath,
-        metadata: Option<StorageMetadata>,
    ) -> anyhow::Result<()> {
-        self.client
-            .put_object(PutObjectRequest {
-                body: Some(StreamingBody::new(ReaderStream::new(from))),
-                bucket: self.bucket_name.clone(),
-                key: to.key().to_owned(),
-                metadata: metadata.map(|m| m.0),
-                ..PutObjectRequest::default()
-            })
-            .await?;
-        Ok(())
+        let mut upload_contents = io::BufWriter::new(std::io::Cursor::new(Vec::new()));
+        io::copy(&mut from, &mut upload_contents)
+            .await
+            .context("Failed to read the upload contents")?;
+        upload_contents
+            .flush()
+            .await
+            .context("Failed to read the upload contents")?;
+        let upload_contents = upload_contents.into_inner().into_inner();
+
+        let (_, code) = self
+            .bucket
+            .put_object(to.key(), &upload_contents)
+            .await
+            .with_context(|| format!("Failed to create s3 object with key {}", to.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during creating object with key '{}', code: {}",
+                to.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
    }

    async fn download(
        &self,
        from: &Self::StoragePath,
        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
-        let object_output = self
-            .client
-            .get_object(GetObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: from.key().to_owned(),
-                ..GetObjectRequest::default()
-            })
-            .await?;
-
-        if let Some(body) = object_output.body {
-            let mut from = io::BufReader::new(body.into_async_read());
-            io::copy(&mut from, to).await?;
+    ) -> anyhow::Result<()> {
+        let (data, code) = self
+            .bucket
+            .get_object(from.key())
+            .await
+            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
+        if code != 200 {
+            Err(anyhow::format_err!(
+                "Received non-200 exit code during downloading object, code: {}",
+                code
+            ))
+        } else {
+            // we don't have to write vector into the destination this way, `to_write_all` would be enough.
+            // but we want to prepare for migration on `rusoto`, that has a streaming HTTP body instead here, with
+            // which it makes more sense to use `io::copy`.
+            io::copy(&mut data.as_slice(), to)
+                .await
+                .context("Failed to write downloaded data into the destination buffer")?;
+            Ok(())
        }
-
-        Ok(object_output.metadata.map(StorageMetadata))
    }

    async fn download_range(
@@ -223,41 +205,44 @@ impl RemoteStorage for S3Bucket {
        start_inclusive: u64,
        end_exclusive: Option<u64>,
        to: &mut (impl io::AsyncWrite + Unpin + Send + Sync),
-    ) -> anyhow::Result<Option<StorageMetadata>> {
+    ) -> anyhow::Result<()> {
        // S3 accepts ranges as https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35
        // and needs both ends to be exclusive
        let end_inclusive = end_exclusive.map(|end| end.saturating_sub(1));
-        let range = Some(match end_inclusive {
-            Some(end_inclusive) => format!("bytes={}-{}", start_inclusive, end_inclusive),
-            None => format!("bytes={}-", start_inclusive),
-        });
-        let object_output = self
-            .client
-            .get_object(GetObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: from.key().to_owned(),
-                range,
-                ..GetObjectRequest::default()
-            })
-            .await?;
-
-        if let Some(body) = object_output.body {
-            let mut from = io::BufReader::new(body.into_async_read());
-            io::copy(&mut from, to).await?;
+        let (data, code) = self
+            .bucket
+            .get_object_range(from.key(), start_inclusive, end_inclusive)
+            .await
+            .with_context(|| format!("Failed to download s3 object with key {}", from.key()))?;
+        if code != 206 {
+            Err(anyhow::format_err!(
+                "Received non-206 exit code during downloading object range, code: {}",
+                code
+            ))
+        } else {
+            // see `download` function above for the comment on why `Vec<u8>` buffer is copied this way
+            io::copy(&mut data.as_slice(), to)
+                .await
+                .context("Failed to write downloaded range into the destination buffer")?;
+            Ok(())
        }
-
-        Ok(object_output.metadata.map(StorageMetadata))
    }

    async fn delete(&self, path: &Self::StoragePath) -> anyhow::Result<()> {
-        self.client
-            .delete_object(DeleteObjectRequest {
-                bucket: self.bucket_name.clone(),
-                key: path.key().to_owned(),
-                ..DeleteObjectRequest::default()
-            })
-            .await?;
-        Ok(())
+        let (_, code) = self
+            .bucket
+            .delete_object(path.key())
+            .await
+            .with_context(|| format!("Failed to delete s3 object with key {}", path.key()))?;
+        if code != 204 {
+            Err(anyhow::format_err!(
+                "Received non-204 exit code during deleting object with key '{}', code: {}",
+                path.key(),
+                code
+            ))
+        } else {
+            Ok(())
+        }
    }
 }

@@ -329,7 +314,7 @@ mod tests {
    #[test]
    fn storage_path_negatives() -> anyhow::Result<()> {
        #[track_caller]
-        fn storage_path_error(storage: &S3Bucket, mismatching_path: &Path) -> String {
+        fn storage_path_error(storage: &S3, mismatching_path: &Path) -> String {
            match storage.storage_path(mismatching_path) {
                Ok(wrong_key) => panic!(
                    "Expected path '{}' to error, but got S3 key: {:?}",
@@ -427,11 +412,15 @@ mod tests {
        Ok(())
    }

-    fn dummy_storage(pageserver_workdir: &'static Path) -> S3Bucket {
-        S3Bucket {
+    fn dummy_storage(pageserver_workdir: &'static Path) -> S3 {
+        S3 {
            pageserver_workdir,
-            client: S3Client::new("us-east-1".parse().unwrap()),
-            bucket_name: "dummy-bucket".to_string(),
+            bucket: Bucket::new(
+                "dummy-bucket",
+                "us-east-1".parse().unwrap(),
+                Credentials::anonymous().unwrap(),
+            )
+            .unwrap(),
            prefix_in_bucket: Some("dummy_prefix/".to_string()),
        }
    }
--- a/pageserver/src/remote_storage/storage_sync/compression.rs
+++ b/pageserver/src/remote_storage/storage_sync/compression.rs
@@ -201,7 +201,8 @@ pub async fn read_archive_header<A: io::AsyncRead + Send + Sync + Unpin>(
        .await
        .context("Failed to decompress a header from the archive")?;

-    ArchiveHeader::des(&header_bytes).context("Failed to deserialize a header from the archive")
+    Ok(ArchiveHeader::des(&header_bytes)
+        .context("Failed to deserialize a header from the archive")?)
 }

 /// Reads the archive metadata out of the archive name:
--- a/pageserver/src/remote_storage/storage_sync/download.rs
+++ b/pageserver/src/remote_storage/storage_sync/download.rs
@@ -225,8 +225,8 @@ async fn read_local_metadata(
    let local_metadata_bytes = fs::read(&local_metadata_path)
        .await
        .context("Failed to read local metadata file bytes")?;
-    TimelineMetadata::from_bytes(&local_metadata_bytes)
-        .context("Failed to read local metadata files bytes")
+    Ok(TimelineMetadata::from_bytes(&local_metadata_bytes)
+        .context("Failed to read local metadata files bytes")?)
 }

 #[cfg(test)]
--- a/pageserver/src/remote_storage/storage_sync/upload.rs
+++ b/pageserver/src/remote_storage/storage_sync/upload.rs
@@ -201,7 +201,6 @@ async fn try_upload_checkpoint<
                .upload(
                    archive_streamer,
                    &remote_storage.storage_path(&timeline_dir.join(&archive_name))?,
-                    None,
                )
                .await
        },
--- a/pageserver/src/repository.rs
+++ b/pageserver/src/repository.rs
@@ -1,9 +1,9 @@
+use crate::keyspace::KeyPartitioning;
 use crate::layered_repository::metadata::TimelineMetadata;
 use crate::remote_storage::RemoteIndex;
 use crate::walrecord::ZenithWalRecord;
 use crate::CheckpointConfig;
 use anyhow::{bail, Result};
-use byteorder::{ByteOrder, BE};
 use bytes::Bytes;
 use serde::{Deserialize, Serialize};
 use std::fmt;
@@ -28,8 +28,6 @@ pub struct Key {
    pub field6: u32,
 }

-pub const KEY_SIZE: usize = 18;
-
 impl Key {
    pub fn next(&self) -> Key {
        self.add(1)
@@ -64,7 +62,7 @@ impl Key {
        key
    }

-    pub fn from_slice(b: &[u8]) -> Self {
+    pub fn from_array(b: [u8; 18]) -> Self {
        Key {
            field1: b[0],
            field2: u32::from_be_bytes(b[1..5].try_into().unwrap()),
@@ -74,15 +72,6 @@ impl Key {
            field6: u32::from_be_bytes(b[14..18].try_into().unwrap()),
        }
    }
-
-    pub fn write_to_byte_slice(&self, buf: &mut [u8]) {
-        buf[0] = self.field1;
-        BE::write_u32(&mut buf[1..5], self.field2);
-        BE::write_u32(&mut buf[5..9], self.field3);
-        BE::write_u32(&mut buf[9..13], self.field4);
-        buf[13] = self.field5;
-        BE::write_u32(&mut buf[14..18], self.field6);
-    }
 }

 pub fn key_range_size(key_range: &Range<Key>) -> u32 {
@@ -383,6 +372,19 @@ pub trait Timeline: Send + Sync {
    /// know anything about them here in the repository.
    fn checkpoint(&self, cconf: CheckpointConfig) -> Result<()>;

+    ///
+    /// Tell the implementation how the keyspace should be partitioned.
+    ///
+    /// FIXME: This is quite a hack. The code in pgdatadir_mapping.rs knows
+    /// which keys exist and what is the logical grouping of them. That's why
+    /// the code there (and in keyspace.rs) decides the partitioning, not the
+    /// layered_repository.rs implementation. That's a layering violation:
+    /// the Repository implementation ought to be responsible for the physical
+    /// layout, but currently it's more convenient to do it in pgdatadir_mapping.rs
+    /// rather than in layered_repository.rs.
+    ///
+    fn hint_partitioning(&self, partitioning: KeyPartitioning, lsn: Lsn) -> Result<()>;
+
    ///
    /// Check that it is valid to request operations with that lsn.
    fn check_lsn_is_in_scope(
@@ -581,7 +583,7 @@ mod tests {
    use lazy_static::lazy_static;

    lazy_static! {
-        static ref TEST_KEY: Key = Key::from_slice(&hex!("112222222233333333444444445500000001"));
+        static ref TEST_KEY: Key = Key::from_array(hex!("112222222233333333444444445500000001"));
    }

    #[test]
--- a/pageserver/src/virtual_file.rs
+++ b/pageserver/src/virtual_file.rs
@@ -65,7 +65,6 @@ lazy_static! {
 /// currently open, the 'handle' can still point to the slot where it was last kept. The
 /// 'tag' field is used to detect whether the handle still is valid or not.
 ///
-#[derive(Debug)]
 pub struct VirtualFile {
    /// Lazy handle to the global file descriptor cache. The slot that this points to
    /// might contain our File, or it may be empty, or it may contain a File that
@@ -89,7 +88,7 @@ pub struct VirtualFile {
    timelineid: String,
 }

-#[derive(Debug, PartialEq, Clone, Copy)]
+#[derive(PartialEq, Clone, Copy)]
 struct SlotHandle {
    /// Index into OPEN_FILES.slots
    index: usize,
--- a/pageserver/src/wal_metadata.rs
+++ b/pageserver/src/wal_metadata.rs
@@ -1,78 +0,0 @@
-use anyhow::Result;
-use once_cell::sync::OnceCell;
-use zenith_utils::lsn::Lsn;
-use std::fs::File;
-use tokio::io::{AsyncReadExt, AsyncWriteExt};
-use bytes::{BufMut, Bytes, BytesMut};
-use serde::{Deserialize, Serialize};
-
-use crate::{config::PageServerConf, repository::Key, walrecord::DecodedBkpBlock};
-
-pub static WAL_METADATA_FILE: OnceCell<File> = OnceCell::new();
-
-#[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug, Serialize, Deserialize)]
-pub struct Page {
-    spcnode: u32,
-    dbnode: u32,
-    relnode: u32,
-    forknum: u8,
-    blkno: u32,
-}
-
-impl Page {
-    pub async fn read<Reader>(buf: &mut Reader) -> Result<Page>
-    where
-        Reader: tokio::io::AsyncRead + Unpin,
-    {
-        let spcnode = buf.read_u32().await?;
-        let dbnode = buf.read_u32().await?;
-        let relnode = buf.read_u32().await?;
-        let forknum = buf.read_u8().await?;
-        let blkno = buf.read_u32().await?;
-        Ok(Page { spcnode, dbnode, relnode, forknum, blkno })
-    }
-
-    pub async fn write(&self, buf: &mut BytesMut) -> Result<()> {
-        buf.put_u32(self.spcnode);
-        buf.put_u32(self.dbnode);
-        buf.put_u32(self.relnode);
-        buf.put_u8(self.forknum);
-        buf.put_u32(self.blkno);
-        Ok(())
-    }
-}
-
-impl From<&DecodedBkpBlock> for Page {
-    fn from(blk: &DecodedBkpBlock) -> Self {
-        Page {
-            spcnode: blk.rnode_spcnode,
-            dbnode: blk.rnode_dbnode,
-            relnode: blk.rnode_relnode,
-            forknum: blk.forknum,
-            blkno: blk.blkno,
-        }
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-pub struct WalEntryMetadata {
-    pub lsn: Lsn,
-    pub size: usize,
-    pub affected_pages: Vec<Page>,
-}
-
-pub fn init(conf: &'static PageServerConf) -> Result<()> {
-    let wal_metadata_file_dir = conf.workdir.join("wal_metadata.log");
-    WAL_METADATA_FILE.set(File::create(wal_metadata_file_dir)?)
-        .expect("wal_metadata file is already created");
-    Ok(())
-}
-
-pub fn write(wal_meta: WalEntryMetadata) -> Result<()> {
-    if let Some(mut file) = WAL_METADATA_FILE.get() {
-        let mut line = serde_json::to_string(&wal_meta)?;
-        line.push('\n');
-        std::io::prelude::Write::write_all(&mut file, line.as_bytes())?;
-    }
-    Ok(())
-}
--- a/pageserver/src/walingest.rs
+++ b/pageserver/src/walingest.rs
@@ -82,7 +82,6 @@ impl<'a, R: Repository> WalIngest<'a, R> {
    ) -> Result<()> {
        let mut modification = timeline.begin_modification(lsn);

-        let recdata_len = recdata.len();
        let mut decoded = decode_wal_record(recdata);
        let mut buf = decoded.record.clone();
        buf.advance(decoded.main_data_offset);
@@ -250,13 +249,6 @@ impl<'a, R: Repository> WalIngest<'a, R> {
            self.ingest_decoded_block(&mut modification, lsn, &decoded, blk)?;
        }

-        // Emit wal entry metadata, if configured to do so
-        crate::wal_metadata::write(crate::wal_metadata::WalEntryMetadata {
-            lsn,
-            size: recdata_len,
-            affected_pages: decoded.blocks.iter().map(|blk| blk.into()).collect()
-        });
-
        // If checkpoint data was updated, store the new version in the repository
        if self.checkpoint_modified {
            let new_checkpoint_bytes = self.checkpoint.encode();
--- a/postgres_ffi/src/xlog_utils.rs
+++ b/postgres_ffi/src/xlog_utils.rs
@@ -495,13 +495,7 @@ mod tests {
            .env("DYLD_LIBRARY_PATH", &lib_path)
            .output()
            .unwrap();
-        assert!(
-            initdb_output.status.success(),
-            "initdb failed. Status: '{}', stdout: '{}', stderr: '{}'",
-            initdb_output.status,
-            String::from_utf8_lossy(&initdb_output.stdout),
-            String::from_utf8_lossy(&initdb_output.stderr),
-        );
+        assert!(initdb_output.status.success());

        // 2. Pick WAL generated by initdb
        let wal_dir = data_dir.join("pg_wal");
--- a/proxy/src/compute.rs
+++ b/proxy/src/compute.rs
@@ -24,7 +24,7 @@ pub enum ConnectionError {
 impl UserFacingError for ConnectionError {}

 /// Compute node connection params.
-#[derive(Serialize, Deserialize, Default)]
+#[derive(Serialize, Deserialize, Debug, Default)]
 pub struct DatabaseInfo {
    pub host: String,
    pub port: u16,
@@ -33,16 +33,6 @@ pub struct DatabaseInfo {
    pub password: Option<String>,
 }

-// Manually implement debug to omit personal and sensitive info
-impl std::fmt::Debug for DatabaseInfo {
-    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
-        fmt.debug_struct("DatabaseInfo")
-            .field("host", &self.host)
-            .field("port", &self.port)
-            .finish()
-    }
-}
-
 /// PostgreSQL version as [`String`].
 pub type Version = String;

--- a/proxy/src/mgmt.rs
+++ b/proxy/src/mgmt.rs
@@ -107,7 +107,7 @@ impl postgres_backend::Handler for MgmtHandler {
 }

 fn try_process_query(pgb: &mut PostgresBackend, query_string: &str) -> anyhow::Result<()> {
-    println!("Got mgmt query [redacted]"); // Content contains password, don't print it
+    println!("Got mgmt query: '{}'", query_string);

    let resp: PsqlSessionResponse = serde_json::from_str(query_string)?;

--- a/test_runner/batch_others/init.py
+++ b/test_runner/batch_others/init.py
--- a/test_runner/batch_others/test_wal_acceptor.py
+++ b/test_runner/batch_others/test_wal_acceptor.py
@@ -12,10 +12,10 @@ from contextlib import closing
 from dataclasses import dataclass, field
 from multiprocessing import Process, Value
 from pathlib import Path
-from fixtures.zenith_fixtures import PgBin, Postgres, Safekeeper, ZenithEnv, ZenithEnvBuilder, PortDistributor, SafekeeperPort, zenith_binpath, PgProtocol
+from fixtures.zenith_fixtures import PgBin, Postgres, Safekeeper, SafekeeperHttpClient, ZenithEnv, ZenithEnvBuilder, PortDistributor, SafekeeperPort, zenith_binpath, PgProtocol
 from fixtures.utils import etcd_path, lsn_to_hex, mkdir_if_needed, lsn_from_hex
 from fixtures.log_helper import log
-from typing import List, Optional, Any
+from typing import Dict, List, Optional, Any


 # basic test, write something in setup with wal acceptors, ensure that commits
@@ -108,14 +108,14 @@ def test_many_timelines(zenith_env_builder: ZenithEnvBuilder):

            for flush_lsn, commit_lsn in zip(m.flush_lsns, m.commit_lsns):
                # Invariant. May be < when transaction is in progress.
-                assert commit_lsn <= flush_lsn, f"timeline_id={timeline_id}, timeline_detail={timeline_detail}, sk_metrics={sk_metrics}"
+                assert commit_lsn <= flush_lsn
            # We only call collect_metrics() after a transaction is confirmed by
            # the compute node, which only happens after a consensus of safekeepers
            # has confirmed the transaction. We assume majority consensus here.
            assert (2 * sum(m.last_record_lsn <= lsn
-                            for lsn in m.flush_lsns) > zenith_env_builder.num_safekeepers), f"timeline_id={timeline_id}, timeline_detail={timeline_detail}, sk_metrics={sk_metrics}"
+                            for lsn in m.flush_lsns) > zenith_env_builder.num_safekeepers)
            assert (2 * sum(m.last_record_lsn <= lsn
-                            for lsn in m.commit_lsns) > zenith_env_builder.num_safekeepers), f"timeline_id={timeline_id}, timeline_detail={timeline_detail}, sk_metrics={sk_metrics}"
+                            for lsn in m.commit_lsns) > zenith_env_builder.num_safekeepers)
            timeline_metrics.append(m)
        log.info(f"{message}: {timeline_metrics}")
        return timeline_metrics
@@ -396,20 +396,39 @@ class ProposerPostgres(PgProtocol):
        """ Path to postgresql.conf """
        return os.path.join(self.pgdata_dir, 'postgresql.conf')

+    def log_path(self) -> str:
+        """ Path to pg.log """
+        return os.path.join(self.pg_data_dir_path(), "pg.log")
+
    def create_dir_config(self, wal_acceptors: str):
        """ Create dir and config for running --sync-safekeepers """

        mkdir_if_needed(self.pg_data_dir_path())
        with open(self.config_file_path(), "w") as f:
            cfg = [
-                "synchronous_standby_names = 'walproposer'\n",
-                "shared_preload_libraries = 'zenith'\n",
+                "wal_keep_size=10TB\n",
+                "shared_preload_libraries=zenith\n",
+                "zenith.page_server_connstring=''\n",
+                "synchronous_commit=on\n",
+                "max_wal_senders=10\n",
+                "wal_log_hints=on\n",
+                "max_replication_slots=10\n",
+                "hot_standby=on\n",
+                "min_wal_size=20GB\n",
+                "max_wal_size=40GB\n",
+                "checkpoint_timeout=60min\n",
+                "log_checkpoints=on\n",
+                "max_connections=100\n",
+                "wal_sender_timeout=0\n",
+                "wal_level=replica\n",
                f"zenith.zenith_timeline = '{self.timeline_id.hex}'\n",
                f"zenith.zenith_tenant = '{self.tenant_id.hex}'\n",
                f"zenith.page_server_connstring = ''\n",
                f"wal_acceptors = '{wal_acceptors}'\n",
                f"listen_addresses = '{self.listen_addr}'\n",
                f"port = '{self.port}'\n",
+                "synchronous_standby_names = 'walproposer'\n",
+                "fsync=off\n",
            ]

            f.writelines(cfg)
@@ -441,8 +460,7 @@ class ProposerPostgres(PgProtocol):
    def start(self):
        """ Start postgres with pg_ctl """

-        log_path = os.path.join(self.pg_data_dir_path(), "pg.log")
-        args = ["pg_ctl", "-D", self.pg_data_dir_path(), "-l", log_path, "-w", "start"]
+        args = ["pg_ctl", "-D", self.pg_data_dir_path(), "-l", self.log_path(), "-w", "start"]
        self.pg_bin.run(args)

    def stop(self):
@@ -536,6 +554,16 @@ def test_timeline_status(zenith_env_builder: ZenithEnvBuilder):
    assert epoch_after_reboot > epoch


+@dataclass
+class SafekeeperProc:
+    """ An object representing a running safekeeper daemon. """
+    proc: 'subprocess.CompletedProcess[bytes]'
+    port: SafekeeperPort
+
+    def http_client(self) -> SafekeeperHttpClient:
+        return SafekeeperHttpClient(port=self.port.http)
+
+
 class SafekeeperEnv:
    def __init__(self,
                 repo_dir: Path,
@@ -547,7 +575,7 @@ class SafekeeperEnv:
        self.pg_bin = pg_bin
        self.num_safekeepers = num_safekeepers
        self.bin_safekeeper = os.path.join(str(zenith_binpath), 'safekeeper')
-        self.safekeepers: Optional[List[subprocess.CompletedProcess[Any]]] = None
+        self.safekeepers: Optional[List[SafekeeperProc]] = None
        self.postgres: Optional[ProposerPostgres] = None
        self.tenant_id: Optional[uuid.UUID] = None
        self.timeline_id: Optional[uuid.UUID] = None
@@ -571,7 +599,7 @@ class SafekeeperEnv:

        return self

-    def start_safekeeper(self, i):
+    def start_safekeeper(self, i) -> "SafekeeperProc":
        port = SafekeeperPort(
            pg=self.port_distributor.get_port(),
            http=self.port_distributor.get_port(),
@@ -594,10 +622,12 @@ class SafekeeperEnv:
        ]

        log.info(f'Running command "{" ".join(args)}"')
-        return subprocess.run(args, check=True)

-    def get_safekeeper_connstrs(self):
-        return ','.join([sk_proc.args[2] for sk_proc in self.safekeepers])
+        return SafekeeperProc(subprocess.run(args, check=True), port)
+
+    def get_safekeeper_connstrs(self) -> str:
+        assert self.safekeepers is not None
+        return ','.join([sk.proc.args[2] for sk in self.safekeepers])

    def create_postgres(self):
        pgdata_dir = os.path.join(self.repo_dir, "proposer_pgdata")
@@ -629,8 +659,8 @@ class SafekeeperEnv:
        if self.postgres is not None:
            self.postgres.stop()
        if self.safekeepers is not None:
-            for sk_proc in self.safekeepers:
-                self.kill_safekeeper(sk_proc.args[6])
+            for sk in self.safekeepers:
+                self.kill_safekeeper(sk.proc.args[6])


 def test_safekeeper_without_pageserver(test_output_dir: str,
--- a/test_runner/batch_others/test_wal_restore.py
+++ b/test_runner/batch_others/test_wal_restore.py
@@ -1,38 +0,0 @@
-import os
-import subprocess
-
-from fixtures.utils import mkdir_if_needed
-from fixtures.zenith_fixtures import (ZenithEnvBuilder,
-                                      VanillaPostgres,
-                                      PortDistributor,
-                                      PgBin,
-                                      base_dir,
-                                      vanilla_pg,
-                                      pg_distrib_dir)
-from fixtures.log_helper import log
-
-
-def test_wal_restore(zenith_env_builder: ZenithEnvBuilder,
-                     test_output_dir,
-                     port_distributor: PortDistributor):
-    zenith_env_builder.num_safekeepers = 1
-    env = zenith_env_builder.init_start()
-    env.zenith_cli.create_branch("test_wal_restore")
-    pg = env.postgres.create_start('test_wal_restore')
-    pg.safe_psql("create table t as select generate_series(1,1000000)")
-    tenant_id = pg.safe_psql("show zenith.zenith_tenant")[0][0]
-    env.zenith_cli.pageserver_stop()
-    port = port_distributor.get_port()
-    data_dir = os.path.join(test_output_dir, 'pgsql.restored')
-    restored = VanillaPostgres(data_dir, PgBin(test_output_dir), port)
-    subprocess.call([
-        'bash',
-        os.path.join(base_dir, 'zenith_utils/scripts/restore_from_wal.sh'),
-        os.path.join(pg_distrib_dir, 'bin'),
-        os.path.join(test_output_dir, 'repo/safekeepers/sk1/{}/*'.format(tenant_id)),
-        data_dir,
-        str(port)
-    ])
-    restored.start()
-    assert restored.safe_psql('select count(*) from t') == [(1000000, )]
-    restored.stop()
--- a/test_runner/fixtures/zenith_fixtures.py
+++ b/test_runner/fixtures/zenith_fixtures.py
@@ -638,13 +638,6 @@ class ZenithEnv:
        """ Get list of safekeeper endpoints suitable for wal_acceptors GUC  """
        return ','.join([f'localhost:{wa.port.pg}' for wa in self.safekeepers])

-    def run_psbench(self, timeline):
-        wal_metadata_filename = os.path.join(self.repo_dir, "wal_metadata.log")
-        psbench_binpath = os.path.join(str(zenith_binpath), 'psbench')
-        tenant_hex = self.initial_tenant.hex
-        args = [psbench_binpath, wal_metadata_filename, tenant_hex, timeline]
-        return subprocess.run(args, capture_output=True).stdout.decode("UTF-8").strip()
-
    @cached_property
    def auth_keys(self) -> AuthKeys:
        pub = (Path(self.repo_dir) / 'auth_public_key.pem').read_bytes()
@@ -1707,6 +1700,7 @@ class SafekeeperMetrics:
    # As a consequence, values may differ from real original int64s.
    flush_lsn_inexact: Dict[Tuple[str, str], int] = field(default_factory=dict)
    commit_lsn_inexact: Dict[Tuple[str, str], int] = field(default_factory=dict)
+    flush_wal_count: Dict[Tuple[str, str], int] = field(default_factory=dict)


 class SafekeeperHttpClient(requests.Session):
@@ -1741,6 +1735,11 @@ class SafekeeperHttpClient(requests.Session):
                all_metrics_text,
                re.MULTILINE):
            metrics.commit_lsn_inexact[(match.group(1), match.group(2))] = int(match.group(3))
+        for match in re.finditer(
+                r'^safekeeper_flush_wal_seconds_count{tenant_id="([0-9a-f]+)",timeline_id="([0-9a-f]+)"} (\S+)$',
+                all_metrics_text,
+                re.MULTILINE):
+            metrics.flush_wal_count[(match.group(1), match.group(2))] = int(match.group(3))
        return metrics


--- a/test_runner/performance/test_pageserver.py
+++ b/test_runner/performance/test_pageserver.py
@@ -1,48 +0,0 @@
-from contextlib import closing
-from fixtures.zenith_fixtures import ZenithEnv, PgBin
-from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
-
-
-def test_get_page(zenith_simple_env: ZenithEnv,
-                  zenbenchmark: ZenithBenchmarker,
-                  pg_bin: PgBin):
-    env = zenith_simple_env
-    env.zenith_cli.create_branch("test_pageserver", "empty")
-    pg = env.postgres.create_start('test_pageserver')
-    tenant_hex = env.initial_tenant.hex
-    timeline = pg.safe_psql("SHOW zenith.zenith_timeline")[0][0]
-
-    # Long-lived cursor, useful for flushing
-    psconn = env.pageserver.connect()
-    pscur = psconn.cursor()
-
-    with closing(pg.connect()) as conn:
-        with conn.cursor() as cur:
-            workload = "pgbench"
-
-            print(f"Running workload {workload}")
-            if workload == "hot page":
-                cur.execute('create table t (i integer);')
-                cur.execute('insert into t values (0);')
-                for i in range(100000):
-                    cur.execute(f'update t set i = {i};')
-            elif workload == "pgbench":
-                pg_bin.run_capture(['pgbench', '-s5', '-i', pg.connstr()])
-                pg_bin.run_capture(['pgbench', '-c1', '-t5000', pg.connstr()])
-            elif workload == "pgbench big":
-                pg_bin.run_capture(['pgbench', '-s100', '-i', pg.connstr()])
-                pg_bin.run_capture(['pgbench', '-c1', '-t100000', pg.connstr()])
-            elif workload == "pgbench long":
-                pg_bin.run_capture(['pgbench', '-s100', '-i', pg.connstr()])
-                pg_bin.run_capture(['pgbench', '-c1', '-t1000000', pg.connstr()])
-
-            pscur.execute(f"checkpoint {env.initial_tenant.hex} {timeline} 0")
-
-    output = env.run_psbench(timeline)
-    for line in output.split("\n"):
-        tokens = line.split(" ")
-        report = tokens[0]
-        name = tokens[1]
-        value = tokens[2]
-        unit = tokens[3] if len(tokens) > 3 else ""
-        zenbenchmark.record(name, value, unit, report=report)
--- a/test_runner/performance/test_random_writes.py
+++ b/test_runner/performance/test_random_writes.py
@@ -49,15 +49,7 @@ def test_random_writes(zenith_with_baseline: PgCompare):
                        count integer default 0
                    );
                """)
-
-                # Insert n_rows in batches to avoid query timeouts
-                rows_inserted = 0
-                while rows_inserted < n_rows:
-                    rows_to_insert = min(1000 * 1000, n_rows - rows_inserted)
-                    low = rows_inserted + 1
-                    high = rows_inserted + rows_to_insert
-                    cur.execute(f"INSERT INTO Big (pk) values (generate_series({low},{high}))")
-                    rows_inserted += rows_to_insert
+                cur.execute(f"INSERT INTO Big (pk) values (generate_series(1,{n_rows}))")

            # Get table size (can't be predicted because padding and alignment)
            cur.execute("SELECT pg_relation_size('Big');")
--- a/test_runner/performance/test_walproposer.py
+++ b/test_runner/performance/test_walproposer.py
@@ -0,0 +1,258 @@
+from pathlib import Path
+import os
+from time import sleep
+from typing import Callable, Dict
+import uuid
+
+from fixtures.zenith_fixtures import PgBin, PgProtocol, PortDistributor
+from fixtures.log_helper import log
+from fixtures.benchmark_fixture import MetricReport, ZenithBenchmarker
+from batch_others.test_wal_acceptor import ProposerPostgres, SafekeeperEnv
+from fixtures.utils import lsn_from_hex, mkdir_if_needed
+
+pytest_plugins = ("fixtures.zenith_fixtures", "fixtures.benchmark_fixture")
+
+
+def test_walproposer_pgbench(test_output_dir: str,
+                             port_distributor: PortDistributor,
+                             pg_bin: PgBin,
+                             zenbenchmark: ZenithBenchmarker):
+    # Create the environment in the test-specific output dir
+    repo_dir = Path(os.path.join(test_output_dir, "repo"))
+
+    env = SafekeeperEnv(
+        repo_dir,
+        port_distributor,
+        pg_bin,
+        num_safekeepers=1,
+    )
+
+    with env:
+        env.init()
+        assert env.postgres is not None
+
+        def calc_flushes() -> int:
+            assert env.safekeepers is not None
+            assert env.tenant_id is not None
+            assert env.timeline_id is not None
+            metrics = env.safekeepers[0].http_client().get_metrics()
+            return int(metrics.flush_wal_count[(env.tenant_id.hex, env.timeline_id.hex)])
+
+        pgbench_env = {
+            "PGHOST": env.postgres.listen_addr,
+            "PGPORT": str(env.postgres.port),
+            "PGDATA": env.postgres.pg_data_dir_path(),
+            "PGUSER": "zenith_admin",
+            "PGDATABASE": "postgres"
+        }
+
+        run_pgbench(pg_bin, zenbenchmark, pgbench_env, calc_flushes, env.postgres)
+
+
+def test_vanilla_pgbench(test_output_dir: str,
+                         port_distributor: PortDistributor,
+                         pg_bin: PgBin,
+                         zenbenchmark: ZenithBenchmarker):
+    # Create the environment in the test-specific output dir
+    repo_dir = os.path.join(test_output_dir, "repo")
+    mkdir_if_needed(repo_dir)
+
+    pgdata_master = os.path.join(repo_dir, "pgdata_master")
+    pgdata_replica = os.path.join(repo_dir, "pgdata_replica")
+
+    master = ProposerPostgres(pgdata_master,
+                              pg_bin,
+                              uuid.uuid4(),
+                              uuid.uuid4(),
+                              "127.0.0.1",
+                              port_distributor.get_port())
+
+    common_config = [
+        "wal_keep_size=10TB\n",
+        "shared_preload_libraries=zenith\n",
+        "zenith.page_server_connstring=''\n",
+        "synchronous_commit=on\n",
+        "max_wal_senders=10\n",
+        "wal_log_hints=on\n",
+        "max_replication_slots=10\n",
+        "hot_standby=on\n",
+        "min_wal_size=20GB\n",
+        "max_wal_size=40GB\n",
+        "checkpoint_timeout=60min\n",
+        "log_checkpoints=on\n",
+        "max_connections=100\n",
+        "wal_sender_timeout=0\n",
+        "wal_level=replica\n",
+    ]
+
+    master.initdb()
+    mkdir_if_needed(master.pg_data_dir_path())
+    with open(master.config_file_path(), "w") as f:
+        cfg = [
+            "fsync=off\n",
+            f"listen_addresses = '{master.listen_addr}'\n",
+            f"port = '{master.port}'\n",
+            "synchronous_standby_names = 'ANY 1 (s1)'\n",
+        ] + common_config
+
+        f.writelines(cfg)
+
+    with open(os.path.join(pgdata_master, "pg_hba.conf"), "w") as f:
+        pg_hba = """
+host     all             all             0.0.0.0/0               trust
+host     all             all             ::/0                    trust
+local    all             all                                     trust
+host     all             all        127.0.0.1/32                 trust
+host     all             all        ::1/128                      trust
+host      replication     all             0.0.0.0/0               trust
+host      replication     all             ::/0                    trust
+# Allow replication connections from localhost, by a user with the
+# replication privilege.
+local   replication     all                                     trust
+host    replication     all             127.0.0.1/32            trust
+host    replication     all             ::1/128                 trust
+"""
+        f.writelines(pg_hba)
+
+    master_env = {
+        "PGHOST": master.listen_addr,
+        "PGPORT": str(master.port),
+        "PGDATA": master.pg_data_dir_path(),
+        "PGUSER": "zenith_admin",
+        "PGDATABASE": "postgres"
+    }
+
+    master.start()
+    master.safe_psql("SELECT pg_create_physical_replication_slot('s1');")
+    pg_bin.run(["pg_basebackup", "-D", pgdata_replica], env=master_env)
+
+    replica = ProposerPostgres(pgdata_replica,
+                               pg_bin,
+                               uuid.uuid4(),
+                               uuid.uuid4(),
+                               "127.0.0.1",
+                               port_distributor.get_port())
+
+    with open(replica.config_file_path(), "w") as f:
+        cfg = [
+            "fsync=on\n",
+            f"listen_addresses = '{replica.listen_addr}'\n",
+            f"port = '{replica.port}'\n",
+            "primary_slot_name = 's1'\n",
+            f"primary_conninfo = 'application_name=s1 user=zenith_admin host={master.listen_addr} channel_binding=disable port={master.port} sslmode=disable sslcompression=0 sslsni=1 ssl_min_protocol_version=TLSv1.2 gssencmode=disable krbsrvname=postgres target_session_attrs=any'\n",
+        ] + common_config
+
+        f.writelines(cfg)
+
+    with open(os.path.join(pgdata_replica, "standby.signal"), "w") as f:
+        pass
+
+    replica.start()
+
+    def calc_flushes() -> int:
+        return int(replica.safe_psql("SELECT wal_sync FROM pg_stat_wal")[0][0])
+
+    run_pgbench(pg_bin, zenbenchmark, master_env, calc_flushes, master)
+
+    replica.stop()
+    master.stop()
+
+
+def run_pgbench(pg_bin: PgBin,
+                zenbenchmark: ZenithBenchmarker,
+                pgbench_env: Dict[str, str],
+                calc_flushes: Callable[[], int],
+                postgres: PgProtocol,
+                scale=200,
+                conns_count=32,
+                pgbench_time=60):
+
+    step0_lsn = lsn_from_hex(postgres.safe_psql('select pg_current_wal_insert_lsn()')[0][0])
+    step0_flush_cnt = calc_flushes()
+    log.info(f"step0_lsn: {step0_lsn}")
+    log.info(f"step0_flush_cnt: {step0_flush_cnt}")
+
+    cmd = ["pgbench", "-i", "-s", str(scale)]
+    basepath = pg_bin.run_capture(cmd, pgbench_env)
+    pgbench_init = basepath + '.stderr'
+
+    with open(pgbench_init, 'r') as stdout_f:
+        stdout = stdout_f.readlines()
+        stats_str = stdout[-1]
+        log.info(stats_str)
+
+    init_seconds = float(stats_str.split()[2])
+
+    zenbenchmark.record("pgbench_init", init_seconds, unit="s", report=MetricReport.LOWER_IS_BETTER)
+
+    wal_init_size = get_dir_size(os.path.join(pgbench_env["PGDATA"], 'pg_wal'))
+    zenbenchmark.record('wal_init_size',
+                        wal_init_size / (1024 * 1024),
+                        'MB',
+                        report=MetricReport.LOWER_IS_BETTER)
+
+    step1_lsn = lsn_from_hex(postgres.safe_psql('select pg_current_wal_insert_lsn()')[0][0])
+    step1_flush_cnt = calc_flushes()
+    log.info(f"step1_lsn: {step1_lsn}")
+    log.info(f"step1_flush_cnt: {step1_flush_cnt}")
+
+    zenbenchmark.record("init_wal_bytes_per_fsync",
+                        (step1_lsn - step0_lsn) / (step1_flush_cnt - step0_flush_cnt),
+                        unit="b",
+                        report=MetricReport.HIGHER_IS_BETTER)
+
+    cmd = [
+        "pgbench",
+        "-c",
+        str(conns_count),
+        "-N",
+        "-P",
+        "1",
+        "-T",
+        str(pgbench_time),
+    ]
+    basepath = pg_bin.run_capture(cmd, pgbench_env)
+    pgbench_run = basepath + '.stdout'
+
+    with open(pgbench_run, 'r') as stdout_f:
+        stdout = stdout_f.readlines()
+        for line in stdout:
+            if "number of transactions actually processed:" in line:
+                transactions_processed = int(line.split()[-1])
+        stats_str = stdout[-1]
+        log.info(stats_str)
+
+    pgbench_tps = float(stats_str.split()[2])
+
+    step2_lsn = lsn_from_hex(postgres.safe_psql('select pg_current_wal_insert_lsn()')[0][0])
+    step2_flush_cnt = calc_flushes()
+    log.info(f"step2_lsn: {step2_lsn}")
+    log.info(f"step2_flush_cnt: {step2_flush_cnt}")
+
+    zenbenchmark.record("tps_pgbench", pgbench_tps, unit="", report=MetricReport.HIGHER_IS_BETTER)
+
+    zenbenchmark.record("tx_wal_bytes_per_fsync",
+                        (step2_lsn - step1_lsn) / (step2_flush_cnt - step1_flush_cnt),
+                        unit="b",
+                        report=MetricReport.HIGHER_IS_BETTER)
+
+    zenbenchmark.record("txes_per_fsync",
+                        transactions_processed / (step2_flush_cnt - step1_flush_cnt),
+                        unit="",
+                        report=MetricReport.HIGHER_IS_BETTER)
+
+    wal_size = get_dir_size(os.path.join(pgbench_env["PGDATA"], 'pg_wal'))
+    zenbenchmark.record('wal_size',
+                        wal_size / (1024 * 1024),
+                        'MB',
+                        report=MetricReport.LOWER_IS_BETTER)
+
+
+def get_dir_size(path: str) -> int:
+    """Return size in bytes."""
+    totalbytes = 0
+    for root, dirs, files in os.walk(path):
+        for name in files:
+            totalbytes += os.path.getsize(os.path.join(root, name))
+
+    return totalbytes
--- a/vendor/postgres
+++ b/vendor/postgres
--- a/walkeeper/Cargo.toml
+++ b/walkeeper/Cargo.toml
@@ -14,7 +14,8 @@ serde_json = "1"
 tracing = "0.1.27"
 clap = "3.0"
 daemonize = "0.4.1"
-tokio = { version = "1.17", features = ["macros", "fs"] }
+rust-s3 = { version = "0.28", default-features = false, features = ["no-verify-ssl", "tokio-rustls-tls"] }
+tokio = { version = "1.17", features = ["macros"] }
 postgres-protocol = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 anyhow = "1.0"
@@ -29,9 +30,6 @@ hex = "0.4.3"
 const_format = "0.2.21"
 tokio-postgres = { git = "https://github.com/zenithdb/rust-postgres.git", rev="2949d98df52587d562986aad155dd4e889e408b7" }
 etcd-client = "0.8.3"
-tokio-util = { version = "0.7", features = ["io"] }
-rusoto_core = "0.47"
-rusoto_s3 = "0.47"

 postgres_ffi = { path = "../postgres_ffi" }
 zenith_metrics = { path = "../zenith_metrics" }
--- a/walkeeper/src/http/routes.rs
+++ b/walkeeper/src/http/routes.rs
@@ -31,7 +31,7 @@ struct SafekeeperStatus {
 async fn status_handler(request: Request<Body>) -> Result<Response<Body>, ApiError> {
    let conf = get_conf(&request);
    let status = SafekeeperStatus { id: conf.my_id };
-    json_response(StatusCode::OK, status)
+    Ok(json_response(StatusCode::OK, status)?)
 }

 fn get_conf(request: &Request<Body>) -> &SafeKeeperConf {
@@ -106,7 +106,7 @@ async fn timeline_status_handler(request: Request<Body>) -> Result<Response<Body
        remote_consistent_lsn: inmem.remote_consistent_lsn,
        flush_lsn,
    };
-    json_response(StatusCode::OK, status)
+    Ok(json_response(StatusCode::OK, status)?)
 }

 async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<Body>, ApiError> {
@@ -119,7 +119,7 @@ async fn timeline_create_handler(mut request: Request<Body>) -> Result<Response<
    GlobalTimelines::create(get_conf(&request), zttid, request_data.peer_ids)
        .map_err(ApiError::from_err)?;

-    json_response(StatusCode::CREATED, ())
+    Ok(json_response(StatusCode::CREATED, ())?)
 }

 /// Safekeeper http router.
--- a/walkeeper/src/s3_offload.rs
+++ b/walkeeper/src/s3_offload.rs
@@ -2,19 +2,19 @@
 // Offload old WAL segments to S3 and remove them locally
 //

-use anyhow::Context;
+use anyhow::Result;
 use postgres_ffi::xlog_utils::*;
-use rusoto_core::credential::StaticProvider;
-use rusoto_core::{HttpClient, Region};
-use rusoto_s3::{ListObjectsV2Request, PutObjectRequest, S3Client, StreamingBody, S3};
+use s3::bucket::Bucket;
+use s3::creds::Credentials;
+use s3::region::Region;
 use std::collections::HashSet;
 use std::env;
+use std::fs::{self, File};
+use std::io::prelude::*;
 use std::path::Path;
 use std::time::SystemTime;
-use tokio::fs::{self, File};
 use tokio::runtime;
 use tokio::time::sleep;
-use tokio_util::io::ReaderStream;
 use tracing::*;
 use walkdir::WalkDir;

@@ -39,12 +39,11 @@ pub fn thread_main(conf: SafeKeeperConf) {
 }

 async fn offload_files(
-    client: &S3Client,
-    bucket_name: &str,
+    bucket: &Bucket,
    listing: &HashSet<String>,
    dir_path: &Path,
    conf: &SafeKeeperConf,
-) -> anyhow::Result<u64> {
+) -> Result<u64> {
    let horizon = SystemTime::now() - conf.ttl.unwrap();
    let mut n: u64 = 0;
    for entry in WalkDir::new(dir_path) {
@@ -58,17 +57,12 @@ async fn offload_files(
            let relpath = path.strip_prefix(&conf.workdir).unwrap();
            let s3path = String::from("walarchive/") + relpath.to_str().unwrap();
            if !listing.contains(&s3path) {
-                let file = File::open(&path).await?;
-                client
-                    .put_object(PutObjectRequest {
-                        body: Some(StreamingBody::new(ReaderStream::new(file))),
-                        bucket: bucket_name.to_string(),
-                        key: s3path,
-                        ..PutObjectRequest::default()
-                    })
-                    .await?;
+                let mut file = File::open(&path)?;
+                let mut content = Vec::new();
+                file.read_to_end(&mut content)?;
+                bucket.put_object(s3path, &content).await?;

-                fs::remove_file(&path).await?;
+                fs::remove_file(&path)?;
                n += 1;
            }
        }
@@ -76,59 +70,35 @@ async fn offload_files(
    Ok(n)
 }

-async fn main_loop(conf: &SafeKeeperConf) -> anyhow::Result<()> {
+async fn main_loop(conf: &SafeKeeperConf) -> Result<()> {
    let region = Region::Custom {
-        name: env::var("S3_REGION").context("S3_REGION env var is not set")?,
-        endpoint: env::var("S3_ENDPOINT").context("S3_ENDPOINT env var is not set")?,
+        region: env::var("S3_REGION").unwrap(),
+        endpoint: env::var("S3_ENDPOINT").unwrap(),
    };
+    let credentials = Credentials::new(
+        Some(&env::var("S3_ACCESSKEY").unwrap()),
+        Some(&env::var("S3_SECRET").unwrap()),
+        None,
+        None,
+        None,
+    )
+    .unwrap();

-    let client = S3Client::new_with(
-        HttpClient::new().context("Failed to create S3 http client")?,
-        StaticProvider::new_minimal(
-            env::var("S3_ACCESSKEY").context("S3_ACCESSKEY env var is not set")?,
-            env::var("S3_SECRET").context("S3_SECRET env var is not set")?,
-        ),
-        region,
-    );
-
-    let bucket_name = "zenith-testbucket";
+    // Create Bucket in REGION for BUCKET
+    let bucket = Bucket::new_with_path_style("zenith-testbucket", region, credentials)?;

    loop {
-        let listing = gather_wal_entries(&client, bucket_name).await?;
-        let n = offload_files(&client, bucket_name, &listing, &conf.workdir, conf).await?;
+        // List out contents of directory
+        let results = bucket
+            .list("walarchive/".to_string(), Some("".to_string()))
+            .await?;
+        let listing = results
+            .iter()
+            .flat_map(|b| b.contents.iter().map(|o| o.key.clone()))
+            .collect();
+
+        let n = offload_files(&bucket, &listing, &conf.workdir, conf).await?;
        info!("Offload {} files to S3", n);
        sleep(conf.ttl.unwrap()).await;
    }
 }
-
-async fn gather_wal_entries(
-    client: &S3Client,
-    bucket_name: &str,
-) -> anyhow::Result<HashSet<String>> {
-    let mut document_keys = HashSet::new();
-
-    let mut continuation_token = None::<String>;
-    loop {
-        let response = client
-            .list_objects_v2(ListObjectsV2Request {
-                bucket: bucket_name.to_string(),
-                prefix: Some("walarchive/".to_string()),
-                continuation_token,
-                ..ListObjectsV2Request::default()
-            })
-            .await?;
-        document_keys.extend(
-            response
-                .contents
-                .unwrap_or_default()
-                .into_iter()
-                .filter_map(|o| o.key),
-        );
-
-        continuation_token = response.continuation_token;
-        if continuation_token.is_none() {
-            break;
-        }
-    }
-    Ok(document_keys)
-}
--- a/walkeeper/src/safekeeper.rs
+++ b/walkeeper/src/safekeeper.rs
@@ -517,16 +517,14 @@ where
    pub fn new(
        ztli: ZTimelineId,
        control_store: CTRL,
-        mut wal_store: WAL,
+        wal_store: WAL,
        state: SafeKeeperState,
-    ) -> Result<SafeKeeper<CTRL, WAL>> {
+    ) -> SafeKeeper<CTRL, WAL> {
        if state.timeline_id != ZTimelineId::from([0u8; 16]) && ztli != state.timeline_id {
-            bail!("Calling SafeKeeper::new with inconsistent ztli ({}) and SafeKeeperState.server.timeline_id ({})", ztli, state.timeline_id);
+            panic!("Calling SafeKeeper::new with inconsistent ztli ({}) and SafeKeeperState.server.timeline_id ({})", ztli, state.timeline_id);
        }

-        wal_store.init_storage(&state)?;
-
-        Ok(SafeKeeper {
+        SafeKeeper {
            metrics: SafeKeeperMetrics::new(state.tenant_id, ztli),
            global_commit_lsn: state.commit_lsn,
            epoch_start_lsn: Lsn(0),
@@ -539,7 +537,7 @@ where
            s: state,
            control_store,
            wal_store,
-        })
+        }
    }

    /// Get history of term switches for the available WAL
@@ -879,7 +877,7 @@ mod tests {
        };
        let wal_store = DummyWalStore { lsn: Lsn(0) };
        let ztli = ZTimelineId::from([0u8; 16]);
-        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty()).unwrap();
+        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty());

        // check voting for 1 is ok
        let vote_request = ProposerAcceptorMessage::VoteRequest(VoteRequest { term: 1 });
@@ -894,7 +892,7 @@ mod tests {
        let storage = InMemoryState {
            persisted_state: state.clone(),
        };
-        sk = SafeKeeper::new(ztli, storage, sk.wal_store, state).unwrap();
+        sk = SafeKeeper::new(ztli, storage, sk.wal_store, state);

        // and ensure voting second time for 1 is not ok
        vote_resp = sk.process_msg(&vote_request);
@@ -911,7 +909,7 @@ mod tests {
        };
        let wal_store = DummyWalStore { lsn: Lsn(0) };
        let ztli = ZTimelineId::from([0u8; 16]);
-        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty()).unwrap();
+        let mut sk = SafeKeeper::new(ztli, storage, wal_store, SafeKeeperState::empty());

        let mut ar_hdr = AppendRequestHeader {
            term: 1,
--- a/walkeeper/src/timeline.rs
+++ b/walkeeper/src/timeline.rs
@@ -100,7 +100,7 @@ impl SharedState {
        let state = SafeKeeperState::new(zttid, peer_ids);
        let control_store = control_file::FileStorage::new(zttid, conf);
        let wal_store = wal_storage::PhysicalStorage::new(zttid, conf);
-        let mut sk = SafeKeeper::new(zttid.timeline_id, control_store, wal_store, state)?;
+        let mut sk = SafeKeeper::new(zttid.timeline_id, control_store, wal_store, state);
        sk.control_store.persist(&sk.s)?;

        Ok(Self {
@@ -127,7 +127,7 @@ impl SharedState {

        Ok(Self {
            notified_commit_lsn: Lsn(0),
-            sk: SafeKeeper::new(zttid.timeline_id, control_store, wal_store, state)?,
+            sk: SafeKeeper::new(zttid.timeline_id, control_store, wal_store, state),
            replicas: Vec::new(),
            active: false,
            num_computes: 0,
--- a/zenith_utils/scripts/restore_from_wal.sh
+++ b/zenith_utils/scripts/restore_from_wal.sh
@@ -1,20 +0,0 @@
-PG_BIN=$1
-WAL_PATH=$2
-DATA_DIR=$3
-PORT=$4
-SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
-rm -fr $DATA_DIR
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -D $DATA_DIR --sysid=$SYSID
-echo port=$PORT >> $DATA_DIR/postgresql.conf
-REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
-declare -i WAL_SIZE=$REDO_POS+114
-$PG_BIN/pg_ctl -D $DATA_DIR -l logfile start
-$PG_BIN/pg_ctl -D $DATA_DIR -l logfile stop -m immediate
-cp $DATA_DIR/pg_wal/000000010000000000000001 .
-cp $WAL_PATH/* $DATA_DIR/pg_wal/
-if [ -f $DATA_DIR/pg_wal/*.partial ]
-then
-	(cd $DATA_DIR/pg_wal ; for partial in \*.partial ; do mv $partial `basename $partial .partial` ; done)
-fi
-dd if=000000010000000000000001 of=$DATA_DIR/pg_wal/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc
-rm -f 000000010000000000000001
--- a/zenith_utils/scripts/restore_from_wal_archive.sh
+++ b/zenith_utils/scripts/restore_from_wal_archive.sh
@@ -1,20 +0,0 @@
-PG_BIN=$1
-WAL_PATH=$2
-DATA_DIR=$3
-PORT=$4
-SYSID=`od -A n -j 24 -N 8 -t d8 $WAL_PATH/000000010000000000000002* | cut -c 3-`
-rm -fr $DATA_DIR /tmp/pg_wals
-mkdir /tmp/pg_wals
-env -i LD_LIBRARY_PATH=$PG_BIN/../lib $PG_BIN/initdb -E utf8 -U zenith_admin -D $DATA_DIR --sysid=$SYSID
-echo port=$PORT >> $DATA_DIR/postgresql.conf
-REDO_POS=0x`$PG_BIN/pg_controldata -D $DATA_DIR | fgrep "REDO location"| cut -c 42-`
-declare -i WAL_SIZE=$REDO_POS+114
-cp $WAL_PATH/* /tmp/pg_wals
-if [ -f $DATA_DIR/pg_wal/*.partial ]
-then
-	(cd /tmp/pg_wals ; for partial in \*.partial ; do  mv $partial `basename $partial .partial` ; done)
-fi
-dd if=$DATA_DIR/pg_wal/000000010000000000000001 of=/tmp/pg_wals/000000010000000000000001 bs=$WAL_SIZE count=1 conv=notrunc
-echo > $DATA_DIR/recovery.signal
-rm -f $DATA_DIR/pg_wal/*
-echo "restore_command = 'cp /tmp/pg_wals/%f %p'" >> $DATA_DIR/postgresql.conf
--- a/zenith_utils/src/http/error.rs
+++ b/zenith_utils/src/http/error.rs
@@ -17,9 +17,6 @@ pub enum ApiError {
    #[error("NotFound: {0}")]
    NotFound(String),

-    #[error("Conflict: {0}")]
-    Conflict(String),
-
    #[error(transparent)]
    InternalServerError(#[from] anyhow::Error),
 }
@@ -45,9 +42,6 @@ impl ApiError {
            ApiError::NotFound(_) => {
                HttpErrorBody::response_from_msg_and_status(self.to_string(), StatusCode::NOT_FOUND)
            }
-            ApiError::Conflict(_) => {
-                HttpErrorBody::response_from_msg_and_status(self.to_string(), StatusCode::CONFLICT)
-            }
            ApiError::InternalServerError(err) => HttpErrorBody::response_from_msg_and_status(
                err.to_string(),
                StatusCode::INTERNAL_SERVER_ERROR,
--- a/zenith_utils/src/http/json.rs
+++ b/zenith_utils/src/http/json.rs
@@ -10,8 +10,8 @@ pub async fn json_request<T: for<'de> Deserialize<'de>>(
    let whole_body = hyper::body::aggregate(request.body_mut())
        .await
        .map_err(ApiError::from_err)?;
-    serde_json::from_reader(whole_body.reader())
-        .map_err(|err| ApiError::BadRequest(format!("Failed to parse json request {}", err)))
+    Ok(serde_json::from_reader(whole_body.reader())
+        .map_err(|err| ApiError::BadRequest(format!("Failed to parse json request {}", err)))?)
 }

 pub fn json_response<T: Serialize>(